1
//===-- AMDILPeepholeOptimizer.cpp - AMDGPU Peephole optimizations ---------===//
3
// The LLVM Compiler Infrastructure
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
9
//==-----------------------------------------------------------------------===//
11
#define DEBUG_TYPE "PeepholeOpt"
13
#define DEBUGME (DebugFlag && isCurrentDebugType(DEBUG_TYPE))
18
#include "AMDILDevices.h"
19
#include "AMDGPUInstrInfo.h"
20
#include "llvm/ADT/Statistic.h"
21
#include "llvm/ADT/StringExtras.h"
22
#include "llvm/ADT/StringRef.h"
23
#include "llvm/ADT/Twine.h"
24
#include "llvm/IR/Constants.h"
25
#include "llvm/CodeGen/MachineFunction.h"
26
#include "llvm/CodeGen/MachineFunctionAnalysis.h"
27
#include "llvm/IR/Function.h"
28
#include "llvm/IR/Instructions.h"
29
#include "llvm/IR/Module.h"
30
#include "llvm/Support/Debug.h"
31
#include "llvm/Support/MathExtras.h"
36
STATISTIC(PointerAssignments, "Number of dynamic pointer "
37
"assigments discovered");
38
STATISTIC(PointerSubtract, "Number of pointer subtractions discovered");
42
// The Peephole optimization pass is used to do simple last minute optimizations
43
// that are required for correct code or to remove redundant functions
48
class LLVM_LIBRARY_VISIBILITY AMDGPUPeepholeOpt : public FunctionPass {
52
AMDGPUPeepholeOpt(TargetMachine &tm);
54
const char *getPassName() const;
55
bool runOnFunction(Function &F);
56
bool doInitialization(Module &M);
57
bool doFinalization(Module &M);
58
void getAnalysisUsage(AnalysisUsage &AU) const;
61
// Function to initiate all of the instruction level optimizations.
62
bool instLevelOptimizations(BasicBlock::iterator *inst);
63
// Quick check to see if we need to dump all of the pointers into the
64
// arena. If this is correct, then we set all pointers to exist in arena. This
65
// is a workaround for aliasing of pointers in a struct/union.
66
bool dumpAllIntoArena(Function &F);
67
// Because I don't want to invalidate any pointers while in the
68
// safeNestedForEachFunction. I push atomic conversions to a vector and handle
69
// it later. This function does the conversions if required.
70
void doAtomicConversionIfNeeded(Function &F);
71
// Because __amdil_is_constant cannot be properly evaluated if
72
// optimizations are disabled, the call's are placed in a vector
73
// and evaluated after the __amdil_image* functions are evaluated
74
// which should allow the __amdil_is_constant function to be
75
// evaluated correctly.
76
void doIsConstCallConversionIfNeeded();
80
CodeGenOpt::Level optLevel;
81
// Run a series of tests to see if we can optimize a CALL instruction.
82
bool optimizeCallInst(BasicBlock::iterator *bbb);
83
// A peephole optimization to optimize bit extract sequences.
84
bool optimizeBitExtract(Instruction *inst);
85
// A peephole optimization to optimize bit insert sequences.
86
bool optimizeBitInsert(Instruction *inst);
87
bool setupBitInsert(Instruction *base,
91
// Expand the bit field insert instruction on versions of OpenCL that
93
bool expandBFI(CallInst *CI);
94
// Expand the bit field mask instruction on version of OpenCL that
96
bool expandBFM(CallInst *CI);
97
// On 7XX and 8XX operations, we do not have 24 bit signed operations. So in
98
// this case we need to expand them. These functions check for 24bit functions
100
bool isSigned24BitOps(CallInst *CI);
101
void expandSigned24BitOps(CallInst *CI);
102
// One optimization that can occur is that if the required workgroup size is
103
// specified then the result of get_local_size is known at compile time and
104
// can be returned accordingly.
105
bool isRWGLocalOpt(CallInst *CI);
106
// On northern island cards, the division is slightly less accurate than on
107
// previous generations, so we need to utilize a more accurate division. So we
108
// can translate the accurate divide to a normal divide on all other cards.
109
bool convertAccurateDivide(CallInst *CI);
110
void expandAccurateDivide(CallInst *CI);
111
// If the alignment is set incorrectly, it can produce really inefficient
112
// code. This checks for this scenario and fixes it if possible.
113
bool correctMisalignedMemOp(Instruction *inst);
115
// If we are in no opt mode, then we need to make sure that
116
// local samplers are properly propagated as constant propagation
117
// doesn't occur and we need to know the value of kernel defined
118
// samplers at compile time.
119
bool propagateSamplerInst(CallInst *CI);
123
// Group of functions that recursively calculate the size of a structure based
124
// on it's sub-types.
125
size_t getTypeSize(Type * const T, bool dereferencePtr = false);
126
size_t getTypeSize(StructType * const ST, bool dereferencePtr = false);
127
size_t getTypeSize(IntegerType * const IT, bool dereferencePtr = false);
128
size_t getTypeSize(FunctionType * const FT,bool dereferencePtr = false);
129
size_t getTypeSize(ArrayType * const AT, bool dereferencePtr = false);
130
size_t getTypeSize(VectorType * const VT, bool dereferencePtr = false);
131
size_t getTypeSize(PointerType * const PT, bool dereferencePtr = false);
132
size_t getTypeSize(OpaqueType * const OT, bool dereferencePtr = false);
136
const AMDGPUSubtarget *mSTM;
137
SmallVector< std::pair<CallInst *, Function *>, 16> atomicFuncs;
138
SmallVector<CallInst *, 16> isConstVec;
139
}; // class AMDGPUPeepholeOpt
140
char AMDGPUPeepholeOpt::ID = 0;
142
// A template function that has two levels of looping before calling the
143
// function with a pointer to the current iterator.
144
template<class InputIterator, class SecondIterator, class Function>
145
Function safeNestedForEach(InputIterator First, InputIterator Last,
146
SecondIterator S, Function F) {
147
for ( ; First != Last; ++First) {
148
SecondIterator sf, sl;
149
for (sf = First->begin(), sl = First->end();
159
} // anonymous namespace
163
createAMDGPUPeepholeOpt(TargetMachine &tm) {
164
return new AMDGPUPeepholeOpt(tm);
168
AMDGPUPeepholeOpt::AMDGPUPeepholeOpt(TargetMachine &tm)
169
: FunctionPass(ID), TM(tm) {
171
optLevel = TM.getOptLevel();
175
AMDGPUPeepholeOpt::~AMDGPUPeepholeOpt() {
179
AMDGPUPeepholeOpt::getPassName() const {
180
return "AMDGPU PeepHole Optimization Pass";
184
containsPointerType(Type *Ty) {
188
switch(Ty->getTypeID()) {
191
case Type::StructTyID: {
192
const StructType *ST = dyn_cast<StructType>(Ty);
193
for (StructType::element_iterator stb = ST->element_begin(),
194
ste = ST->element_end(); stb != ste; ++stb) {
195
if (!containsPointerType(*stb)) {
202
case Type::VectorTyID:
203
case Type::ArrayTyID:
204
return containsPointerType(dyn_cast<SequentialType>(Ty)->getElementType());
205
case Type::PointerTyID:
212
AMDGPUPeepholeOpt::dumpAllIntoArena(Function &F) {
213
bool dumpAll = false;
214
for (Function::const_arg_iterator cab = F.arg_begin(),
215
cae = F.arg_end(); cab != cae; ++cab) {
216
const Argument *arg = cab;
217
const PointerType *PT = dyn_cast<PointerType>(arg->getType());
221
Type *DereferencedType = PT->getElementType();
222
if (!dyn_cast<StructType>(DereferencedType)
226
if (!containsPointerType(DereferencedType)) {
229
// FIXME: Because a pointer inside of a struct/union may be aliased to
230
// another pointer we need to take the conservative approach and place all
231
// pointers into the arena until more advanced detection is implemented.
237
AMDGPUPeepholeOpt::doIsConstCallConversionIfNeeded() {
238
if (isConstVec.empty()) {
241
for (unsigned x = 0, y = isConstVec.size(); x < y; ++x) {
242
CallInst *CI = isConstVec[x];
243
Constant *CV = dyn_cast<Constant>(CI->getOperand(0));
244
Type *aType = Type::getInt32Ty(*mCTX);
245
Value *Val = (CV != NULL) ? ConstantInt::get(aType, 1)
246
: ConstantInt::get(aType, 0);
247
CI->replaceAllUsesWith(Val);
248
CI->eraseFromParent();
253
AMDGPUPeepholeOpt::doAtomicConversionIfNeeded(Function &F) {
254
// Don't do anything if we don't have any atomic operations.
255
if (atomicFuncs.empty()) {
258
// Change the function name for the atomic if it is required
259
uint32_t size = atomicFuncs.size();
260
for (uint32_t x = 0; x < size; ++x) {
261
atomicFuncs[x].first->setOperand(
262
atomicFuncs[x].first->getNumOperands()-1,
263
atomicFuncs[x].second);
267
if (mConvertAtomics) {
273
AMDGPUPeepholeOpt::runOnFunction(Function &MF) {
276
mSTM = &TM.getSubtarget<AMDGPUSubtarget>();
280
mCTX = &MF.getType()->getContext();
281
mConvertAtomics = true;
282
safeNestedForEach(MF.begin(), MF.end(), MF.begin()->begin(),
283
std::bind1st(std::mem_fun(&AMDGPUPeepholeOpt::instLevelOptimizations),
286
doAtomicConversionIfNeeded(MF);
287
doIsConstCallConversionIfNeeded();
296
AMDGPUPeepholeOpt::optimizeCallInst(BasicBlock::iterator *bbb) {
297
Instruction *inst = (*bbb);
298
CallInst *CI = dyn_cast<CallInst>(inst);
302
if (isSigned24BitOps(CI)) {
303
expandSigned24BitOps(CI);
305
CI->eraseFromParent();
308
if (propagateSamplerInst(CI)) {
311
if (expandBFI(CI) || expandBFM(CI)) {
313
CI->eraseFromParent();
316
if (convertAccurateDivide(CI)) {
317
expandAccurateDivide(CI);
319
CI->eraseFromParent();
323
StringRef calleeName = CI->getOperand(CI->getNumOperands()-1)->getName();
324
if (calleeName.startswith("__amdil_is_constant")) {
325
// If we do not have optimizations, then this
326
// cannot be properly evaluated, so we add the
327
// call instruction to a vector and process
328
// them at the end of processing after the
329
// samplers have been correctly handled.
330
if (optLevel == CodeGenOpt::None) {
331
isConstVec.push_back(CI);
334
Constant *CV = dyn_cast<Constant>(CI->getOperand(0));
335
Type *aType = Type::getInt32Ty(*mCTX);
336
Value *Val = (CV != NULL) ? ConstantInt::get(aType, 1)
337
: ConstantInt::get(aType, 0);
338
CI->replaceAllUsesWith(Val);
340
CI->eraseFromParent();
345
if (calleeName.equals("__amdil_is_asic_id_i32")) {
346
ConstantInt *CV = dyn_cast<ConstantInt>(CI->getOperand(0));
347
Type *aType = Type::getInt32Ty(*mCTX);
350
Val = ConstantInt::get(aType,
351
mSTM->device()->getDeviceFlag() & CV->getZExtValue());
353
Val = ConstantInt::get(aType, 0);
355
CI->replaceAllUsesWith(Val);
357
CI->eraseFromParent();
360
Function *F = dyn_cast<Function>(CI->getOperand(CI->getNumOperands()-1));
364
if (F->getName().startswith("__atom") && !CI->getNumUses()
365
&& F->getName().find("_xchg") == StringRef::npos) {
366
std::string buffer(F->getName().str() + "_noret");
367
F = dyn_cast<Function>(
368
F->getParent()->getOrInsertFunction(buffer, F->getFunctionType()));
369
atomicFuncs.push_back(std::make_pair(CI, F));
372
if (!mSTM->device()->isSupported(AMDGPUDeviceInfo::ArenaSegment)
373
&& !mSTM->device()->isSupported(AMDGPUDeviceInfo::MultiUAV)) {
376
if (!mConvertAtomics) {
379
StringRef name = F->getName();
380
if (name.startswith("__atom") && name.find("_g") != StringRef::npos) {
381
mConvertAtomics = false;
387
AMDGPUPeepholeOpt::setupBitInsert(Instruction *base,
393
dbgs() << "Null pointer passed into function.\n";
398
if (base->getOpcode() == Instruction::Shl) {
399
shift = dyn_cast<Constant>(base->getOperand(1));
400
} else if (base->getOpcode() == Instruction::And) {
401
mask = dyn_cast<Constant>(base->getOperand(1));
405
dbgs() << "Failed setup with no Shl or And instruction on base opcode!\n";
407
// If the base is neither a Shl or a And, we don't fit any of the patterns above.
410
src = dyn_cast<Instruction>(base->getOperand(0));
413
dbgs() << "Failed setup since the base operand is not an instruction!\n";
417
// If we find an 'and' operation, then we don't need to
418
// find the next operation as we already know the
419
// bits that are valid at this point.
423
if (src->getOpcode() == Instruction::Shl && !shift) {
424
shift = dyn_cast<Constant>(src->getOperand(1));
425
src = dyn_cast<Instruction>(src->getOperand(0));
426
} else if (src->getOpcode() == Instruction::And && !mask) {
427
mask = dyn_cast<Constant>(src->getOperand(1));
429
if (!mask && !shift) {
431
dbgs() << "Failed setup since both mask and shift are NULL!\n";
433
// Did not find a constant mask or a shift.
439
AMDGPUPeepholeOpt::optimizeBitInsert(Instruction *inst) {
443
if (!inst->isBinaryOp()) {
446
if (inst->getOpcode() != Instruction::Or) {
449
if (optLevel == CodeGenOpt::None) {
452
// We want to do an optimization on a sequence of ops that in the end equals a
453
// single ISA instruction.
454
// The base pattern for this optimization is - ((A & B) << C) | ((D & E) << F)
455
// Some simplified versions of this pattern are as follows:
456
// (A & B) | (D & E) when B & E == 0 && C == 0 && F == 0
457
// ((A & B) << C) | (D & E) when B ^ E == 0 && (1 << C) >= E
458
// (A & B) | ((D & E) << F) when B ^ E == 0 && (1 << F) >= B
459
// (A & B) | (D << F) when (1 << F) >= B
460
// (A << C) | (D & E) when (1 << C) >= E
461
if (mSTM->device()->getGeneration() == AMDGPUDeviceInfo::HD4XXX) {
462
// The HD4XXX hardware doesn't support the ubit_insert instruction.
465
Type *aType = inst->getType();
466
bool isVector = aType->isVectorTy();
468
// This optimization only works on 32bit integers.
469
if (aType->getScalarType()
470
!= Type::getInt32Ty(inst->getContext())) {
474
const VectorType *VT = dyn_cast<VectorType>(aType);
475
numEle = VT->getNumElements();
476
// We currently cannot support more than 4 elements in a intrinsic and we
477
// cannot support Vec3 types.
478
if (numEle > 4 || numEle == 3) {
482
// TODO: Handle vectors.
485
dbgs() << "!!! Vectors are not supported yet!\n";
489
Instruction *LHSSrc = NULL, *RHSSrc = NULL;
490
Constant *LHSMask = NULL, *RHSMask = NULL;
491
Constant *LHSShift = NULL, *RHSShift = NULL;
492
Instruction *LHS = dyn_cast<Instruction>(inst->getOperand(0));
493
Instruction *RHS = dyn_cast<Instruction>(inst->getOperand(1));
494
if (!setupBitInsert(LHS, LHSSrc, LHSMask, LHSShift)) {
496
dbgs() << "Found an OR Operation that failed setup!\n";
498
if (LHS) { LHS->dump(); }
499
if (LHSSrc) { LHSSrc->dump(); }
500
if (LHSMask) { LHSMask->dump(); }
501
if (LHSShift) { LHSShift->dump(); }
503
// There was an issue with the setup for BitInsert.
506
if (!setupBitInsert(RHS, RHSSrc, RHSMask, RHSShift)) {
508
dbgs() << "Found an OR Operation that failed setup!\n";
510
if (RHS) { RHS->dump(); }
511
if (RHSSrc) { RHSSrc->dump(); }
512
if (RHSMask) { RHSMask->dump(); }
513
if (RHSShift) { RHSShift->dump(); }
515
// There was an issue with the setup for BitInsert.
519
dbgs() << "Found an OR operation that can possible be optimized to ubit insert!\n";
520
dbgs() << "Op: "; inst->dump();
521
dbgs() << "LHS: "; if (LHS) { LHS->dump(); } else { dbgs() << "(None)\n"; }
522
dbgs() << "LHS Src: "; if (LHSSrc) { LHSSrc->dump(); } else { dbgs() << "(None)\n"; }
523
dbgs() << "LHS Mask: "; if (LHSMask) { LHSMask->dump(); } else { dbgs() << "(None)\n"; }
524
dbgs() << "LHS Shift: "; if (LHSShift) { LHSShift->dump(); } else { dbgs() << "(None)\n"; }
525
dbgs() << "RHS: "; if (RHS) { RHS->dump(); } else { dbgs() << "(None)\n"; }
526
dbgs() << "RHS Src: "; if (RHSSrc) { RHSSrc->dump(); } else { dbgs() << "(None)\n"; }
527
dbgs() << "RHS Mask: "; if (RHSMask) { RHSMask->dump(); } else { dbgs() << "(None)\n"; }
528
dbgs() << "RHS Shift: "; if (RHSShift) { RHSShift->dump(); } else { dbgs() << "(None)\n"; }
530
Constant *offset = NULL;
531
Constant *width = NULL;
532
uint32_t lhsMaskVal = 0, rhsMaskVal = 0;
533
uint32_t lhsShiftVal = 0, rhsShiftVal = 0;
534
uint32_t lhsMaskWidth = 0, rhsMaskWidth = 0;
535
uint32_t lhsMaskOffset = 0, rhsMaskOffset = 0;
536
lhsMaskVal = (LHSMask
537
? dyn_cast<ConstantInt>(LHSMask)->getZExtValue() : 0);
538
rhsMaskVal = (RHSMask
539
? dyn_cast<ConstantInt>(RHSMask)->getZExtValue() : 0);
540
lhsShiftVal = (LHSShift
541
? dyn_cast<ConstantInt>(LHSShift)->getZExtValue() : 0);
542
rhsShiftVal = (RHSShift
543
? dyn_cast<ConstantInt>(RHSShift)->getZExtValue() : 0);
544
lhsMaskWidth = lhsMaskVal ? CountPopulation_32(lhsMaskVal) : 32 - lhsShiftVal;
545
rhsMaskWidth = rhsMaskVal ? CountPopulation_32(rhsMaskVal) : 32 - rhsShiftVal;
546
lhsMaskOffset = lhsMaskVal ? CountTrailingZeros_32(lhsMaskVal) : lhsShiftVal;
547
rhsMaskOffset = rhsMaskVal ? CountTrailingZeros_32(rhsMaskVal) : rhsShiftVal;
548
// TODO: Handle the case of A & B | D & ~B(i.e. inverted masks).
549
if ((lhsMaskVal || rhsMaskVal) && !(lhsMaskVal ^ rhsMaskVal)) {
552
if (lhsMaskOffset >= (rhsMaskWidth + rhsMaskOffset)) {
553
offset = ConstantInt::get(aType, lhsMaskOffset, false);
554
width = ConstantInt::get(aType, lhsMaskWidth, false);
556
if (!isMask_32(lhsMaskVal) && !isShiftedMask_32(lhsMaskVal)) {
560
LHSSrc = BinaryOperator::Create(Instruction::LShr, LHSSrc, offset,
562
} else if (lhsShiftVal != lhsMaskOffset) {
563
LHSSrc = BinaryOperator::Create(Instruction::LShr, LHSSrc, offset,
567
dbgs() << "Optimizing LHS!\n";
569
} else if (rhsMaskOffset >= (lhsMaskWidth + lhsMaskOffset)) {
570
offset = ConstantInt::get(aType, rhsMaskOffset, false);
571
width = ConstantInt::get(aType, rhsMaskWidth, false);
574
if (!isMask_32(rhsMaskVal) && !isShiftedMask_32(rhsMaskVal)) {
578
LHSSrc = BinaryOperator::Create(Instruction::LShr, LHSSrc, offset,
580
} else if (rhsShiftVal != rhsMaskOffset) {
581
LHSSrc = BinaryOperator::Create(Instruction::LShr, LHSSrc, offset,
585
dbgs() << "Optimizing RHS!\n";
589
dbgs() << "Failed constraint 3!\n";
594
dbgs() << "Width: "; if (width) { width->dump(); } else { dbgs() << "(0)\n"; }
595
dbgs() << "Offset: "; if (offset) { offset->dump(); } else { dbgs() << "(0)\n"; }
596
dbgs() << "LHSSrc: "; if (LHSSrc) { LHSSrc->dump(); } else { dbgs() << "(0)\n"; }
597
dbgs() << "RHSSrc: "; if (RHSSrc) { RHSSrc->dump(); } else { dbgs() << "(0)\n"; }
599
if (!offset || !width) {
601
dbgs() << "Either width or offset are NULL, failed detection!\n";
605
// Lets create the function signature.
606
std::vector<Type *> callTypes;
607
callTypes.push_back(aType);
608
callTypes.push_back(aType);
609
callTypes.push_back(aType);
610
callTypes.push_back(aType);
611
FunctionType *funcType = FunctionType::get(aType, callTypes, false);
612
std::string name = "__amdil_ubit_insert";
613
if (isVector) { name += "_v" + itostr(numEle) + "u32"; } else { name += "_u32"; }
615
dyn_cast<Function>(inst->getParent()->getParent()->getParent()->
616
getOrInsertFunction(StringRef(name), funcType));
617
Value *Operands[4] = {
623
CallInst *CI = CallInst::Create(Func, Operands, "BitInsertOpt");
625
dbgs() << "Old Inst: ";
627
dbgs() << "New Inst: ";
631
CI->insertBefore(inst);
632
inst->replaceAllUsesWith(CI);
637
AMDGPUPeepholeOpt::optimizeBitExtract(Instruction *inst) {
641
if (!inst->isBinaryOp()) {
644
if (inst->getOpcode() != Instruction::And) {
647
if (optLevel == CodeGenOpt::None) {
650
// We want to do some simple optimizations on Shift right/And patterns. The
651
// basic optimization is to turn (A >> B) & C where A is a 32bit type, B is a
652
// value smaller than 32 and C is a mask. If C is a constant value, then the
653
// following transformation can occur. For signed integers, it turns into the
654
// function call dst = __amdil_ibit_extract(log2(C), B, A) For unsigned
655
// integers, it turns into the function call dst =
656
// __amdil_ubit_extract(log2(C), B, A) The function __amdil_[u|i]bit_extract
657
// can be found in Section 7.9 of the ATI IL spec of the stream SDK for
658
// Evergreen hardware.
659
if (mSTM->device()->getGeneration() == AMDGPUDeviceInfo::HD4XXX) {
660
// This does not work on HD4XXX hardware.
663
Type *aType = inst->getType();
664
bool isVector = aType->isVectorTy();
666
// XXX Support vector types
671
// This only works on 32bit integers
672
if (aType->getScalarType()
673
!= Type::getInt32Ty(inst->getContext())) {
677
const VectorType *VT = dyn_cast<VectorType>(aType);
678
numEle = VT->getNumElements();
679
// We currently cannot support more than 4 elements in a intrinsic and we
680
// cannot support Vec3 types.
681
if (numEle > 4 || numEle == 3) {
685
BinaryOperator *ShiftInst = dyn_cast<BinaryOperator>(inst->getOperand(0));
686
// If the first operand is not a shift instruction, then we can return as it
687
// doesn't match this pattern.
688
if (!ShiftInst || !ShiftInst->isShift()) {
691
// If we are a shift left, then we need don't match this pattern.
692
if (ShiftInst->getOpcode() == Instruction::Shl) {
695
bool isSigned = ShiftInst->isArithmeticShift();
696
Constant *AndMask = dyn_cast<Constant>(inst->getOperand(1));
697
Constant *ShrVal = dyn_cast<Constant>(ShiftInst->getOperand(1));
698
// Lets make sure that the shift value and the and mask are constant integers.
699
if (!AndMask || !ShrVal) {
702
Constant *newMaskConst;
703
Constant *shiftValConst;
705
// Handle the vector case
706
std::vector<Constant *> maskVals;
707
std::vector<Constant *> shiftVals;
708
ConstantVector *AndMaskVec = dyn_cast<ConstantVector>(AndMask);
709
ConstantVector *ShrValVec = dyn_cast<ConstantVector>(ShrVal);
710
Type *scalarType = AndMaskVec->getType()->getScalarType();
711
assert(AndMaskVec->getNumOperands() ==
712
ShrValVec->getNumOperands() && "cannot have a "
713
"combination where the number of elements to a "
714
"shift and an and are different!");
715
for (size_t x = 0, y = AndMaskVec->getNumOperands(); x < y; ++x) {
716
ConstantInt *AndCI = dyn_cast<ConstantInt>(AndMaskVec->getOperand(x));
717
ConstantInt *ShiftIC = dyn_cast<ConstantInt>(ShrValVec->getOperand(x));
718
if (!AndCI || !ShiftIC) {
721
uint32_t maskVal = (uint32_t)AndCI->getZExtValue();
722
if (!isMask_32(maskVal)) {
725
maskVal = (uint32_t)CountTrailingOnes_32(maskVal);
726
uint32_t shiftVal = (uint32_t)ShiftIC->getZExtValue();
727
// If the mask or shiftval is greater than the bitcount, then break out.
728
if (maskVal >= 32 || shiftVal >= 32) {
731
// If the mask val is greater than the the number of original bits left
732
// then this optimization is invalid.
733
if (maskVal > (32 - shiftVal)) {
736
maskVals.push_back(ConstantInt::get(scalarType, maskVal, isSigned));
737
shiftVals.push_back(ConstantInt::get(scalarType, shiftVal, isSigned));
739
newMaskConst = ConstantVector::get(maskVals);
740
shiftValConst = ConstantVector::get(shiftVals);
742
// Handle the scalar case
743
uint32_t maskVal = (uint32_t)dyn_cast<ConstantInt>(AndMask)->getZExtValue();
744
// This must be a mask value where all lower bits are set to 1 and then any
745
// bit higher is set to 0.
746
if (!isMask_32(maskVal)) {
749
maskVal = (uint32_t)CountTrailingOnes_32(maskVal);
750
// Count the number of bits set in the mask, this is the width of the
751
// resulting bit set that is extracted from the source value.
752
uint32_t shiftVal = (uint32_t)dyn_cast<ConstantInt>(ShrVal)->getZExtValue();
753
// If the mask or shift val is greater than the bitcount, then break out.
754
if (maskVal >= 32 || shiftVal >= 32) {
757
// If the mask val is greater than the the number of original bits left then
758
// this optimization is invalid.
759
if (maskVal > (32 - shiftVal)) {
762
newMaskConst = ConstantInt::get(aType, maskVal, isSigned);
763
shiftValConst = ConstantInt::get(aType, shiftVal, isSigned);
765
// Lets create the function signature.
766
std::vector<Type *> callTypes;
767
callTypes.push_back(aType);
768
callTypes.push_back(aType);
769
callTypes.push_back(aType);
770
FunctionType *funcType = FunctionType::get(aType, callTypes, false);
771
std::string name = "llvm.AMDGPU.bit.extract.u32";
773
name += ".v" + itostr(numEle) + "i32";
777
// Lets create the function.
779
dyn_cast<Function>(inst->getParent()->getParent()->getParent()->
780
getOrInsertFunction(StringRef(name), funcType));
781
Value *Operands[3] = {
782
ShiftInst->getOperand(0),
786
// Lets create the Call with the operands
787
CallInst *CI = CallInst::Create(Func, Operands, "ByteExtractOpt");
788
CI->setDoesNotAccessMemory();
789
CI->insertBefore(inst);
790
inst->replaceAllUsesWith(CI);
795
AMDGPUPeepholeOpt::expandBFI(CallInst *CI) {
799
Value *LHS = CI->getOperand(CI->getNumOperands() - 1);
800
if (!LHS->getName().startswith("__amdil_bfi")) {
803
Type* type = CI->getOperand(0)->getType();
804
Constant *negOneConst = NULL;
805
if (type->isVectorTy()) {
806
std::vector<Constant *> negOneVals;
807
negOneConst = ConstantInt::get(CI->getContext(),
808
APInt(32, StringRef("-1"), 10));
810
y = dyn_cast<VectorType>(type)->getNumElements(); x < y; ++x) {
811
negOneVals.push_back(negOneConst);
813
negOneConst = ConstantVector::get(negOneVals);
815
negOneConst = ConstantInt::get(CI->getContext(),
816
APInt(32, StringRef("-1"), 10));
818
// __amdil_bfi => (A & B) | (~A & C)
819
BinaryOperator *lhs =
820
BinaryOperator::Create(Instruction::And, CI->getOperand(0),
821
CI->getOperand(1), "bfi_and", CI);
822
BinaryOperator *rhs =
823
BinaryOperator::Create(Instruction::Xor, CI->getOperand(0), negOneConst,
825
rhs = BinaryOperator::Create(Instruction::And, rhs, CI->getOperand(2),
827
lhs = BinaryOperator::Create(Instruction::Or, lhs, rhs, "bfi_or", CI);
828
CI->replaceAllUsesWith(lhs);
833
AMDGPUPeepholeOpt::expandBFM(CallInst *CI) {
837
Value *LHS = CI->getOperand(CI->getNumOperands() - 1);
838
if (!LHS->getName().startswith("__amdil_bfm")) {
841
// __amdil_bfm => ((1 << (src0 & 0x1F)) - 1) << (src1 & 0x1f)
842
Constant *newMaskConst = NULL;
843
Constant *newShiftConst = NULL;
844
Type* type = CI->getOperand(0)->getType();
845
if (type->isVectorTy()) {
846
std::vector<Constant*> newMaskVals, newShiftVals;
847
newMaskConst = ConstantInt::get(Type::getInt32Ty(*mCTX), 0x1F);
848
newShiftConst = ConstantInt::get(Type::getInt32Ty(*mCTX), 1);
850
y = dyn_cast<VectorType>(type)->getNumElements(); x < y; ++x) {
851
newMaskVals.push_back(newMaskConst);
852
newShiftVals.push_back(newShiftConst);
854
newMaskConst = ConstantVector::get(newMaskVals);
855
newShiftConst = ConstantVector::get(newShiftVals);
857
newMaskConst = ConstantInt::get(Type::getInt32Ty(*mCTX), 0x1F);
858
newShiftConst = ConstantInt::get(Type::getInt32Ty(*mCTX), 1);
860
BinaryOperator *lhs =
861
BinaryOperator::Create(Instruction::And, CI->getOperand(0),
862
newMaskConst, "bfm_mask", CI);
863
lhs = BinaryOperator::Create(Instruction::Shl, newShiftConst,
865
lhs = BinaryOperator::Create(Instruction::Sub, lhs,
866
newShiftConst, "bfm_sub", CI);
867
BinaryOperator *rhs =
868
BinaryOperator::Create(Instruction::And, CI->getOperand(1),
869
newMaskConst, "bfm_mask", CI);
870
lhs = BinaryOperator::Create(Instruction::Shl, lhs, rhs, "bfm_shl", CI);
871
CI->replaceAllUsesWith(lhs);
876
AMDGPUPeepholeOpt::instLevelOptimizations(BasicBlock::iterator *bbb) {
877
Instruction *inst = (*bbb);
878
if (optimizeCallInst(bbb)) {
881
if (optimizeBitExtract(inst)) {
884
if (optimizeBitInsert(inst)) {
887
if (correctMisalignedMemOp(inst)) {
893
AMDGPUPeepholeOpt::correctMisalignedMemOp(Instruction *inst) {
894
LoadInst *linst = dyn_cast<LoadInst>(inst);
895
StoreInst *sinst = dyn_cast<StoreInst>(inst);
897
Type* Ty = inst->getType();
899
alignment = linst->getAlignment();
900
Ty = inst->getType();
902
alignment = sinst->getAlignment();
903
Ty = sinst->getValueOperand()->getType();
907
unsigned size = getTypeSize(Ty);
908
if (size == alignment || size < alignment) {
911
if (!Ty->isStructTy()) {
916
linst->setAlignment(0);
919
sinst->setAlignment(0);
926
AMDGPUPeepholeOpt::isSigned24BitOps(CallInst *CI) {
930
Value *LHS = CI->getOperand(CI->getNumOperands() - 1);
931
std::string namePrefix = LHS->getName().substr(0, 14);
932
if (namePrefix != "__amdil_imad24" && namePrefix != "__amdil_imul24"
933
&& namePrefix != "__amdil__imul24_high") {
936
if (mSTM->device()->usesHardware(AMDGPUDeviceInfo::Signed24BitOps)) {
943
AMDGPUPeepholeOpt::expandSigned24BitOps(CallInst *CI) {
944
assert(isSigned24BitOps(CI) && "Must be a "
945
"signed 24 bit operation to call this function!");
946
Value *LHS = CI->getOperand(CI->getNumOperands()-1);
947
// On 7XX and 8XX we do not have signed 24bit, so we need to
948
// expand it to the following:
949
// imul24 turns into 32bit imul
950
// imad24 turns into 32bit imad
951
// imul24_high turns into 32bit imulhigh
952
if (LHS->getName().substr(0, 14) == "__amdil_imad24") {
953
Type *aType = CI->getOperand(0)->getType();
954
bool isVector = aType->isVectorTy();
955
int numEle = isVector ? dyn_cast<VectorType>(aType)->getNumElements() : 1;
956
std::vector<Type*> callTypes;
957
callTypes.push_back(CI->getOperand(0)->getType());
958
callTypes.push_back(CI->getOperand(1)->getType());
959
callTypes.push_back(CI->getOperand(2)->getType());
960
FunctionType *funcType =
961
FunctionType::get(CI->getOperand(0)->getType(), callTypes, false);
962
std::string name = "__amdil_imad";
964
name += "_v" + itostr(numEle) + "i32";
968
Function *Func = dyn_cast<Function>(
969
CI->getParent()->getParent()->getParent()->
970
getOrInsertFunction(StringRef(name), funcType));
971
Value *Operands[3] = {
976
CallInst *nCI = CallInst::Create(Func, Operands, "imad24");
977
nCI->insertBefore(CI);
978
CI->replaceAllUsesWith(nCI);
979
} else if (LHS->getName().substr(0, 14) == "__amdil_imul24") {
980
BinaryOperator *mulOp =
981
BinaryOperator::Create(Instruction::Mul, CI->getOperand(0),
982
CI->getOperand(1), "imul24", CI);
983
CI->replaceAllUsesWith(mulOp);
984
} else if (LHS->getName().substr(0, 19) == "__amdil_imul24_high") {
985
Type *aType = CI->getOperand(0)->getType();
987
bool isVector = aType->isVectorTy();
988
int numEle = isVector ? dyn_cast<VectorType>(aType)->getNumElements() : 1;
989
std::vector<Type*> callTypes;
990
callTypes.push_back(CI->getOperand(0)->getType());
991
callTypes.push_back(CI->getOperand(1)->getType());
992
FunctionType *funcType =
993
FunctionType::get(CI->getOperand(0)->getType(), callTypes, false);
994
std::string name = "__amdil_imul_high";
996
name += "_v" + itostr(numEle) + "i32";
1000
Function *Func = dyn_cast<Function>(
1001
CI->getParent()->getParent()->getParent()->
1002
getOrInsertFunction(StringRef(name), funcType));
1003
Value *Operands[2] = {
1007
CallInst *nCI = CallInst::Create(Func, Operands, "imul24_high");
1008
nCI->insertBefore(CI);
1009
CI->replaceAllUsesWith(nCI);
1014
AMDGPUPeepholeOpt::isRWGLocalOpt(CallInst *CI) {
1016
&& CI->getOperand(CI->getNumOperands() - 1)->getName()
1017
== "__amdil_get_local_size_int");
1021
AMDGPUPeepholeOpt::convertAccurateDivide(CallInst *CI) {
1025
if (mSTM->device()->getGeneration() == AMDGPUDeviceInfo::HD6XXX
1026
&& (mSTM->getDeviceName() == "cayman")) {
1029
return CI->getOperand(CI->getNumOperands() - 1)->getName().substr(0, 20)
1030
== "__amdil_improved_div";
1034
AMDGPUPeepholeOpt::expandAccurateDivide(CallInst *CI) {
1035
assert(convertAccurateDivide(CI)
1036
&& "expanding accurate divide can only happen if it is expandable!");
1037
BinaryOperator *divOp =
1038
BinaryOperator::Create(Instruction::FDiv, CI->getOperand(0),
1039
CI->getOperand(1), "fdiv32", CI);
1040
CI->replaceAllUsesWith(divOp);
1044
AMDGPUPeepholeOpt::propagateSamplerInst(CallInst *CI) {
1045
if (optLevel != CodeGenOpt::None) {
1053
unsigned funcNameIdx = 0;
1054
funcNameIdx = CI->getNumOperands() - 1;
1055
StringRef calleeName = CI->getOperand(funcNameIdx)->getName();
1056
if (calleeName != "__amdil_image2d_read_norm"
1057
&& calleeName != "__amdil_image2d_read_unnorm"
1058
&& calleeName != "__amdil_image3d_read_norm"
1059
&& calleeName != "__amdil_image3d_read_unnorm") {
1063
unsigned samplerIdx = 2;
1065
Value *sampler = CI->getOperand(samplerIdx);
1066
LoadInst *lInst = dyn_cast<LoadInst>(sampler);
1071
if (lInst->getPointerAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) {
1075
GlobalVariable *gv = dyn_cast<GlobalVariable>(lInst->getPointerOperand());
1076
// If we are loading from what is not a global value, then we
1082
// If we don't have an initializer or we have an initializer and
1083
// the initializer is not a 32bit integer, we fail.
1084
if (!gv->hasInitializer()
1085
|| !gv->getInitializer()->getType()->isIntegerTy(32)) {
1089
// Now that we have the global variable initializer, lets replace
1090
// all uses of the load instruction with the samplerVal and
1091
// reparse the __amdil_is_constant() function.
1092
Constant *samplerVal = gv->getInitializer();
1093
lInst->replaceAllUsesWith(samplerVal);
1098
AMDGPUPeepholeOpt::doInitialization(Module &M) {
1103
AMDGPUPeepholeOpt::doFinalization(Module &M) {
1108
AMDGPUPeepholeOpt::getAnalysisUsage(AnalysisUsage &AU) const {
1109
AU.addRequired<MachineFunctionAnalysis>();
1110
FunctionPass::getAnalysisUsage(AU);
1111
AU.setPreservesAll();
1114
size_t AMDGPUPeepholeOpt::getTypeSize(Type * const T, bool dereferencePtr) {
1119
switch (T->getTypeID()) {
1120
case Type::X86_FP80TyID:
1121
case Type::FP128TyID:
1122
case Type::PPC_FP128TyID:
1123
case Type::LabelTyID:
1124
assert(0 && "These types are not supported by this backend");
1126
case Type::FloatTyID:
1127
case Type::DoubleTyID:
1128
size = T->getPrimitiveSizeInBits() >> 3;
1130
case Type::PointerTyID:
1131
size = getTypeSize(dyn_cast<PointerType>(T), dereferencePtr);
1133
case Type::IntegerTyID:
1134
size = getTypeSize(dyn_cast<IntegerType>(T), dereferencePtr);
1136
case Type::StructTyID:
1137
size = getTypeSize(dyn_cast<StructType>(T), dereferencePtr);
1139
case Type::ArrayTyID:
1140
size = getTypeSize(dyn_cast<ArrayType>(T), dereferencePtr);
1142
case Type::FunctionTyID:
1143
size = getTypeSize(dyn_cast<FunctionType>(T), dereferencePtr);
1145
case Type::VectorTyID:
1146
size = getTypeSize(dyn_cast<VectorType>(T), dereferencePtr);
1152
size_t AMDGPUPeepholeOpt::getTypeSize(StructType * const ST,
1153
bool dereferencePtr) {
1159
StructType::element_iterator eib;
1160
StructType::element_iterator eie;
1161
for (eib = ST->element_begin(), eie = ST->element_end(); eib != eie; ++eib) {
1163
size += getTypeSize(curType, dereferencePtr);
1168
size_t AMDGPUPeepholeOpt::getTypeSize(IntegerType * const IT,
1169
bool dereferencePtr) {
1170
return IT ? (IT->getBitWidth() >> 3) : 0;
1173
size_t AMDGPUPeepholeOpt::getTypeSize(FunctionType * const FT,
1174
bool dereferencePtr) {
1175
assert(0 && "Should not be able to calculate the size of an function type");
1179
size_t AMDGPUPeepholeOpt::getTypeSize(ArrayType * const AT,
1180
bool dereferencePtr) {
1181
return (size_t)(AT ? (getTypeSize(AT->getElementType(),
1182
dereferencePtr) * AT->getNumElements())
1186
size_t AMDGPUPeepholeOpt::getTypeSize(VectorType * const VT,
1187
bool dereferencePtr) {
1188
return VT ? (VT->getBitWidth() >> 3) : 0;
1191
size_t AMDGPUPeepholeOpt::getTypeSize(PointerType * const PT,
1192
bool dereferencePtr) {
1196
Type *CT = PT->getElementType();
1197
if (CT->getTypeID() == Type::StructTyID &&
1198
PT->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) {
1199
return getTypeSize(dyn_cast<StructType>(CT));
1200
} else if (dereferencePtr) {
1202
for (size_t x = 0, y = PT->getNumContainedTypes(); x < y; ++x) {
1203
size += getTypeSize(PT->getContainedType(x), dereferencePtr);
1211
size_t AMDGPUPeepholeOpt::getTypeSize(OpaqueType * const OT,
1212
bool dereferencePtr) {
1213
//assert(0 && "Should not be able to calculate the size of an opaque type");
added
removed
lib/Target/R600/AMDILPeepholeOptimizer.cpp
