1
//===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===//
3
// The LLVM Compiler Infrastructure
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
8
//===----------------------------------------------------------------------===//
10
// This file implements the IntrinsicLowering class.
12
//===----------------------------------------------------------------------===//
14
#include "llvm/Constants.h"
15
#include "llvm/DerivedTypes.h"
16
#include "llvm/Module.h"
17
#include "llvm/Type.h"
18
#include "llvm/CodeGen/IntrinsicLowering.h"
19
#include "llvm/Support/CallSite.h"
20
#include "llvm/Support/ErrorHandling.h"
21
#include "llvm/Support/IRBuilder.h"
22
#include "llvm/Support/raw_ostream.h"
23
#include "llvm/Target/TargetData.h"
24
#include "llvm/ADT/SmallVector.h"
27
template <class ArgIt>
28
static void EnsureFunctionExists(Module &M, const char *Name,
29
ArgIt ArgBegin, ArgIt ArgEnd,
31
// Insert a correctly-typed definition now.
32
std::vector<const Type *> ParamTys;
33
for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
34
ParamTys.push_back(I->getType());
35
M.getOrInsertFunction(Name, FunctionType::get(RetTy, ParamTys, false));
38
static void EnsureFPIntrinsicsExist(Module &M, Function *Fn,
40
const char *DName, const char *LDName) {
41
// Insert definitions for all the floating point types.
42
switch((int)Fn->arg_begin()->getType()->getTypeID()) {
44
EnsureFunctionExists(M, FName, Fn->arg_begin(), Fn->arg_end(),
45
Type::getFloatTy(M.getContext()));
47
case Type::DoubleTyID:
48
EnsureFunctionExists(M, DName, Fn->arg_begin(), Fn->arg_end(),
49
Type::getDoubleTy(M.getContext()));
51
case Type::X86_FP80TyID:
53
case Type::PPC_FP128TyID:
54
EnsureFunctionExists(M, LDName, Fn->arg_begin(), Fn->arg_end(),
55
Fn->arg_begin()->getType());
60
/// ReplaceCallWith - This function is used when we want to lower an intrinsic
61
/// call to a call of an external function. This handles hard cases such as
62
/// when there was already a prototype for the external function, and if that
63
/// prototype doesn't match the arguments we expect to pass in.
64
template <class ArgIt>
65
static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI,
66
ArgIt ArgBegin, ArgIt ArgEnd,
68
// If we haven't already looked up this function, check to see if the
69
// program already contains a function with this name.
70
Module *M = CI->getParent()->getParent()->getParent();
71
// Get or insert the definition now.
72
std::vector<const Type *> ParamTys;
73
for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
74
ParamTys.push_back((*I)->getType());
75
Constant* FCache = M->getOrInsertFunction(NewFn,
76
FunctionType::get(RetTy, ParamTys, false));
78
IRBuilder<> Builder(CI->getParent(), CI);
79
SmallVector<Value *, 8> Args(ArgBegin, ArgEnd);
80
CallInst *NewCI = Builder.CreateCall(FCache, Args.begin(), Args.end());
81
NewCI->setName(CI->getName());
83
CI->replaceAllUsesWith(NewCI);
87
// VisualStudio defines setjmp as _setjmp
88
#if defined(_MSC_VER) && defined(setjmp)
89
#define setjmp_undefined_for_visual_studio
93
void IntrinsicLowering::AddPrototypes(Module &M) {
94
LLVMContext &Context = M.getContext();
95
for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I)
96
if (I->isDeclaration() && !I->use_empty())
97
switch (I->getIntrinsicID()) {
99
case Intrinsic::setjmp:
100
EnsureFunctionExists(M, "setjmp", I->arg_begin(), I->arg_end(),
101
Type::getInt32Ty(M.getContext()));
103
case Intrinsic::longjmp:
104
EnsureFunctionExists(M, "longjmp", I->arg_begin(), I->arg_end(),
105
Type::getVoidTy(M.getContext()));
107
case Intrinsic::siglongjmp:
108
EnsureFunctionExists(M, "abort", I->arg_end(), I->arg_end(),
109
Type::getVoidTy(M.getContext()));
111
case Intrinsic::memcpy:
112
M.getOrInsertFunction("memcpy",
113
Type::getInt8PtrTy(Context),
114
Type::getInt8PtrTy(Context),
115
Type::getInt8PtrTy(Context),
116
TD.getIntPtrType(Context), (Type *)0);
118
case Intrinsic::memmove:
119
M.getOrInsertFunction("memmove",
120
Type::getInt8PtrTy(Context),
121
Type::getInt8PtrTy(Context),
122
Type::getInt8PtrTy(Context),
123
TD.getIntPtrType(Context), (Type *)0);
125
case Intrinsic::memset:
126
M.getOrInsertFunction("memset",
127
Type::getInt8PtrTy(Context),
128
Type::getInt8PtrTy(Context),
129
Type::getInt32Ty(M.getContext()),
130
TD.getIntPtrType(Context), (Type *)0);
132
case Intrinsic::sqrt:
133
EnsureFPIntrinsicsExist(M, I, "sqrtf", "sqrt", "sqrtl");
136
EnsureFPIntrinsicsExist(M, I, "sinf", "sin", "sinl");
139
EnsureFPIntrinsicsExist(M, I, "cosf", "cos", "cosl");
142
EnsureFPIntrinsicsExist(M, I, "powf", "pow", "powl");
145
EnsureFPIntrinsicsExist(M, I, "logf", "log", "logl");
147
case Intrinsic::log2:
148
EnsureFPIntrinsicsExist(M, I, "log2f", "log2", "log2l");
150
case Intrinsic::log10:
151
EnsureFPIntrinsicsExist(M, I, "log10f", "log10", "log10l");
154
EnsureFPIntrinsicsExist(M, I, "expf", "exp", "expl");
156
case Intrinsic::exp2:
157
EnsureFPIntrinsicsExist(M, I, "exp2f", "exp2", "exp2l");
162
/// LowerBSWAP - Emit the code to lower bswap of V before the specified
164
static Value *LowerBSWAP(LLVMContext &Context, Value *V, Instruction *IP) {
165
assert(V->getType()->isIntegerTy() && "Can't bswap a non-integer type!");
167
unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
169
IRBuilder<> Builder(IP->getParent(), IP);
172
default: llvm_unreachable("Unhandled type size of value to byteswap!");
174
Value *Tmp1 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8),
176
Value *Tmp2 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8),
178
V = Builder.CreateOr(Tmp1, Tmp2, "bswap.i16");
182
Value *Tmp4 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 24),
184
Value *Tmp3 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8),
186
Value *Tmp2 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8),
188
Value *Tmp1 = Builder.CreateLShr(V,ConstantInt::get(V->getType(), 24),
190
Tmp3 = Builder.CreateAnd(Tmp3,
191
ConstantInt::get(Type::getInt32Ty(Context), 0xFF0000),
193
Tmp2 = Builder.CreateAnd(Tmp2,
194
ConstantInt::get(Type::getInt32Ty(Context), 0xFF00),
196
Tmp4 = Builder.CreateOr(Tmp4, Tmp3, "bswap.or1");
197
Tmp2 = Builder.CreateOr(Tmp2, Tmp1, "bswap.or2");
198
V = Builder.CreateOr(Tmp4, Tmp2, "bswap.i32");
202
Value *Tmp8 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 56),
204
Value *Tmp7 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 40),
206
Value *Tmp6 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 24),
208
Value *Tmp5 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8),
210
Value* Tmp4 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8),
212
Value* Tmp3 = Builder.CreateLShr(V,
213
ConstantInt::get(V->getType(), 24),
215
Value* Tmp2 = Builder.CreateLShr(V,
216
ConstantInt::get(V->getType(), 40),
218
Value* Tmp1 = Builder.CreateLShr(V,
219
ConstantInt::get(V->getType(), 56),
221
Tmp7 = Builder.CreateAnd(Tmp7,
222
ConstantInt::get(Type::getInt64Ty(Context),
223
0xFF000000000000ULL),
225
Tmp6 = Builder.CreateAnd(Tmp6,
226
ConstantInt::get(Type::getInt64Ty(Context),
229
Tmp5 = Builder.CreateAnd(Tmp5,
230
ConstantInt::get(Type::getInt64Ty(Context),
233
Tmp4 = Builder.CreateAnd(Tmp4,
234
ConstantInt::get(Type::getInt64Ty(Context),
237
Tmp3 = Builder.CreateAnd(Tmp3,
238
ConstantInt::get(Type::getInt64Ty(Context),
241
Tmp2 = Builder.CreateAnd(Tmp2,
242
ConstantInt::get(Type::getInt64Ty(Context),
245
Tmp8 = Builder.CreateOr(Tmp8, Tmp7, "bswap.or1");
246
Tmp6 = Builder.CreateOr(Tmp6, Tmp5, "bswap.or2");
247
Tmp4 = Builder.CreateOr(Tmp4, Tmp3, "bswap.or3");
248
Tmp2 = Builder.CreateOr(Tmp2, Tmp1, "bswap.or4");
249
Tmp8 = Builder.CreateOr(Tmp8, Tmp6, "bswap.or5");
250
Tmp4 = Builder.CreateOr(Tmp4, Tmp2, "bswap.or6");
251
V = Builder.CreateOr(Tmp8, Tmp4, "bswap.i64");
258
/// LowerCTPOP - Emit the code to lower ctpop of V before the specified
260
static Value *LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP) {
261
assert(V->getType()->isIntegerTy() && "Can't ctpop a non-integer type!");
263
static const uint64_t MaskValues[6] = {
264
0x5555555555555555ULL, 0x3333333333333333ULL,
265
0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
266
0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL
269
IRBuilder<> Builder(IP->getParent(), IP);
271
unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
272
unsigned WordSize = (BitSize + 63) / 64;
273
Value *Count = ConstantInt::get(V->getType(), 0);
275
for (unsigned n = 0; n < WordSize; ++n) {
276
Value *PartValue = V;
277
for (unsigned i = 1, ct = 0; i < (BitSize>64 ? 64 : BitSize);
279
Value *MaskCst = ConstantInt::get(V->getType(), MaskValues[ct]);
280
Value *LHS = Builder.CreateAnd(PartValue, MaskCst, "cppop.and1");
281
Value *VShift = Builder.CreateLShr(PartValue,
282
ConstantInt::get(V->getType(), i),
284
Value *RHS = Builder.CreateAnd(VShift, MaskCst, "cppop.and2");
285
PartValue = Builder.CreateAdd(LHS, RHS, "ctpop.step");
287
Count = Builder.CreateAdd(PartValue, Count, "ctpop.part");
289
V = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 64),
298
/// LowerCTLZ - Emit the code to lower ctlz of V before the specified
300
static Value *LowerCTLZ(LLVMContext &Context, Value *V, Instruction *IP) {
302
IRBuilder<> Builder(IP->getParent(), IP);
304
unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
305
for (unsigned i = 1; i < BitSize; i <<= 1) {
306
Value *ShVal = ConstantInt::get(V->getType(), i);
307
ShVal = Builder.CreateLShr(V, ShVal, "ctlz.sh");
308
V = Builder.CreateOr(V, ShVal, "ctlz.step");
311
V = Builder.CreateNot(V);
312
return LowerCTPOP(Context, V, IP);
315
static void ReplaceFPIntrinsicWithCall(CallInst *CI, const char *Fname,
317
const char *LDname) {
319
switch (CI->getArgOperand(0)->getType()->getTypeID()) {
320
default: llvm_unreachable("Invalid type in intrinsic");
321
case Type::FloatTyID:
322
ReplaceCallWith(Fname, CI, CS.arg_begin(), CS.arg_end(),
323
Type::getFloatTy(CI->getContext()));
325
case Type::DoubleTyID:
326
ReplaceCallWith(Dname, CI, CS.arg_begin(), CS.arg_end(),
327
Type::getDoubleTy(CI->getContext()));
329
case Type::X86_FP80TyID:
330
case Type::FP128TyID:
331
case Type::PPC_FP128TyID:
332
ReplaceCallWith(LDname, CI, CS.arg_begin(), CS.arg_end(),
333
CI->getArgOperand(0)->getType());
338
void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
339
IRBuilder<> Builder(CI->getParent(), CI);
340
LLVMContext &Context = CI->getContext();
342
const Function *Callee = CI->getCalledFunction();
343
assert(Callee && "Cannot lower an indirect call!");
346
switch (Callee->getIntrinsicID()) {
347
case Intrinsic::not_intrinsic:
348
report_fatal_error("Cannot lower a call to a non-intrinsic function '"+
349
Callee->getName() + "'!");
351
report_fatal_error("Code generator does not support intrinsic function '"+
352
Callee->getName()+"'!");
354
// The setjmp/longjmp intrinsics should only exist in the code if it was
355
// never optimized (ie, right out of the CFE), or if it has been hacked on
356
// by the lowerinvoke pass. In both cases, the right thing to do is to
357
// convert the call to an explicit setjmp or longjmp call.
358
case Intrinsic::setjmp: {
359
Value *V = ReplaceCallWith("setjmp", CI, CS.arg_begin(), CS.arg_end(),
360
Type::getInt32Ty(Context));
361
if (!CI->getType()->isVoidTy())
362
CI->replaceAllUsesWith(V);
365
case Intrinsic::sigsetjmp:
366
if (!CI->getType()->isVoidTy())
367
CI->replaceAllUsesWith(Constant::getNullValue(CI->getType()));
370
case Intrinsic::longjmp: {
371
ReplaceCallWith("longjmp", CI, CS.arg_begin(), CS.arg_end(),
372
Type::getVoidTy(Context));
376
case Intrinsic::siglongjmp: {
377
// Insert the call to abort
378
ReplaceCallWith("abort", CI, CS.arg_end(), CS.arg_end(),
379
Type::getVoidTy(Context));
382
case Intrinsic::ctpop:
383
CI->replaceAllUsesWith(LowerCTPOP(Context, CI->getArgOperand(0), CI));
386
case Intrinsic::bswap:
387
CI->replaceAllUsesWith(LowerBSWAP(Context, CI->getArgOperand(0), CI));
390
case Intrinsic::ctlz:
391
CI->replaceAllUsesWith(LowerCTLZ(Context, CI->getArgOperand(0), CI));
394
case Intrinsic::cttz: {
395
// cttz(x) -> ctpop(~X & (X-1))
396
Value *Src = CI->getArgOperand(0);
397
Value *NotSrc = Builder.CreateNot(Src);
398
NotSrc->setName(Src->getName() + ".not");
399
Value *SrcM1 = ConstantInt::get(Src->getType(), 1);
400
SrcM1 = Builder.CreateSub(Src, SrcM1);
401
Src = LowerCTPOP(Context, Builder.CreateAnd(NotSrc, SrcM1), CI);
402
CI->replaceAllUsesWith(Src);
406
case Intrinsic::stacksave:
407
case Intrinsic::stackrestore: {
409
errs() << "WARNING: this target does not support the llvm.stack"
410
<< (Callee->getIntrinsicID() == Intrinsic::stacksave ?
411
"save" : "restore") << " intrinsic.\n";
413
if (Callee->getIntrinsicID() == Intrinsic::stacksave)
414
CI->replaceAllUsesWith(Constant::getNullValue(CI->getType()));
418
case Intrinsic::returnaddress:
419
case Intrinsic::frameaddress:
420
errs() << "WARNING: this target does not support the llvm."
421
<< (Callee->getIntrinsicID() == Intrinsic::returnaddress ?
422
"return" : "frame") << "address intrinsic.\n";
423
CI->replaceAllUsesWith(ConstantPointerNull::get(
424
cast<PointerType>(CI->getType())));
427
case Intrinsic::prefetch:
428
break; // Simply strip out prefetches on unsupported architectures
430
case Intrinsic::pcmarker:
431
break; // Simply strip out pcmarker on unsupported architectures
432
case Intrinsic::readcyclecounter: {
433
errs() << "WARNING: this target does not support the llvm.readcyclecoun"
434
<< "ter intrinsic. It is being lowered to a constant 0\n";
435
CI->replaceAllUsesWith(ConstantInt::get(Type::getInt64Ty(Context), 0));
439
case Intrinsic::dbg_declare:
440
break; // Simply strip out debugging intrinsics
442
case Intrinsic::eh_exception:
443
case Intrinsic::eh_selector:
444
CI->replaceAllUsesWith(Constant::getNullValue(CI->getType()));
447
case Intrinsic::eh_typeid_for:
448
// Return something different to eh_selector.
449
CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 1));
452
case Intrinsic::var_annotation:
453
break; // Strip out annotate intrinsic
455
case Intrinsic::memcpy: {
456
const IntegerType *IntPtr = TD.getIntPtrType(Context);
457
Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
458
/* isSigned */ false);
460
Ops[0] = CI->getArgOperand(0);
461
Ops[1] = CI->getArgOperand(1);
463
ReplaceCallWith("memcpy", CI, Ops, Ops+3, CI->getArgOperand(0)->getType());
466
case Intrinsic::memmove: {
467
const IntegerType *IntPtr = TD.getIntPtrType(Context);
468
Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
469
/* isSigned */ false);
471
Ops[0] = CI->getArgOperand(0);
472
Ops[1] = CI->getArgOperand(1);
474
ReplaceCallWith("memmove", CI, Ops, Ops+3, CI->getArgOperand(0)->getType());
477
case Intrinsic::memset: {
478
const IntegerType *IntPtr = TD.getIntPtrType(Context);
479
Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
480
/* isSigned */ false);
482
Ops[0] = CI->getArgOperand(0);
483
// Extend the amount to i32.
484
Ops[1] = Builder.CreateIntCast(CI->getArgOperand(1),
485
Type::getInt32Ty(Context),
486
/* isSigned */ false);
488
ReplaceCallWith("memset", CI, Ops, Ops+3, CI->getArgOperand(0)->getType());
491
case Intrinsic::sqrt: {
492
ReplaceFPIntrinsicWithCall(CI, "sqrtf", "sqrt", "sqrtl");
495
case Intrinsic::log: {
496
ReplaceFPIntrinsicWithCall(CI, "logf", "log", "logl");
499
case Intrinsic::log2: {
500
ReplaceFPIntrinsicWithCall(CI, "log2f", "log2", "log2l");
503
case Intrinsic::log10: {
504
ReplaceFPIntrinsicWithCall(CI, "log10f", "log10", "log10l");
507
case Intrinsic::exp: {
508
ReplaceFPIntrinsicWithCall(CI, "expf", "exp", "expl");
511
case Intrinsic::exp2: {
512
ReplaceFPIntrinsicWithCall(CI, "exp2f", "exp2", "exp2l");
515
case Intrinsic::pow: {
516
ReplaceFPIntrinsicWithCall(CI, "powf", "pow", "powl");
519
case Intrinsic::flt_rounds:
520
// Lower to "round to the nearest"
521
if (!CI->getType()->isVoidTy())
522
CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 1));
524
case Intrinsic::invariant_start:
525
case Intrinsic::lifetime_start:
526
// Discard region information.
527
CI->replaceAllUsesWith(UndefValue::get(CI->getType()));
529
case Intrinsic::invariant_end:
530
case Intrinsic::lifetime_end:
531
// Discard region information.
535
assert(CI->use_empty() &&
536
"Lowering should have eliminated any uses of the intrinsic call!");
537
CI->eraseFromParent();