summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/CodeGen/CodeGenPrepare.cpp29
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp187
-rw-r--r--llvm/lib/Transforms/Utils/Local.cpp202
3 files changed, 239 insertions, 179 deletions
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 03e57787307..4326ba99538 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -5211,6 +5211,24 @@ bool CodeGenPrepare::optimizeInst(Instruction *I, bool& ModifiedDT) {
return false;
}
+/// Given an OR instruction, check to see if this is a bitreverse
+/// idiom. If so, insert the new intrinsic and return true.
+static bool makeBitReverse(Instruction &I, const DataLayout &DL,
+ const TargetLowering &TLI) {
+ if (!I.getType()->isIntegerTy() ||
+ !TLI.isOperationLegalOrCustom(ISD::BITREVERSE,
+ TLI.getValueType(DL, I.getType(), true)))
+ return false;
+
+ SmallVector<Instruction*, 4> Insts;
+ if (!recognizeBitReverseOrBSwapIdiom(&I, false, true, Insts))
+ return false;
+ Instruction *LastInst = Insts.back();
+ I.replaceAllUsesWith(LastInst);
+ RecursivelyDeleteTriviallyDeadInstructions(&I);
+ return true;
+}
+
// In this pass we look for GEP and cast instructions that are used
// across basic blocks and rewrite them to improve basic-block-at-a-time
// selection.
@@ -5226,6 +5244,17 @@ bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, bool& ModifiedDT) {
}
MadeChange |= dupRetToEnableTailCallOpts(&BB);
+ bool MadeBitReverse = true;
+ while (TLI && MadeBitReverse) {
+ MadeBitReverse = false;
+ for (auto &I : reverse(BB)) {
+ if (makeBitReverse(I, *DL, *TLI)) {
+ MadeBitReverse = MadeChange = true;
+ break;
+ }
+ }
+ }
+
return MadeChange;
}
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 95c50d32c82..76cefd97cd8 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -17,6 +17,7 @@
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Transforms/Utils/CmpInstAnalysis.h"
+#include "llvm/Transforms/Utils/Local.h"
using namespace llvm;
using namespace PatternMatch;
@@ -1565,190 +1566,18 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) {
return Changed ? &I : nullptr;
}
-
-/// Analyze the specified subexpression and see if it is capable of providing
-/// pieces of a bswap or bitreverse. The subexpression provides a potential
-/// piece of a bswap or bitreverse if it can be proven that each non-zero bit in
-/// the output of the expression came from a corresponding bit in some other
-/// value. This function is recursive, and the end result is a mapping of
-/// (value, bitnumber) to bitnumber. It is the caller's responsibility to
-/// validate that all `value`s are identical and that the bitnumber to bitnumber
-/// mapping is correct for a bswap or bitreverse.
-///
-/// For example, if the current subexpression if "(shl i32 %X, 24)" then we know
-/// that the expression deposits the low byte of %X into the high byte of the
-/// result and that all other bits are zero. This expression is accepted,
-/// BitValues[24-31] are set to %X and BitProvenance[24-31] are set to [0-7].
-///
-/// This function returns true if the match was unsuccessful and false if so.
-/// On entry to the function the "OverallLeftShift" is a signed integer value
-/// indicating the number of bits that the subexpression is later shifted. For
-/// example, if the expression is later right shifted by 16 bits, the
-/// OverallLeftShift value would be -16 on entry. This is used to specify which
-/// bits of BitValues are actually being set.
-///
-/// Similarly, BitMask is a bitmask where a bit is clear if its corresponding
-/// bit is masked to zero by a user. For example, in (X & 255), X will be
-/// processed with a bytemask of 255. BitMask is always in the local
-/// (OverallLeftShift) coordinate space.
-///
-static bool CollectBitParts(Value *V, int OverallLeftShift, APInt BitMask,
- SmallVectorImpl<Value *> &BitValues,
- SmallVectorImpl<int> &BitProvenance) {
- if (Instruction *I = dyn_cast<Instruction>(V)) {
- // If this is an or instruction, it may be an inner node of the bswap.
- if (I->getOpcode() == Instruction::Or)
- return CollectBitParts(I->getOperand(0), OverallLeftShift, BitMask,
- BitValues, BitProvenance) ||
- CollectBitParts(I->getOperand(1), OverallLeftShift, BitMask,
- BitValues, BitProvenance);
-
- // If this is a logical shift by a constant, recurse with OverallLeftShift
- // and BitMask adjusted.
- if (I->isLogicalShift() && isa<ConstantInt>(I->getOperand(1))) {
- unsigned ShAmt =
- cast<ConstantInt>(I->getOperand(1))->getLimitedValue(~0U);
- // Ensure the shift amount is defined.
- if (ShAmt > BitValues.size())
- return true;
-
- unsigned BitShift = ShAmt;
- if (I->getOpcode() == Instruction::Shl) {
- // X << C -> collect(X, +C)
- OverallLeftShift += BitShift;
- BitMask = BitMask.lshr(BitShift);
- } else {
- // X >>u C -> collect(X, -C)
- OverallLeftShift -= BitShift;
- BitMask = BitMask.shl(BitShift);
- }
-
- if (OverallLeftShift >= (int)BitValues.size())
- return true;
- if (OverallLeftShift <= -(int)BitValues.size())
- return true;
-
- return CollectBitParts(I->getOperand(0), OverallLeftShift, BitMask,
- BitValues, BitProvenance);
- }
-
- // If this is a logical 'and' with a mask that clears bits, clear the
- // corresponding bits in BitMask.
- if (I->getOpcode() == Instruction::And &&
- isa<ConstantInt>(I->getOperand(1))) {
- unsigned NumBits = BitValues.size();
- APInt Bit(I->getType()->getPrimitiveSizeInBits(), 1);
- const APInt &AndMask = cast<ConstantInt>(I->getOperand(1))->getValue();
-
- for (unsigned i = 0; i != NumBits; ++i, Bit <<= 1) {
- // If this bit is masked out by a later operation, we don't care what
- // the and mask is.
- if (BitMask[i] == 0)
- continue;
-
- // If the AndMask is zero for this bit, clear the bit.
- APInt MaskB = AndMask & Bit;
- if (MaskB == 0) {
- BitMask.clearBit(i);
- continue;
- }
-
- // Otherwise, this bit is kept.
- }
-
- return CollectBitParts(I->getOperand(0), OverallLeftShift, BitMask,
- BitValues, BitProvenance);
- }
- }
-
- // Okay, we got to something that isn't a shift, 'or' or 'and'. This must be
- // the input value to the bswap/bitreverse. To be part of a bswap or
- // bitreverse we must be demanding a contiguous range of bits from it.
- unsigned InputBitLen = BitMask.countPopulation();
- unsigned InputBitNo = BitMask.countTrailingZeros();
- if (BitMask.getBitWidth() - BitMask.countLeadingZeros() - InputBitNo !=
- InputBitLen)
- // Not a contiguous set range of bits!
- return true;
-
- // We know we're moving a contiguous range of bits from the input to the
- // output. Record which bits in the output came from which bits in the input.
- unsigned DestBitNo = InputBitNo + OverallLeftShift;
- for (unsigned I = 0; I < InputBitLen; ++I)
- BitProvenance[DestBitNo + I] = InputBitNo + I;
-
- // If the destination bit value is already defined, the values are or'd
- // together, which isn't a bswap/bitreverse (unless it's an or of the same
- // bits).
- if (BitValues[DestBitNo] && BitValues[DestBitNo] != V)
- return true;
- for (unsigned I = 0; I < InputBitLen; ++I)
- BitValues[DestBitNo + I] = V;
-
- return false;
-}
-
-static bool bitTransformIsCorrectForBSwap(unsigned From, unsigned To,
- unsigned BitWidth) {
- if (From % 8 != To % 8)
- return false;
- // Convert from bit indices to byte indices and check for a byte reversal.
- From >>= 3;
- To >>= 3;
- BitWidth >>= 3;
- return From == BitWidth - To - 1;
-}
-
-static bool bitTransformIsCorrectForBitReverse(unsigned From, unsigned To,
- unsigned BitWidth) {
- return From == BitWidth - To - 1;
-}
-
/// Given an OR instruction, check to see if this is a bswap or bitreverse
/// idiom. If so, insert the new intrinsic and return it.
Instruction *InstCombiner::MatchBSwapOrBitReverse(BinaryOperator &I) {
- IntegerType *ITy = dyn_cast<IntegerType>(I.getType());
- if (!ITy)
- return nullptr; // Can't do vectors.
- unsigned BW = ITy->getBitWidth();
-
- /// We keep track of which bit (BitProvenance) inside which value (BitValues)
- /// defines each bit in the result.
- SmallVector<Value *, 8> BitValues(BW, nullptr);
- SmallVector<int, 8> BitProvenance(BW, -1);
-
- // Try to find all the pieces corresponding to the bswap.
- APInt BitMask = APInt::getAllOnesValue(BitValues.size());
- if (CollectBitParts(&I, 0, BitMask, BitValues, BitProvenance))
- return nullptr;
-
- // Check to see if all of the bits come from the same value.
- Value *V = BitValues[0];
- if (!V) return nullptr; // Didn't find a bit? Must be zero.
-
- if (!std::all_of(BitValues.begin(), BitValues.end(),
- [&](const Value *X) { return X == V; }))
- return nullptr;
-
- // Now, is the bit permutation correct for a bswap or a bitreverse? We can
- // only byteswap values with an even number of bytes.
- bool OKForBSwap = BW % 16 == 0, OKForBitReverse = true;;
- for (unsigned i = 0, e = BitValues.size(); i != e; ++i) {
- OKForBSwap &= bitTransformIsCorrectForBSwap(BitProvenance[i], i, BW);
- OKForBitReverse &=
- bitTransformIsCorrectForBitReverse(BitProvenance[i], i, BW);
- }
-
- Intrinsic::ID Intrin;
- if (OKForBSwap)
- Intrin = Intrinsic::bswap;
- else if (OKForBitReverse)
- Intrin = Intrinsic::bitreverse;
- else
+ SmallVector<Instruction*, 4> Insts;
+ if (!recognizeBitReverseOrBSwapIdiom(&I, true, false, Insts))
return nullptr;
+ Instruction *LastInst = Insts.pop_back_val();
+ LastInst->removeFromParent();
- Function *F = Intrinsic::getDeclaration(I.getModule(), Intrin, ITy);
- return CallInst::Create(F, V);
+ for (auto *Inst : Insts)
+ Worklist.Add(Inst);
+ return LastInst;
}
/// We have an expression of the form (A&C)|(B&D). Check if A is (cond?-1:0)
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index e28232c4201..92e3a6b118b 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -1588,3 +1588,205 @@ bool llvm::callsGCLeafFunction(ImmutableCallSite CS) {
return false;
}
+
+/// A potential constituent of a bitreverse or bswap expression. See
+/// collectBitParts for a fuller explanation.
+struct BitPart {
+ BitPart(Value *P, unsigned BW) : Provider(P) {
+ Provenance.resize(BW);
+ }
+
+ /// The Value that this is a bitreverse/bswap of.
+ Value *Provider;
+ /// The "provenance" of each bit. Provenance[A] = B means that bit A
+ /// in Provider becomes bit B in the result of this expression.
+ SmallVector<int8_t, 32> Provenance; // int8_t means max size is i128.
+
+ enum { Unset = -1 };
+};
+
+/// Analyze the specified subexpression and see if it is capable of providing
+/// pieces of a bswap or bitreverse. The subexpression provides a potential
+/// piece of a bswap or bitreverse if it can be proven that each non-zero bit in
+/// the output of the expression came from a corresponding bit in some other
+/// value. This function is recursive, and the end result is a mapping of
+/// bitnumber to bitnumber. It is the caller's responsibility to validate that
+/// the bitnumber to bitnumber mapping is correct for a bswap or bitreverse.
+///
+/// For example, if the current subexpression if "(shl i32 %X, 24)" then we know
+/// that the expression deposits the low byte of %X into the high byte of the
+/// result and that all other bits are zero. This expression is accepted and a
+/// BitPart is returned with Provider set to %X and Provenance[24-31] set to
+/// [0-7].
+///
+/// To avoid revisiting values, the BitPart results are memoized into the
+/// provided map. To avoid unnecessary copying of BitParts, BitParts are
+/// constructed in-place in the \c BPS map. Because of this \c BPS needs to
+/// store BitParts objects, not pointers. As we need the concept of a nullptr
+/// BitParts (Value has been analyzed and the analysis failed), we an Optional
+/// type instead to provide the same functionality.
+///
+/// Because we pass around references into \c BPS, we must use a container that
+/// does not invalidate internal references (std::map instead of DenseMap).
+///
+static const Optional<BitPart> &
+collectBitParts(Value *V, bool MatchBSwaps, bool MatchBitReversals,
+ std::map<Value *, Optional<BitPart>> &BPS) {
+ auto I = BPS.find(V);
+ if (I != BPS.end())
+ return I->second;
+
+ auto &Result = BPS[V] = None;
+ auto BitWidth = cast<IntegerType>(V->getType())->getBitWidth();
+
+ if (Instruction *I = dyn_cast<Instruction>(V)) {
+ // If this is an or instruction, it may be an inner node of the bswap.
+ if (I->getOpcode() == Instruction::Or) {
+ auto &A = collectBitParts(I->getOperand(0), MatchBSwaps,
+ MatchBitReversals, BPS);
+ auto &B = collectBitParts(I->getOperand(1), MatchBSwaps,
+ MatchBitReversals, BPS);
+ if (!A || !B)
+ return Result;
+
+ // Try and merge the two together.
+ if (!A->Provider || A->Provider != B->Provider)
+ return Result;
+
+ Result = BitPart(A->Provider, BitWidth);
+ for (unsigned i = 0; i < A->Provenance.size(); ++i) {
+ if (A->Provenance[i] != BitPart::Unset &&
+ B->Provenance[i] != BitPart::Unset &&
+ A->Provenance[i] != B->Provenance[i])
+ return Result = None;
+
+ if (A->Provenance[i] == BitPart::Unset)
+ Result->Provenance[i] = B->Provenance[i];
+ else
+ Result->Provenance[i] = A->Provenance[i];
+ }
+
+ return Result;
+ }
+
+ // If this is a logical shift by a constant, recurse then shift the result.
+ if (I->isLogicalShift() && isa<ConstantInt>(I->getOperand(1))) {
+ unsigned BitShift =
+ cast<ConstantInt>(I->getOperand(1))->getLimitedValue(~0U);
+ // Ensure the shift amount is defined.
+ if (BitShift > BitWidth)
+ return Result;
+
+ auto &Res = collectBitParts(I->getOperand(0), MatchBSwaps,
+ MatchBitReversals, BPS);
+ if (!Res)
+ return Result;
+ Result = Res;
+
+ // Perform the "shift" on BitProvenance.
+ auto &P = Result->Provenance;
+ if (I->getOpcode() == Instruction::Shl) {
+ P.erase(std::prev(P.end(), BitShift), P.end());
+ P.insert(P.begin(), BitShift, BitPart::Unset);
+ } else {
+ P.erase(P.begin(), std::next(P.begin(), BitShift));
+ P.insert(P.end(), BitShift, BitPart::Unset);
+ }
+
+ return Result;
+ }
+
+ // If this is a logical 'and' with a mask that clears bits, recurse then
+ // unset the appropriate bits.
+ if (I->getOpcode() == Instruction::And &&
+ isa<ConstantInt>(I->getOperand(1))) {
+ APInt Bit(I->getType()->getPrimitiveSizeInBits(), 1);
+ const APInt &AndMask = cast<ConstantInt>(I->getOperand(1))->getValue();
+
+ // Check that the mask allows a multiple of 8 bits for a bswap, for an
+ // early exit.
+ unsigned NumMaskedBits = AndMask.countPopulation();
+ if (!MatchBitReversals && NumMaskedBits % 8 != 0)
+ return Result;
+
+ auto &Res = collectBitParts(I->getOperand(0), MatchBSwaps,
+ MatchBitReversals, BPS);
+ if (!Res)
+ return Result;
+ Result = Res;
+
+ for (unsigned i = 0; i < BitWidth; ++i, Bit <<= 1)
+ // If the AndMask is zero for this bit, clear the bit.
+ if ((AndMask & Bit) == 0)
+ Result->Provenance[i] = BitPart::Unset;
+
+ return Result;
+ }
+ }
+
+ // Okay, we got to something that isn't a shift, 'or' or 'and'. This must be
+ // the input value to the bswap/bitreverse.
+ Result = BitPart(V, BitWidth);
+ for (unsigned i = 0; i < BitWidth; ++i)
+ Result->Provenance[i] = i;
+ return Result;
+}
+
+static bool bitTransformIsCorrectForBSwap(unsigned From, unsigned To,
+ unsigned BitWidth) {
+ if (From % 8 != To % 8)
+ return false;
+ // Convert from bit indices to byte indices and check for a byte reversal.
+ From >>= 3;
+ To >>= 3;
+ BitWidth >>= 3;
+ return From == BitWidth - To - 1;
+}
+
+static bool bitTransformIsCorrectForBitReverse(unsigned From, unsigned To,
+ unsigned BitWidth) {
+ return From == BitWidth - To - 1;
+}
+
+/// Given an OR instruction, check to see if this is a bitreverse
+/// idiom. If so, insert the new intrinsic and return true.
+bool llvm::recognizeBitReverseOrBSwapIdiom(
+ Instruction *I, bool MatchBSwaps, bool MatchBitReversals,
+ SmallVectorImpl<Instruction *> &InsertedInsts) {
+ if (Operator::getOpcode(I) != Instruction::Or)
+ return false;
+ if (!MatchBSwaps && !MatchBitReversals)
+ return false;
+ IntegerType *ITy = dyn_cast<IntegerType>(I->getType());
+ if (!ITy || ITy->getBitWidth() > 128)
+ return false; // Can't do vectors or integers > 128 bits.
+ unsigned BW = ITy->getBitWidth();
+
+ // Try to find all the pieces corresponding to the bswap.
+ std::map<Value *, Optional<BitPart>> BPS;
+ auto Res = collectBitParts(I, MatchBSwaps, MatchBitReversals, BPS);
+ if (!Res)
+ return false;
+ auto &BitProvenance = Res->Provenance;
+
+ // Now, is the bit permutation correct for a bswap or a bitreverse? We can
+ // only byteswap values with an even number of bytes.
+ bool OKForBSwap = BW % 16 == 0, OKForBitReverse = true;
+ for (unsigned i = 0; i < BW; ++i) {
+ OKForBSwap &= bitTransformIsCorrectForBSwap(BitProvenance[i], i, BW);
+ OKForBitReverse &=
+ bitTransformIsCorrectForBitReverse(BitProvenance[i], i, BW);
+ }
+
+ Intrinsic::ID Intrin;
+ if (OKForBSwap && MatchBSwaps)
+ Intrin = Intrinsic::bswap;
+ else if (OKForBitReverse && MatchBitReversals)
+ Intrin = Intrinsic::bitreverse;
+ else
+ return false;
+
+ Function *F = Intrinsic::getDeclaration(I->getModule(), Intrin, ITy);
+ InsertedInsts.push_back(CallInst::Create(F, Res->Provider, "rev", I));
+ return true;
+}
OpenPOWER on IntegriCloud