summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Analysis/CodeMetrics.cpp36
-rw-r--r--llvm/lib/Analysis/LazyValueInfo.cpp12
-rw-r--r--llvm/lib/Analysis/ScalarEvolution.cpp92
-rw-r--r--llvm/lib/Analysis/ValueTracking.cpp109
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp72
-rw-r--r--llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp9
6 files changed, 222 insertions, 108 deletions
diff --git a/llvm/lib/Analysis/CodeMetrics.cpp b/llvm/lib/Analysis/CodeMetrics.cpp
index bdffdd8eb27..859adf0eb94 100644
--- a/llvm/lib/Analysis/CodeMetrics.cpp
+++ b/llvm/lib/Analysis/CodeMetrics.cpp
@@ -76,20 +76,12 @@ void CodeMetrics::collectEphemeralValues(
SmallPtrSet<const Value *, 32> Visited;
SmallVector<const Value *, 16> Worklist;
- for (auto &AssumeVH : AC->assumptions()) {
- if (!AssumeVH)
- continue;
- Instruction *I = cast<Instruction>(AssumeVH);
-
- // Filter out call sites outside of the loop so we don't do a function's
- // worth of work for each of its loops (and, in the common case, ephemeral
- // values in the loop are likely due to @llvm.assume calls in the loop).
- if (!L->contains(I->getParent()))
- continue;
-
- if (EphValues.insert(I).second)
- appendSpeculatableOperands(I, Visited, Worklist);
- }
+ for (auto &B : L->blocks())
+ for (auto &I : *B)
+ if (auto *II = dyn_cast<IntrinsicInst>(&I))
+ if (II->getIntrinsicID() == Intrinsic::assume &&
+ EphValues.insert(II).second)
+ appendSpeculatableOperands(II, Visited, Worklist);
completeEphemeralValues(Visited, Worklist, EphValues);
}
@@ -100,16 +92,12 @@ void CodeMetrics::collectEphemeralValues(
SmallPtrSet<const Value *, 32> Visited;
SmallVector<const Value *, 16> Worklist;
- for (auto &AssumeVH : AC->assumptions()) {
- if (!AssumeVH)
- continue;
- Instruction *I = cast<Instruction>(AssumeVH);
- assert(I->getParent()->getParent() == F &&
- "Found assumption for the wrong function!");
-
- if (EphValues.insert(I).second)
- appendSpeculatableOperands(I, Visited, Worklist);
- }
+ for (auto &B : *F)
+ for (auto &I : B)
+ if (auto *II = dyn_cast<IntrinsicInst>(&I))
+ if (II->getIntrinsicID() == Intrinsic::assume &&
+ EphValues.insert(II).second)
+ appendSpeculatableOperands(II, Visited, Worklist);
completeEphemeralValues(Visited, Worklist, EphValues);
}
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index e51e8217360..f432ce9abc1 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -924,14 +924,16 @@ void LazyValueInfoImpl::intersectAssumeOrGuardBlockValueConstantRange(
if (!BBI)
return;
- for (auto &AssumeVH : AC->assumptions()) {
- if (!AssumeVH)
+ for (auto *U : Val->users()) {
+ auto *II = dyn_cast<IntrinsicInst>(U);
+ if (!II)
continue;
- auto *I = cast<CallInst>(AssumeVH);
- if (!isValidAssumeForContext(I, BBI, DT))
+ if (II->getIntrinsicID() != Intrinsic::assume)
+ continue;
+ if (!isValidAssumeForContext(II, BBI, DT))
continue;
- BBLV = intersect(BBLV, getValueFromCondition(Val, I->getArgOperand(0)));
+ BBLV = intersect(BBLV, getValueFromCondition(Val, II->getArgOperand(0)));
}
// If guards are not used in the module, don't spend time looking for them
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 7962222a210..29a950b67eb 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -1212,6 +1212,7 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op,
SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator),
Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
+ addAffectedFromOperands(S);
return S;
}
@@ -1598,7 +1599,7 @@ const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op,
// these to prove lack of overflow. Use this fact to avoid
// doing extra work that may not pay off.
if (!isa<SCEVCouldNotCompute>(MaxBECount) || HasGuards ||
- !AC.assumptions().empty()) {
+ !AffectedMap.empty()) {
// If the backedge is guarded by a comparison with the pre-inc
// value the addrec is safe. Also, if the entry is guarded by
// a comparison with the start value and the backedge is
@@ -1664,6 +1665,7 @@ const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op,
SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
+ addAffectedFromOperands(S);
return S;
}
@@ -1833,7 +1835,7 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op,
// doing extra work that may not pay off.
if (!isa<SCEVCouldNotCompute>(MaxBECount) || HasGuards ||
- !AC.assumptions().empty()) {
+ !AffectedMap.empty()) {
// If the backedge is guarded by a comparison with the pre-inc
// value the addrec is safe. Also, if the entry is guarded by
// a comparison with the start value and the backedge is
@@ -1891,6 +1893,7 @@ const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op,
SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
Op, Ty);
UniqueSCEVs.InsertNode(S, IP);
+ addAffectedFromOperands(S);
return S;
}
@@ -2444,6 +2447,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator),
O, Ops.size());
UniqueSCEVs.InsertNode(S, IP);
+ addAffectedFromOperands(S);
}
S->setNoWrapFlags(Flags);
return S;
@@ -2736,6 +2740,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
O, Ops.size());
UniqueSCEVs.InsertNode(S, IP);
+ addAffectedFromOperands(S);
}
S->setNoWrapFlags(Flags);
return S;
@@ -2856,6 +2861,7 @@ const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS,
SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator),
LHS, RHS);
UniqueSCEVs.InsertNode(S, IP);
+ addAffectedFromOperands(S);
return S;
}
@@ -3036,6 +3042,7 @@ ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands,
S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator),
O, Operands.size(), L);
UniqueSCEVs.InsertNode(S, IP);
+ addAffectedFromOperands(S);
}
S->setNoWrapFlags(Flags);
return S;
@@ -3191,6 +3198,7 @@ ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator),
O, Ops.size());
UniqueSCEVs.InsertNode(S, IP);
+ addAffectedFromOperands(S);
return S;
}
@@ -3292,6 +3300,7 @@ ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator),
O, Ops.size());
UniqueSCEVs.InsertNode(S, IP);
+ addAffectedFromOperands(S);
return S;
}
@@ -3492,9 +3501,38 @@ const SCEV *ScalarEvolution::getSCEV(Value *V) {
ExprValueMap[Stripped].insert({V, Offset});
}
}
+
+ // If this value is an instruction or an argument, and might be affected by
+ // an assumption, and its SCEV to the AffectedMap.
+ if (isa<Instruction>(V) || isa<Argument>(V)) {
+ for (auto *U : V->users()) {
+ auto *II = dyn_cast<IntrinsicInst>(U);
+ if (!II)
+ continue;
+ if (II->getIntrinsicID() != Intrinsic::assume)
+ continue;
+
+ AffectedMap[S].insert(II);
+ }
+ }
+
return S;
}
+// If one of this SCEV's operands is in the AffectedMap (meaning that it might
+// be affected by an assumption), then this SCEV might be affected by the same
+// assumption.
+void ScalarEvolution::addAffectedFromOperands(const SCEV *S) {
+ if (auto *NS = dyn_cast<SCEVNAryExpr>(S))
+ for (auto *Op : NS->operands()) {
+ auto AMI = AffectedMap.find(Op);
+ if (AMI == AffectedMap.end())
+ continue;
+
+ AffectedMap[S].insert(AMI->second.begin(), AMI->second.end());
+ }
+}
+
const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
@@ -7926,16 +7964,23 @@ ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
}
// Check conditions due to any @llvm.assume intrinsics.
- for (auto &AssumeVH : AC.assumptions()) {
- if (!AssumeVH)
- continue;
- auto *CI = cast<CallInst>(AssumeVH);
- if (!DT.dominates(CI, Latch->getTerminator()))
- continue;
+ auto CheckAssumptions = [&](const SCEV *S) {
+ auto AMI = AffectedMap.find(S);
+ if (AMI != AffectedMap.end())
+ for (auto *Assume : AMI->second) {
+ auto *CI = cast<CallInst>(Assume);
+ if (!DT.dominates(CI, Latch->getTerminator()))
+ continue;
- if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
- return true;
- }
+ if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
+ return true;
+ }
+
+ return false;
+ };
+
+ if (CheckAssumptions(LHS) || CheckAssumptions(RHS))
+ return true;
// If the loop is not reachable from the entry block, we risk running into an
// infinite loop as we walk up into the dom tree. These loops do not matter
@@ -8020,16 +8065,23 @@ ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L,
}
// Check conditions due to any @llvm.assume intrinsics.
- for (auto &AssumeVH : AC.assumptions()) {
- if (!AssumeVH)
- continue;
- auto *CI = cast<CallInst>(AssumeVH);
- if (!DT.dominates(CI, L->getHeader()))
- continue;
+ auto CheckAssumptions = [&](const SCEV *S) {
+ auto AMI = AffectedMap.find(S);
+ if (AMI != AffectedMap.end())
+ for (auto *Assume : AMI->second) {
+ auto *CI = cast<CallInst>(Assume);
+ if (!DT.dominates(CI, L->getHeader()))
+ continue;
- if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
- return true;
- }
+ if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
+ return true;
+ }
+
+ return false;
+ };
+
+ if (CheckAssumptions(LHS) || CheckAssumptions(RHS))
+ return true;
return false;
}
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 950a2fbcff9..aa745eb440d 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -526,31 +526,28 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
unsigned BitWidth = KnownZero.getBitWidth();
- for (auto &AssumeVH : Q.AC->assumptions()) {
- if (!AssumeVH)
+ for (auto *U : V->users()) {
+ auto *II = dyn_cast<IntrinsicInst>(U);
+ if (!II)
continue;
- CallInst *I = cast<CallInst>(AssumeVH);
- assert(I->getParent()->getParent() == Q.CxtI->getParent()->getParent() &&
- "Got assumption for the wrong function!");
- if (Q.isExcluded(I))
+ if (II->getIntrinsicID() != Intrinsic::assume)
+ continue;
+ if (Q.isExcluded(II))
continue;
- // Warning: This loop can end up being somewhat performance sensetive.
- // We're running this loop for once for each value queried resulting in a
- // runtime of ~O(#assumes * #values).
-
- assert(I->getCalledFunction()->getIntrinsicID() == Intrinsic::assume &&
- "must be an assume intrinsic");
-
- Value *Arg = I->getArgOperand(0);
+ Value *Arg = II->getArgOperand(0);
- if (Arg == V && isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ if (Arg == V && isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
assert(BitWidth == 1 && "assume operand is not i1?");
KnownZero.clearAllBits();
KnownOne.setAllBits();
return;
}
+ // Note that the patterns below need to be kept in sync with the code
+ // in InstCombiner::visitCallInst that adds relevant values to each
+ // assume's operand bundles.
+
// The remaining tests are all recursive, so bail out if we hit the limit.
if (Depth == MaxDepth)
continue;
@@ -564,20 +561,20 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
ConstantInt *C;
// assume(v = a)
if (match(Arg, m_c_ICmp(Pred, m_V, m_Value(A))) &&
- Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
KnownZero |= RHSKnownZero;
KnownOne |= RHSKnownOne;
// assume(v & b = a)
} else if (match(Arg,
m_c_ICmp(Pred, m_c_And(m_V, m_Value(B)), m_Value(A))) &&
Pred == ICmpInst::ICMP_EQ &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
APInt MaskKnownZero(BitWidth, 0), MaskKnownOne(BitWidth, 0);
- computeKnownBits(B, MaskKnownZero, MaskKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(B, MaskKnownZero, MaskKnownOne, Depth+1, Query(Q, II));
// For those bits in the mask that are known to be one, we can propagate
// known bits from the RHS to V.
@@ -587,11 +584,11 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
} else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_And(m_V, m_Value(B))),
m_Value(A))) &&
Pred == ICmpInst::ICMP_EQ &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
APInt MaskKnownZero(BitWidth, 0), MaskKnownOne(BitWidth, 0);
- computeKnownBits(B, MaskKnownZero, MaskKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(B, MaskKnownZero, MaskKnownOne, Depth+1, Query(Q, II));
// For those bits in the mask that are known to be one, we can propagate
// inverted known bits from the RHS to V.
@@ -601,11 +598,11 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
} else if (match(Arg,
m_c_ICmp(Pred, m_c_Or(m_V, m_Value(B)), m_Value(A))) &&
Pred == ICmpInst::ICMP_EQ &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0);
- computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, II));
// For those bits in B that are known to be zero, we can propagate known
// bits from the RHS to V.
@@ -615,11 +612,11 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
} else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_Or(m_V, m_Value(B))),
m_Value(A))) &&
Pred == ICmpInst::ICMP_EQ &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0);
- computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, II));
// For those bits in B that are known to be zero, we can propagate
// inverted known bits from the RHS to V.
@@ -629,11 +626,11 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
} else if (match(Arg,
m_c_ICmp(Pred, m_c_Xor(m_V, m_Value(B)), m_Value(A))) &&
Pred == ICmpInst::ICMP_EQ &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0);
- computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, II));
// For those bits in B that are known to be zero, we can propagate known
// bits from the RHS to V. For those bits in B that are known to be one,
@@ -646,11 +643,11 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
} else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_Xor(m_V, m_Value(B))),
m_Value(A))) &&
Pred == ICmpInst::ICMP_EQ &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
APInt BKnownZero(BitWidth, 0), BKnownOne(BitWidth, 0);
- computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(B, BKnownZero, BKnownOne, Depth+1, Query(Q, II));
// For those bits in B that are known to be zero, we can propagate
// inverted known bits from the RHS to V. For those bits in B that are
@@ -663,9 +660,9 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
} else if (match(Arg, m_c_ICmp(Pred, m_Shl(m_V, m_ConstantInt(C)),
m_Value(A))) &&
Pred == ICmpInst::ICMP_EQ &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
// For those bits in RHS that are known, we can propagate them to known
// bits in V shifted to the right by C.
KnownZero |= RHSKnownZero.lshr(C->getZExtValue());
@@ -674,9 +671,9 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
} else if (match(Arg, m_c_ICmp(Pred, m_Not(m_Shl(m_V, m_ConstantInt(C))),
m_Value(A))) &&
Pred == ICmpInst::ICMP_EQ &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
// For those bits in RHS that are known, we can propagate them inverted
// to known bits in V shifted to the right by C.
KnownZero |= RHSKnownOne.lshr(C->getZExtValue());
@@ -687,9 +684,9 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
m_AShr(m_V, m_ConstantInt(C))),
m_Value(A))) &&
Pred == ICmpInst::ICMP_EQ &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
// For those bits in RHS that are known, we can propagate them to known
// bits in V shifted to the right by C.
KnownZero |= RHSKnownZero << C->getZExtValue();
@@ -700,9 +697,9 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
m_AShr(m_V, m_ConstantInt(C)))),
m_Value(A))) &&
Pred == ICmpInst::ICMP_EQ &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
// For those bits in RHS that are known, we can propagate them inverted
// to known bits in V shifted to the right by C.
KnownZero |= RHSKnownOne << C->getZExtValue();
@@ -710,9 +707,9 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
// assume(v >=_s c) where c is non-negative
} else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
Pred == ICmpInst::ICMP_SGE &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
if (RHSKnownZero.isNegative()) {
// We know that the sign bit is zero.
@@ -721,9 +718,9 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
// assume(v >_s c) where c is at least -1.
} else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
Pred == ICmpInst::ICMP_SGT &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
if (RHSKnownOne.isAllOnesValue() || RHSKnownZero.isNegative()) {
// We know that the sign bit is zero.
@@ -732,9 +729,9 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
// assume(v <=_s c) where c is negative
} else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
Pred == ICmpInst::ICMP_SLE &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
if (RHSKnownOne.isNegative()) {
// We know that the sign bit is one.
@@ -743,9 +740,9 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
// assume(v <_s c) where c is non-positive
} else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
Pred == ICmpInst::ICMP_SLT &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
if (RHSKnownZero.isAllOnesValue() || RHSKnownOne.isNegative()) {
// We know that the sign bit is one.
@@ -754,9 +751,9 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
// assume(v <=_u c)
} else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
Pred == ICmpInst::ICMP_ULE &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
// Whatever high bits in c are zero are known to be zero.
KnownZero |=
@@ -764,13 +761,13 @@ static void computeKnownBitsFromAssume(const Value *V, APInt &KnownZero,
// assume(v <_u c)
} else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) &&
Pred == ICmpInst::ICMP_ULT &&
- isValidAssumeForContext(I, Q.CxtI, Q.DT)) {
+ isValidAssumeForContext(II, Q.CxtI, Q.DT)) {
APInt RHSKnownZero(BitWidth, 0), RHSKnownOne(BitWidth, 0);
- computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, I));
+ computeKnownBits(A, RHSKnownZero, RHSKnownOne, Depth+1, Query(Q, II));
// Whatever high bits in c are zero are known to be zero (if c is a power
// of 2, then one more).
- if (isKnownToBeAPowerOfTwo(A, false, Depth + 1, Query(Q, I)))
+ if (isKnownToBeAPowerOfTwo(A, false, Depth + 1, Query(Q, II)))
KnownZero |=
APInt::getHighBitsSet(BitWidth, RHSKnownZero.countLeadingOnes()+1);
else
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 2e50b2f3e06..958bd1110f3 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -2518,6 +2518,78 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
if (KnownOne.isAllOnesValue())
return eraseInstFromFunction(*II);
+ // For assumptions, add to the associated operand bundle the values to which
+ // the assumption might apply.
+ // Note: This code must be kept in-sync with the code in
+ // computeKnownBitsFromAssume in ValueTracking.
+ SmallVector<Value *, 16> Affected;
+ auto AddAffected = [&Affected](Value *V) {
+ if (isa<Argument>(V)) {
+ Affected.push_back(V);
+ } else if (auto *I = dyn_cast<Instruction>(V)) {
+ Affected.push_back(I);
+
+ if (I->getOpcode() == Instruction::BitCast ||
+ I->getOpcode() == Instruction::PtrToInt) {
+ V = I->getOperand(0);
+ if (isa<Instruction>(V) || isa<Argument>(V))
+ Affected.push_back(V);
+ }
+ }
+ };
+
+ CmpInst::Predicate Pred;
+ if (match(IIOperand, m_ICmp(Pred, m_Value(A), m_Value(B)))) {
+ AddAffected(A);
+ AddAffected(B);
+
+ if (Pred == ICmpInst::ICMP_EQ) {
+ // For equality comparisons, we handle the case of bit inversion.
+ auto AddAffectedFromEq = [&AddAffected](Value *V) {
+ Value *A;
+ if (match(V, m_Not(m_Value(A)))) {
+ AddAffected(A);
+ V = A;
+ }
+
+ Value *B;
+ ConstantInt *C;
+ if (match(V,
+ m_CombineOr(m_And(m_Value(A), m_Value(B)),
+ m_CombineOr(m_Or(m_Value(A), m_Value(B)),
+ m_Xor(m_Value(A), m_Value(B)))))) {
+ AddAffected(A);
+ AddAffected(B);
+ } else if (match(V,
+ m_CombineOr(m_Shl(m_Value(A), m_ConstantInt(C)),
+ m_CombineOr(m_LShr(m_Value(A), m_ConstantInt(C)),
+ m_AShr(m_Value(A),
+ m_ConstantInt(C)))))) {
+ AddAffected(A);
+ }
+ };
+
+ AddAffectedFromEq(A);
+ AddAffectedFromEq(B);
+ }
+ }
+
+ // If the list of affected values is the same as the existing list then
+ // there's nothing more to do here.
+ if (!Affected.empty())
+ if (auto OB = CI.getOperandBundle("affected"))
+ if (Affected.size() == OB.getValue().Inputs.size() &&
+ std::equal(Affected.begin(), Affected.end(),
+ OB.getValue().Inputs.begin()))
+ Affected.clear();
+
+ if (!Affected.empty()) {
+ Builder->CreateCall(AssumeIntrinsic, IIOperand,
+ OperandBundleDef("affected", Affected),
+ II->getName());
+ return eraseInstFromFunction(*II);
+ }
+
break;
}
case Intrinsic::experimental_gc_relocate: {
diff --git a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
index c1df3173c0f..82cfbd16ca3 100644
--- a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
+++ b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
@@ -425,9 +425,12 @@ bool AlignmentFromAssumptionsPass::runImpl(Function &F, AssumptionCache &AC,
NewSrcAlignments.clear();
bool Changed = false;
- for (auto &AssumeVH : AC.assumptions())
- if (AssumeVH)
- Changed |= processAssumption(cast<CallInst>(AssumeVH));
+
+ for (auto &B : F)
+ for (auto &I : B)
+ if (auto *II = dyn_cast<IntrinsicInst>(&I))
+ if (II->getIntrinsicID() == Intrinsic::assume)
+ Changed |= processAssumption(II);
return Changed;
}
OpenPOWER on IntegriCloud