summaryrefslogtreecommitdiffstats
path: root/clang/lib/CodeGen/CGOpenMPRuntime.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'clang/lib/CodeGen/CGOpenMPRuntime.cpp')
-rw-r--r--clang/lib/CodeGen/CGOpenMPRuntime.cpp263
1 files changed, 263 insertions, 0 deletions
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 59f352dcd4c..735cacf0b7d 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -19,6 +19,7 @@
#include "clang/AST/Decl.h"
#include "clang/AST/OpenMPClause.h"
#include "clang/AST/StmtOpenMP.h"
+#include "clang/AST/StmtVisitor.h"
#include "clang/Basic/BitmaskEnum.h"
#include "clang/CodeGen/ConstantInitBuilder.h"
#include "llvm/ADT/ArrayRef.h"
@@ -11401,6 +11402,268 @@ bool CGOpenMPRuntime::isNontemporalDecl(const ValueDecl *VD) const {
[VD](const NontemporalDeclsSet &Set) { return Set.count(VD) > 0; });
}
+CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
+ CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal)
+ : CGM(CGF.CGM),
+ NeedToPush(llvm::any_of(S.getClausesOfKind<OMPLastprivateClause>(),
+ [](const OMPLastprivateClause *C) {
+ return C->getKind() ==
+ OMPC_LASTPRIVATE_conditional;
+ })) {
+ assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
+ if (!NeedToPush)
+ return;
+ LastprivateConditionalData &Data =
+ CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
+ for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
+ if (C->getKind() != OMPC_LASTPRIVATE_conditional)
+ continue;
+
+ for (const Expr *Ref : C->varlists()) {
+ Data.DeclToUniqeName.try_emplace(
+ cast<DeclRefExpr>(Ref->IgnoreParenImpCasts())->getDecl(),
+ generateUniqueName(CGM, "pl_cond", Ref));
+ }
+ }
+ Data.IVLVal = IVLVal;
+ // In simd only mode or for simd directives no need to generate threadprivate
+ // references for the loop iteration counter, we can use the original one
+ // since outlining cannot happen in simd regions.
+ if (CGF.getLangOpts().OpenMPSimd ||
+ isOpenMPSimdDirective(S.getDirectiveKind())) {
+ Data.UseOriginalIV = true;
+ return;
+ }
+ llvm::SmallString<16> Buffer;
+ llvm::raw_svector_ostream OS(Buffer);
+ PresumedLoc PLoc =
+ CGM.getContext().getSourceManager().getPresumedLoc(S.getBeginLoc());
+ assert(PLoc.isValid() && "Source location is expected to be always valid.");
+
+ llvm::sys::fs::UniqueID ID;
+ if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
+ CGM.getDiags().Report(diag::err_cannot_open_file)
+ << PLoc.getFilename() << EC.message();
+ OS << "$pl_cond_" << ID.getDevice() << "_" << ID.getFile() << "_"
+ << PLoc.getLine() << "_" << PLoc.getColumn() << "$iv";
+ Data.IVName = OS.str();
+
+ // Global loop counter. Required to handle inner parallel-for regions.
+ // global_iv = &iv;
+ QualType PtrIVTy = CGM.getContext().getPointerType(IVLVal.getType());
+ Address GlobIVAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
+ CGF, PtrIVTy, Data.IVName);
+ LValue GlobIVLVal = CGF.MakeAddrLValue(GlobIVAddr, PtrIVTy);
+ CGF.EmitStoreOfScalar(IVLVal.getPointer(CGF), GlobIVLVal);
+}
+
+CGOpenMPRuntime::LastprivateConditionalRAII::~LastprivateConditionalRAII() {
+ if (!NeedToPush)
+ return;
+ CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
+}
+
+namespace {
+/// Checks if the lastprivate conditional variable is referenced in LHS.
+class LastprivateConditionalRefChecker final
+ : public ConstStmtVisitor<LastprivateConditionalRefChecker, bool> {
+ CodeGenFunction &CGF;
+ ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM;
+ const Expr *FoundE = nullptr;
+ const Decl *FoundD = nullptr;
+ StringRef UniqueDeclName;
+ LValue IVLVal;
+ StringRef IVName;
+ SourceLocation Loc;
+ bool UseOriginalIV = false;
+
+public:
+ bool VisitDeclRefExpr(const DeclRefExpr *E) {
+ for (const CGOpenMPRuntime::LastprivateConditionalData &D :
+ llvm::reverse(LPM)) {
+ auto It = D.DeclToUniqeName.find(E->getDecl());
+ if (It == D.DeclToUniqeName.end())
+ continue;
+ FoundE = E;
+ FoundD = E->getDecl()->getCanonicalDecl();
+ UniqueDeclName = It->getSecond();
+ IVLVal = D.IVLVal;
+ IVName = D.IVName;
+ UseOriginalIV = D.UseOriginalIV;
+ break;
+ }
+ return FoundE == E;
+ }
+ bool VisitMemberExpr(const MemberExpr *E) {
+ if (!CGF.IsWrappedCXXThis(E->getBase()))
+ return false;
+ for (const CGOpenMPRuntime::LastprivateConditionalData &D :
+ llvm::reverse(LPM)) {
+ auto It = D.DeclToUniqeName.find(E->getMemberDecl());
+ if (It == D.DeclToUniqeName.end())
+ continue;
+ FoundE = E;
+ FoundD = E->getMemberDecl()->getCanonicalDecl();
+ UniqueDeclName = It->getSecond();
+ IVLVal = D.IVLVal;
+ IVName = D.IVName;
+ UseOriginalIV = D.UseOriginalIV;
+ break;
+ }
+ return FoundE == E;
+ }
+ bool VisitStmt(const Stmt *S) {
+ for (const Stmt *Child : S->children()) {
+ if (!Child)
+ continue;
+ if (const auto *E = dyn_cast<Expr>(Child))
+ if (!E->isGLValue())
+ continue;
+ if (Visit(Child))
+ return true;
+ }
+ return false;
+ }
+ explicit LastprivateConditionalRefChecker(
+ CodeGenFunction &CGF,
+ ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM)
+ : CGF(CGF), LPM(LPM) {}
+ std::tuple<const Expr *, const Decl *, StringRef, LValue, StringRef, bool>
+ getFoundData() const {
+ return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, IVName,
+ UseOriginalIV);
+ }
+};
+} // namespace
+
+void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
+ const Expr *LHS) {
+ if (CGF.getLangOpts().OpenMP < 50)
+ return;
+ LastprivateConditionalRefChecker Checker(CGF, LastprivateConditionalStack);
+ if (!Checker.Visit(LHS))
+ return;
+ const Expr *FoundE;
+ const Decl *FoundD;
+ StringRef UniqueDeclName;
+ LValue IVLVal;
+ StringRef IVName;
+ bool UseOriginalIV;
+ std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, IVName, UseOriginalIV) =
+ Checker.getFoundData();
+
+ // Last updated loop counter for the lastprivate conditional var.
+ // int<xx> last_iv = 0;
+ llvm::Type *LLIVTy = CGF.ConvertTypeForMem(IVLVal.getType());
+ llvm::Constant *LastIV =
+ getOrCreateInternalVariable(LLIVTy, UniqueDeclName + "$iv");
+ cast<llvm::GlobalVariable>(LastIV)->setAlignment(
+ IVLVal.getAlignment().getAsAlign());
+ LValue LastIVLVal = CGF.MakeNaturalAlignAddrLValue(LastIV, IVLVal.getType());
+
+ // Private address of the lastprivate conditional in the current context.
+ // priv_a
+ LValue LVal = CGF.EmitLValue(FoundE);
+ // Last value of the lastprivate conditional.
+ // decltype(priv_a) last_a;
+ llvm::Constant *Last = getOrCreateInternalVariable(
+ LVal.getAddress(CGF).getElementType(), UniqueDeclName);
+ cast<llvm::GlobalVariable>(Last)->setAlignment(
+ LVal.getAlignment().getAsAlign());
+ LValue LastLVal =
+ CGF.MakeAddrLValue(Last, LVal.getType(), LVal.getAlignment());
+
+ // Global loop counter. Required to handle inner parallel-for regions.
+ // global_iv
+ if (!UseOriginalIV) {
+ QualType PtrIVTy = CGM.getContext().getPointerType(IVLVal.getType());
+ Address IVAddr = getAddrOfArtificialThreadPrivate(CGF, PtrIVTy, IVName);
+ IVLVal =
+ CGF.EmitLoadOfPointerLValue(IVAddr, PtrIVTy->castAs<PointerType>());
+ }
+ llvm::Value *IVVal = CGF.EmitLoadOfScalar(IVLVal, FoundE->getExprLoc());
+
+ // #pragma omp critical(a)
+ // if (last_iv <= iv) {
+ // last_iv = iv;
+ // last_a = priv_a;
+ // }
+ auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
+ FoundE](CodeGenFunction &CGF, PrePostActionTy &Action) {
+ Action.Enter(CGF);
+ llvm::Value *LastIVVal =
+ CGF.EmitLoadOfScalar(LastIVLVal, FoundE->getExprLoc());
+ // (last_iv <= global_iv) ? Check if the variable is updated and store new
+ // value in global var.
+ llvm::Value *CmpRes;
+ if (IVLVal.getType()->isSignedIntegerType()) {
+ CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal);
+ } else {
+ assert(IVLVal.getType()->isUnsignedIntegerType() &&
+ "Loop iteration variable must be integer.");
+ CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal);
+ }
+ llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lp_cond_then");
+ llvm::BasicBlock *ExitBB = CGF.createBasicBlock("lp_cond_exit");
+ CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
+ // {
+ CGF.EmitBlock(ThenBB);
+
+ // last_iv = global_iv;
+ CGF.EmitStoreOfScalar(IVVal, LastIVLVal);
+
+ // last_a = priv_a;
+ switch (CGF.getEvaluationKind(LVal.getType())) {
+ case TEK_Scalar: {
+ llvm::Value *PrivVal = CGF.EmitLoadOfScalar(LVal, FoundE->getExprLoc());
+ CGF.EmitStoreOfScalar(PrivVal, LastLVal);
+ break;
+ }
+ case TEK_Complex: {
+ CodeGenFunction::ComplexPairTy PrivVal =
+ CGF.EmitLoadOfComplex(LVal, FoundE->getExprLoc());
+ CGF.EmitStoreOfComplex(PrivVal, LastLVal, /*isInit=*/false);
+ break;
+ }
+ case TEK_Aggregate:
+ llvm_unreachable(
+ "Aggregates are not supported in lastprivate conditional.");
+ }
+ // }
+ CGF.EmitBranch(ExitBB);
+ // There is no need to emit line number for unconditional branch.
+ (void)ApplyDebugLocation::CreateEmpty(CGF);
+ CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
+ };
+
+ if (CGM.getLangOpts().OpenMPSimd) {
+ // Do not emit as a critical region as no parallel region could be emitted.
+ RegionCodeGenTy ThenRCG(CodeGen);
+ ThenRCG(CGF);
+ } else {
+ emitCriticalRegion(CGF, UniqueDeclName, CodeGen, FoundE->getExprLoc());
+ }
+}
+
+void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate(
+ CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD,
+ SourceLocation Loc) {
+ if (CGF.getLangOpts().OpenMP < 50)
+ return;
+ auto It = LastprivateConditionalStack.back().DeclToUniqeName.find(VD);
+ assert(It != LastprivateConditionalStack.back().DeclToUniqeName.end() &&
+ "Unknown lastprivate conditional variable.");
+ StringRef UniqueName = It->getSecond();
+ llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(UniqueName);
+ // The variable was not updated in the region - exit.
+ if (!GV)
+ return;
+ LValue LPLVal = CGF.MakeAddrLValue(
+ GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment());
+ llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc);
+ CGF.EmitStoreOfScalar(Res, PrivLVal);
+}
+
llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
OpenPOWER on IntegriCloud