[OPENMP] Codegen for 'atomic update' construct.

Adds atomic update codegen for the following forms of expressions:

x binop= expr;
x++;
++x;
x--;
--x;
x = x binop expr;
x = expr binop x;
If x and expr are integer and binop is associative or x is a LHS in a RHS of the assignment expression, and atomics are allowed for type of x on the target platform atomicrmw instruction is emitted.
Otherwise compare-and-swap sequence is emitted:

bb:
...
atomic load <x>
cont:
<expected> = phi [ <x>, label %bb ], [ <new_failed>, %cont ]
<desired> = <expected> binop <expr>
<res> = cmpxchg atomic &<x>, desired, expected
<new_failed> = <res>.field1;
br <res>field2, label %exit, label %cont
exit:
...

Differential Revision: http://reviews.llvm.org/D8536

llvm-svn: 233513

4 files changed, 246 insertions, 53 deletions

diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index b7222fc1cd6..d51c4fdb596 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -209,7 +209,7 @@ namespace {
     /// \param IsWeak true if atomic operation is weak, false otherwise.
     /// \returns Pair of values: previous value from storage (value type) and
     /// boolean flag (i1 type) with true if success and false otherwise.
-    std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchange(
+    std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
         RValue Expected, RValue Desired,
         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
@@ -235,13 +235,13 @@ namespace {
     /// \brief Emits atomic load as LLVM instruction.
     llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
     /// \brief Emits atomic compare-and-exchange op as a libcall.
-    std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeLibcall(
-        llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
+    std::pair<RValue, llvm::Value *> EmitAtomicCompareExchangeLibcall(
+        RValue Expected, RValue DesiredAddr,
         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent);
     /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
-    std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
-        llvm::Value *Expected, llvm::Value *Desired,
+    std::pair<RValue, llvm::Value *> EmitAtomicCompareExchangeOp(
+        RValue Expected, RValue Desired,
         llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
         llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
         bool IsWeak = false);
@@ -1291,7 +1291,7 @@ llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
   if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
     llvm::Value *Value = RVal.getScalarVal();
     if (isa<llvm::IntegerType>(Value->getType()))
-      return Value;
+      return CGF.EmitToMemory(Value, ValueTy);
     else {
       llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
           CGF.getLLVMContext(),
@@ -1312,13 +1312,15 @@ llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
                                        getAtomicAlignment().getQuantity());
 }
 
-std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
-    llvm::Value *Expected, llvm::Value *Desired, llvm::AtomicOrdering Success,
+std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
+    RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
     llvm::AtomicOrdering Failure, bool IsWeak) {
   // Do the atomic store.
+  auto *ExpectedVal = convertRValueToInt(Expected);
+  auto *DesiredVal = convertRValueToInt(Desired);
   auto *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
-  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, Expected, Desired, Success,
-                                               Failure);
+  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
+                                               Success, Failure);
   // Other decoration.
   Inst->setVolatile(LVal.isVolatileQualified());
   Inst->setWeak(IsWeak);
@@ -1326,16 +1328,20 @@ std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
   // Okay, turn that back into the original value type.
   auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
   auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
-  return std::make_pair(PreviousVal, SuccessFailureVal);
+  return std::make_pair(
+      ConvertIntToValueOrAtomic(PreviousVal, AggValueSlot::ignored(),
+                                SourceLocation(), /*AsValue=*/false),
+      SuccessFailureVal);
 }
 
-std::pair<llvm::Value *, llvm::Value *>
-AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
-                                             llvm::Value *DesiredAddr,
+std::pair<RValue, llvm::Value *>
+AtomicInfo::EmitAtomicCompareExchangeLibcall(RValue Expected, RValue Desired,
                                              llvm::AtomicOrdering Success,
                                              llvm::AtomicOrdering Failure) {
   // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
   // void *desired, int success, int failure);
+  auto *ExpectedAddr = materializeRValue(Expected);
+  auto *DesiredAddr = materializeRValue(Desired);
   CallArgList Args;
   Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
   Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
@@ -1352,12 +1358,14 @@ AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
            CGF.getContext().IntTy);
   auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
                                               CGF.getContext().BoolTy, Args);
-  auto *PreviousVal = CGF.Builder.CreateAlignedLoad(
-      ExpectedAddr, getValueAlignment().getQuantity());
-  return std::make_pair(PreviousVal, SuccessFailureRVal.getScalarVal());
+
+  return std::make_pair(
+      convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
+                          SourceLocation(), /*AsValue=*/false),
+      SuccessFailureRVal.getScalarVal());
 }
 
-std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
+std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
     RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
     llvm::AtomicOrdering Failure, bool IsWeak) {
   if (Failure >= Success)
@@ -1366,20 +1374,15 @@ std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
 
   // Check whether we should use a library call.
   if (shouldUseLibcall()) {
-    auto *ExpectedAddr = materializeRValue(Expected);
     // Produce a source address.
-    auto *DesiredAddr = materializeRValue(Desired);
-    return EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, Success,
+    return EmitAtomicCompareExchangeLibcall(Expected, Desired, Success,
                                             Failure);
   }
 
   // If we've got a scalar value of the right size, try to avoid going
   // through memory.
-  auto *ExpectedIntVal = convertRValueToInt(Expected);
-  auto *DesiredIntVal = convertRValueToInt(Desired);
-
-  return EmitAtomicCompareExchangeOp(ExpectedIntVal, DesiredIntVal, Success,
-                                     Failure, IsWeak);
+  return EmitAtomicCompareExchangeOp(Expected, Desired, Success, Failure,
+                                     IsWeak);
 }
 
 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
@@ -1498,20 +1501,14 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
       atomics.getAtomicType(), SourceLocation()));
   // Try to write new value using cmpxchg operation
   auto Pair = atomics.EmitAtomicCompareExchange(OriginalRValue, NewRValue, AO);
-  llvm::Value *OldValue = Pair.first;
-  if (!atomics.shouldUseLibcall())
-    // Convert integer value to original atomic type
-    OldValue = atomics.ConvertIntToValueOrAtomic(
-                           OldValue, AggValueSlot::ignored(), SourceLocation(),
-                           /*AsValue=*/false).getScalarVal();
-  PHI->addIncoming(OldValue, ContBB);
+  PHI->addIncoming(Pair.first.getScalarVal(), ContBB);
   Builder.CreateCondBr(Pair.second, ExitBB, ContBB);
   EmitBlock(ExitBB, /*IsFinished=*/true);
 }
 
 /// Emit a compare-and-exchange op for atomic type.
 ///
-std::pair<RValue, RValue> CodeGenFunction::EmitAtomicCompareExchange(
+std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
     LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
     llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
     AggValueSlot Slot) {
@@ -1525,13 +1522,78 @@ std::pair<RValue, RValue> CodeGenFunction::EmitAtomicCompareExchange(
              Obj.getAddress()->getType()->getPointerElementType());
   AtomicInfo Atomics(*this, Obj);
 
-  auto Pair = Atomics.EmitAtomicCompareExchange(Expected, Desired, Success,
-                                                Failure, IsWeak);
-  return std::make_pair(Atomics.shouldUseLibcall()
-                            ? RValue::get(Pair.first)
-                            : Atomics.ConvertIntToValueOrAtomic(
-                                  Pair.first, Slot, Loc, /*AsValue=*/true),
-                        RValue::get(Pair.second));
+  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
+                                           IsWeak);
+}
+
+void CodeGenFunction::EmitAtomicUpdate(
+    LValue LVal, llvm::AtomicOrdering AO,
+    const std::function<RValue(RValue)> &UpdateOp, bool IsVolatile) {
+  AtomicInfo Atomics(*this, LVal);
+  LValue AtomicLVal = Atomics.getAtomicLValue();
+
+  // Atomic load of prev value.
+  RValue OldRVal =
+      Atomics.EmitAtomicLoad(AggValueSlot::ignored(), SourceLocation(),
+                             /*AsValue=*/false, AO, IsVolatile);
+  bool IsScalar = OldRVal.isScalar();
+  auto *OldVal =
+      IsScalar ? OldRVal.getScalarVal() : Atomics.convertRValueToInt(OldRVal);
+  // For non-simple lvalues perform compare-and-swap procedure.
+  auto *ContBB = createBasicBlock("atomic_cont");
+  auto *ExitBB = createBasicBlock("atomic_exit");
+  auto *CurBB = Builder.GetInsertBlock();
+  EmitBlock(ContBB);
+  llvm::PHINode *PHI = Builder.CreatePHI(OldVal->getType(),
+                                         /*NumReservedValues=*/2);
+  PHI->addIncoming(OldVal, CurBB);
+  RValue OriginalRValue =
+      IsScalar ? RValue::get(PHI) : Atomics.ConvertIntToValueOrAtomic(
+                                        PHI, AggValueSlot::ignored(),
+                                        SourceLocation(), /*AsValue=*/false);
+  // Build new lvalue for temp address
+  LValue UpdateLVal;
+  llvm::Value *Ptr = nullptr;
+  RValue UpRVal;
+  if (AtomicLVal.isSimple()) {
+    UpRVal = OriginalRValue;
+  } else {
+    // Build new lvalue for temp address
+    Ptr = Atomics.materializeRValue(OriginalRValue);
+    if (AtomicLVal.isBitField())
+      UpdateLVal =
+          LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
+                               AtomicLVal.getType(), AtomicLVal.getAlignment());
+    else if (AtomicLVal.isVectorElt())
+      UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
+                                         AtomicLVal.getType(),
+                                         AtomicLVal.getAlignment());
+    else {
+      assert(AtomicLVal.isExtVectorElt());
+      UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
+                                            AtomicLVal.getType(),
+                                            AtomicLVal.getAlignment());
+    }
+    UpdateLVal.setTBAAInfo(LVal.getTBAAInfo());
+    UpRVal = EmitLoadOfLValue(UpdateLVal, SourceLocation());
+  }
+  // Store new value in the corresponding memory area
+  RValue NewRVal = UpdateOp(UpRVal);
+  if (!AtomicLVal.isSimple()) {
+    EmitStoreThroughLValue(NewRVal, UpdateLVal);
+    // Load new value
+    NewRVal = RValue::get(
+        EmitLoadOfScalar(Ptr, AtomicLVal.isVolatile(),
+                         Atomics.getAtomicAlignment().getQuantity(),
+                         Atomics.getAtomicType(), SourceLocation()));
+  }
+  // Try to write new value using cmpxchg operation
+  auto Pair = Atomics.EmitAtomicCompareExchange(OriginalRValue, NewRVal, AO);
+  OldVal = IsScalar ? Pair.first.getScalarVal()
+                    : Atomics.convertRValueToInt(Pair.first);
+  PHI->addIncoming(OldVal, ContBB);
+  Builder.CreateCondBr(Pair.second, ExitBB, ContBB);
+  EmitBlock(ExitBB, /*IsFinished=*/true);
 }
 
 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 16ce69d7999..e9361e28291 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -1845,10 +1845,9 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     llvm::BasicBlock *opBB = Builder.GetInsertBlock();
     llvm::BasicBlock *contBB = CGF.createBasicBlock("atomic_cont", CGF.CurFn);
     auto Pair = CGF.EmitAtomicCompareExchange(
-        LV, RValue::get(atomicPHI), RValue::get(CGF.EmitToMemory(value, type)),
-        E->getExprLoc());
-    llvm::Value *old = Pair.first.getScalarVal();
-    llvm::Value *success = Pair.second.getScalarVal();
+        LV, RValue::get(atomicPHI), RValue::get(value), E->getExprLoc());
+    llvm::Value *old = CGF.EmitToMemory(Pair.first.getScalarVal(), type);
+    llvm::Value *success = Pair.second;
     atomicPHI->addIncoming(old, opBB);
     Builder.CreateCondBr(success, contBB, opBB);
     Builder.SetInsertPoint(contBB);
@@ -2189,10 +2188,9 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue(
     llvm::BasicBlock *opBB = Builder.GetInsertBlock();
     llvm::BasicBlock *contBB = CGF.createBasicBlock("atomic_cont", CGF.CurFn);
     auto Pair = CGF.EmitAtomicCompareExchange(
-        LHSLV, RValue::get(atomicPHI),
-        RValue::get(CGF.EmitToMemory(Result, LHSTy)), E->getExprLoc());
-    llvm::Value *old = Pair.first.getScalarVal();
-    llvm::Value *success = Pair.second.getScalarVal();
+        LHSLV, RValue::get(atomicPHI), RValue::get(Result), E->getExprLoc());
+    llvm::Value *old = CGF.EmitToMemory(Pair.first.getScalarVal(), LHSTy);
+    llvm::Value *success = Pair.second;
     atomicPHI->addIncoming(old, opBB);
     Builder.CreateCondBr(success, contBB, opBB);
     Builder.SetInsertPoint(contBB);
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index f74e47a4ca5..1be1a4d3231 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -1119,9 +1119,136 @@ static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
     CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
 }
 
+static Optional<llvm::AtomicRMWInst::BinOp>
+getCompatibleAtomicRMWBinOp(ASTContext &Context, BinaryOperatorKind Op,
+                            bool IsXLHSInRHSPart, LValue XLValue,
+                            RValue ExprRValue) {
+  Optional<llvm::AtomicRMWInst::BinOp> RMWOp;
+  // Allow atomicrmw only if 'x' and 'expr' are integer values, lvalue for 'x'
+  // expression is simple and atomic is allowed for the given type for the
+  // target platform.
+  if (ExprRValue.isScalar() &&
+      ExprRValue.getScalarVal()->getType()->isIntegerTy() &&
+      XLValue.isSimple() &&
+      (isa<llvm::ConstantInt>(ExprRValue.getScalarVal()) ||
+       (ExprRValue.getScalarVal()->getType() ==
+        XLValue.getAddress()->getType()->getPointerElementType())) &&
+      Context.getTargetInfo().hasBuiltinAtomic(
+          Context.getTypeSize(XLValue.getType()),
+          Context.toBits(XLValue.getAlignment()))) {
+    switch (Op) {
+    case BO_Add:
+      RMWOp = llvm::AtomicRMWInst::Add;
+      break;
+    case BO_Sub:
+      if (IsXLHSInRHSPart) {
+        RMWOp = llvm::AtomicRMWInst::Sub;
+      }
+      break;
+    case BO_And:
+      RMWOp = llvm::AtomicRMWInst::And;
+      break;
+    case BO_Or:
+      RMWOp = llvm::AtomicRMWInst::Or;
+      break;
+    case BO_Xor:
+      RMWOp = llvm::AtomicRMWInst::Xor;
+      break;
+    case BO_Mul:
+    case BO_Div:
+    case BO_Rem:
+    case BO_Shl:
+    case BO_Shr:
+      break;
+    case BO_PtrMemD:
+    case BO_PtrMemI:
+    case BO_LT:
+    case BO_GT:
+    case BO_LE:
+    case BO_GE:
+    case BO_EQ:
+    case BO_NE:
+    case BO_LAnd:
+    case BO_LOr:
+    case BO_Assign:
+    case BO_MulAssign:
+    case BO_DivAssign:
+    case BO_RemAssign:
+    case BO_AddAssign:
+    case BO_SubAssign:
+    case BO_ShlAssign:
+    case BO_ShrAssign:
+    case BO_AndAssign:
+    case BO_XorAssign:
+    case BO_OrAssign:
+    case BO_Comma:
+      llvm_unreachable("Unexpected binary operation in 'atomic update'.");
+    }
+  }
+  return std::move(RMWOp);
+}
+
+static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
+                                    const Expr *X, const Expr *E,
+                                    const Expr *UE, bool IsXLHSInRHSPart,
+                                    SourceLocation Loc) {
+  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
+         "Update expr in 'atomic update' must be a binary operator.");
+  auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
+  // Update expressions are allowed to have the following forms:
+  // x binop= expr; -> xrval + expr;
+  // x++, ++x -> xrval + 1;
+  // x--, --x -> xrval - 1;
+  // x = x binop expr; -> xrval binop expr
+  // x = expr Op x; - > expr binop xrval;
+  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
+  LValue XLValue = CGF.EmitLValue(X);
+  RValue ExprRValue = CGF.EmitAnyExpr(E);
+  const auto &Op =
+      getCompatibleAtomicRMWBinOp(CGF.CGM.getContext(), BOUE->getOpcode(),
+                                  IsXLHSInRHSPart, XLValue, ExprRValue);
+  auto AO = IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic;
+  if (Op) {
+    auto *ExprVal = ExprRValue.getScalarVal();
+    if (auto *IC = dyn_cast<llvm::ConstantInt>(ExprVal)) {
+      ExprVal = CGF.Builder.CreateIntCast(
+          IC, XLValue.getAddress()->getType()->getPointerElementType(),
+          XLValue.getType()->hasSignedIntegerRepresentation());
+    }
+    CGF.Builder.CreateAtomicRMW(*Op, XLValue.getAddress(), ExprVal, AO);
+  } else {
+    auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
+    auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
+    CodeGenFunction::OpaqueValueMapping MapExpr(
+        CGF, IsXLHSInRHSPart ? RHS : LHS, ExprRValue);
+    auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
+    if (XLValue.isGlobalReg()) {
+      // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
+      // 'xrval'.
+      CodeGenFunction::OpaqueValueMapping MapX(
+          CGF, XRValExpr, CGF.EmitLoadOfLValue(XLValue, Loc));
+      CGF.EmitStoreThroughLValue(CGF.EmitAnyExpr(UE), XLValue);
+    } else {
+      // Perform compare-and-swap procedure.
+      CGF.EmitAtomicUpdate(
+          XLValue, AO, [&CGF, &UE, &XRValExpr](RValue XRVal) -> RValue {
+            CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRVal);
+            return CGF.EmitAnyExpr(UE);
+          }, /*IsVolatile=*/false);
+    }
+  }
+  // OpenMP, 2.12.6, atomic Construct
+  // Any atomic construct with a seq_cst clause forces the atomically
+  // performed operation to include an implicit flush operation without a
+  // list.
+  if (IsSeqCst)
+    CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
+}
+
 static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
                               bool IsSeqCst, const Expr *X, const Expr *V,
-                              const Expr *E, SourceLocation Loc) {
+                              const Expr *E, const Expr *UE,
+                              bool IsXLHSInRHSPart, SourceLocation Loc) {
   switch (Kind) {
   case OMPC_read:
     EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
@@ -1129,7 +1256,10 @@ static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
   case OMPC_write:
     EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
     break;
+  case OMPC_unknown:
   case OMPC_update:
+    EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
+    break;
   case OMPC_capture:
     llvm_unreachable("CodeGen for 'omp atomic clause' is not supported yet.");
   case OMPC_if:
@@ -1156,7 +1286,6 @@ static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
   case OMPC_untied:
   case OMPC_threadprivate:
   case OMPC_mergeable:
-  case OMPC_unknown:
     llvm_unreachable("Clause is not allowed in 'omp atomic'.");
   }
 }
@@ -1179,7 +1308,7 @@ void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
   InlinedOpenMPRegionScopeRAII Region(*this, S);
 
   EmitOMPAtomicExpr(*this, Kind, IsSeqCst, S.getX(), S.getV(), S.getExpr(),
-                    S.getLocStart());
+                    S.getUpdateExpr(), S.isXLHSInRHSPart(), S.getLocStart());
 }
 
 void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) {
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 151eb7e48d9..2473ab6898e 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -2123,12 +2123,16 @@ public:
   void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
                        bool IsVolatile, bool isInit);
 
-  std::pair<RValue, RValue> EmitAtomicCompareExchange(
+  std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
       LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
       llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
       llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
       bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());
 
+  void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
+                        const std::function<RValue(RValue)> &UpdateOp,
+                        bool IsVolatile);
+
   /// EmitToMemory - Change a scalar value from its value
   /// representation to its in-memory representation.
   llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);