[analyzer] Support for OSObjects out parameters in RetainCountChecker

rdar://46357478
rdar://47121327

Differential Revision: https://reviews.llvm.org/D56240

llvm-svn: 350982

3 files changed, 217 insertions, 72 deletions

diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
index 371221229b2..0652af85664 100644
--- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
@@ -529,38 +529,92 @@ void RetainCountChecker::processSummaryOfInlined(const RetainSummary &Summ,
   C.addTransition(state);
 }
 
-static ProgramStateRef updateOutParameter(ProgramStateRef State,
-                                          SVal ArgVal,
-                                          ArgEffectKind Effect) {
-  auto *ArgRegion = dyn_cast_or_null<TypedValueRegion>(ArgVal.getAsRegion());
-  if (!ArgRegion)
-    return State;
-
-  QualType PointeeTy = ArgRegion->getValueType();
-  if (!coreFoundation::isCFObjectRef(PointeeTy))
-    return State;
-
-  SVal PointeeVal = State->getSVal(ArgRegion);
-  SymbolRef Pointee = PointeeVal.getAsLocSymbol();
-  if (!Pointee)
-    return State;
-
-  switch (Effect) {
-  case UnretainedOutParameter:
-    State = setRefBinding(State, Pointee,
-                          RefVal::makeNotOwned(ObjKind::CF, PointeeTy));
-    break;
-  case RetainedOutParameter:
-    // Do nothing. Retained out parameters will either point to a +1 reference
-    // or NULL, but the way you check for failure differs depending on the API.
-    // Consequently, we don't have a good way to track them yet.
-    break;
+static bool shouldEscapeRegion(const MemRegion *R) {
 
-  default:
-    llvm_unreachable("only for out parameters");
+  // We do not currently model what happens when a symbol is
+  // assigned to a struct field, so be conservative here and let the symbol
+  // go. TODO: This could definitely be improved upon.
+  return !R->hasStackStorage() || !isa<VarRegion>(R);
+}
+
+static SmallVector<ProgramStateRef, 2>
+updateOutParameters(ProgramStateRef State, const RetainSummary &Summ,
+                    const CallEvent &CE) {
+
+  SVal L = CE.getReturnValue();
+
+  // Splitting is required to support out parameters,
+  // as out parameters might be created only on the "success" branch.
+  // We want to avoid eagerly splitting unless out parameters are actually
+  // needed.
+  bool SplitNecessary = false;
+  for (auto &P : Summ.getArgEffects())
+    if (P.second.getKind() == RetainedOutParameterOnNonZero ||
+        P.second.getKind() == RetainedOutParameterOnZero)
+      SplitNecessary = true;
+
+  ProgramStateRef AssumeNonZeroReturn = State;
+  ProgramStateRef AssumeZeroReturn = State;
+
+  if (SplitNecessary) {
+    if (auto DL = L.getAs<DefinedOrUnknownSVal>()) {
+      AssumeNonZeroReturn = AssumeNonZeroReturn->assume(*DL, true);
+      AssumeZeroReturn = AssumeZeroReturn->assume(*DL, false);
+    }
   }
 
-  return State;
+  for (unsigned idx = 0, e = CE.getNumArgs(); idx != e; ++idx) {
+    SVal ArgVal = CE.getArgSVal(idx);
+    ArgEffect AE = Summ.getArg(idx);
+
+    auto *ArgRegion = dyn_cast_or_null<TypedValueRegion>(ArgVal.getAsRegion());
+    if (!ArgRegion)
+      continue;
+
+    QualType PointeeTy = ArgRegion->getValueType();
+    SVal PointeeVal = State->getSVal(ArgRegion);
+    SymbolRef Pointee = PointeeVal.getAsLocSymbol();
+    if (!Pointee)
+      continue;
+
+    if (shouldEscapeRegion(ArgRegion))
+      continue;
+
+    auto makeNotOwnedParameter = [&](ProgramStateRef St) {
+      return setRefBinding(St, Pointee,
+                           RefVal::makeNotOwned(AE.getObjKind(), PointeeTy));
+    };
+    auto makeOwnedParameter = [&](ProgramStateRef St) {
+      return setRefBinding(St, Pointee,
+                           RefVal::makeOwned(ObjKind::OS, PointeeTy));
+    };
+
+    switch (AE.getKind()) {
+    case UnretainedOutParameter:
+      AssumeNonZeroReturn = makeNotOwnedParameter(AssumeNonZeroReturn);
+      AssumeZeroReturn = makeNotOwnedParameter(AssumeZeroReturn);
+      break;
+    case RetainedOutParameter:
+      AssumeNonZeroReturn = makeOwnedParameter(AssumeNonZeroReturn);
+      AssumeZeroReturn = makeOwnedParameter(AssumeZeroReturn);
+      break;
+    case RetainedOutParameterOnNonZero:
+      AssumeNonZeroReturn = makeOwnedParameter(AssumeNonZeroReturn);
+      break;
+    case RetainedOutParameterOnZero:
+      AssumeZeroReturn = makeOwnedParameter(AssumeZeroReturn);
+      break;
+    default:
+      break;
+    }
+  }
+
+  if (SplitNecessary) {
+    return {AssumeNonZeroReturn, AssumeZeroReturn};
+  } else {
+    assert(AssumeZeroReturn == AssumeNonZeroReturn);
+    return {AssumeZeroReturn};
+  }
 }
 
 void RetainCountChecker::checkSummary(const RetainSummary &Summ,
@@ -582,10 +636,7 @@ void RetainCountChecker::checkSummary(const RetainSummary &Summ,
     SVal V = CallOrMsg.getArgSVal(idx);
 
     ArgEffect Effect = Summ.getArg(idx);
-    if (Effect.getKind() == RetainedOutParameter ||
-        Effect.getKind() == UnretainedOutParameter) {
-      state = updateOutParameter(state, V, Effect.getKind());
-    } else if (SymbolRef Sym = V.getAsLocSymbol()) {
+    if (SymbolRef Sym = V.getAsLocSymbol()) {
       if (const RefVal *T = getRefBinding(state, Sym)) {
 
         if (shouldEscapeOSArgumentOnCall(CallOrMsg, idx, T))
@@ -661,10 +712,15 @@ void RetainCountChecker::checkSummary(const RetainSummary &Summ,
       state = setRefBinding(state, Sym, *updatedRefVal);
   }
 
-  if (DeallocSent) {
-    C.addTransition(state, C.getPredecessor(), &DeallocSentTag);
-  } else {
-    C.addTransition(state);
+  SmallVector<ProgramStateRef, 2> Out =
+      updateOutParameters(state, Summ, CallOrMsg);
+
+  for (ProgramStateRef St : Out) {
+    if (DeallocSent) {
+      C.addTransition(St, C.getPredecessor(), &DeallocSentTag);
+    } else {
+      C.addTransition(St);
+    }
   }
 }
 
@@ -700,6 +756,8 @@ ProgramStateRef RetainCountChecker::updateSymbol(ProgramStateRef state,
   switch (AE.getKind()) {
     case UnretainedOutParameter:
     case RetainedOutParameter:
+    case RetainedOutParameterOnZero:
+    case RetainedOutParameterOnNonZero:
       llvm_unreachable("Applies to pointer-to-pointer parameters, which should "
                        "not have ref state.");
 
@@ -1094,29 +1152,10 @@ void RetainCountChecker::checkBind(SVal loc, SVal val, const Stmt *S,
   //
   // (1) we are binding to something that is not a memory region.
   // (2) we are binding to a memregion that does not have stack storage
-  // (3) we are binding to a memregion with stack storage that the store
-  //     does not understand.
   ProgramStateRef state = C.getState();
 
   if (auto regionLoc = loc.getAs<loc::MemRegionVal>()) {
-    escapes = !regionLoc->getRegion()->hasStackStorage();
-
-    if (!escapes) {
-      // To test (3), generate a new state with the binding added.  If it is
-      // the same state, then it escapes (since the store cannot represent
-      // the binding).
-      // Do this only if we know that the store is not supposed to generate the
-      // same state.
-      SVal StoredVal = state->getSVal(regionLoc->getRegion());
-      if (StoredVal != val)
-        escapes = (state == (state->bindLoc(*regionLoc, val, C.getLocationContext())));
-    }
-    if (!escapes) {
-      // Case 4: We do not currently model what happens when a symbol is
-      // assigned to a struct field, so be conservative here and let the symbol
-      // go. TODO: This could definitely be improved upon.
-      escapes = !isa<VarRegion>(regionLoc->getRegion());
-    }
+    escapes = shouldEscapeRegion(regionLoc->getRegion());
   }
 
   // If we are storing the value into an auto function scope variable annotated
diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
index 4309603862a..cda1a928de1 100644
--- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
@@ -113,11 +113,31 @@ static bool shouldGenerateNote(llvm::raw_string_ostream &os,
   return true;
 }
 
+/// Finds argument index of the out paramter in the call {@code S}
+/// corresponding to the symbol {@code Sym}.
+/// If none found, returns None.
+static Optional<unsigned> findArgIdxOfSymbol(ProgramStateRef CurrSt,
+                                             const LocationContext *LCtx,
+                                             SymbolRef &Sym,
+                                             Optional<CallEventRef<>> CE) {
+  if (!CE)
+    return None;
+
+  for (unsigned Idx = 0; Idx < (*CE)->getNumArgs(); Idx++)
+    if (const MemRegion *MR = (*CE)->getArgSVal(Idx).getAsRegion())
+      if (const auto *TR = dyn_cast<TypedValueRegion>(MR))
+        if (CurrSt->getSVal(MR, TR->getValueType()).getAsSymExpr() == Sym)
+          return Idx;
+
+  return None;
+}
+
 static void generateDiagnosticsForCallLike(ProgramStateRef CurrSt,
                                            const LocationContext *LCtx,
                                            const RefVal &CurrV, SymbolRef &Sym,
                                            const Stmt *S,
                                            llvm::raw_string_ostream &os) {
+  CallEventManager &Mgr = CurrSt->getStateManager().getCallEventManager();
   if (const CallExpr *CE = dyn_cast<CallExpr>(S)) {
     // Get the name of the callee (if it is available)
     // from the tracked SVal.
@@ -139,7 +159,6 @@ static void generateDiagnosticsForCallLike(ProgramStateRef CurrSt,
     os << "Operator 'new'";
   } else {
     assert(isa<ObjCMessageExpr>(S));
-    CallEventManager &Mgr = CurrSt->getStateManager().getCallEventManager();
     CallEventRef<ObjCMethodCall> Call =
         Mgr.getObjCMethodCall(cast<ObjCMessageExpr>(S), CurrSt, LCtx);
 
@@ -156,7 +175,15 @@ static void generateDiagnosticsForCallLike(ProgramStateRef CurrSt,
     }
   }
 
-   os << " returns ";
+  Optional<CallEventRef<>> CE = Mgr.getCall(S, CurrSt, LCtx);
+  auto Idx = findArgIdxOfSymbol(CurrSt, LCtx, Sym, CE);
+
+  // If index is not found, we assume that the symbol was returned.
+  if (!Idx) {
+    os << " returns ";
+  } else {
+    os << " writes ";
+  }
 
   if (CurrV.getObjKind() == ObjKind::CF) {
     os << "a Core Foundation object of type '"
@@ -185,6 +212,25 @@ static void generateDiagnosticsForCallLike(ProgramStateRef CurrSt,
     assert(CurrV.isNotOwned());
     os << "+0 retain count";
   }
+
+  if (Idx) {
+    os << " into an out parameter '";
+    const ParmVarDecl *PVD = (*CE)->parameters()[*Idx];
+    PVD->getNameForDiagnostic(os, PVD->getASTContext().getPrintingPolicy(),
+                              /*Qualified=*/false);
+    os << "'";
+
+    QualType RT = (*CE)->getResultType();
+    if (!RT.isNull() && !RT->isVoidType()) {
+      SVal RV = (*CE)->getReturnValue();
+      if (CurrSt->isNull(RV).isConstrainedTrue()) {
+        os << " (assuming the call returns zero)";
+      } else if (CurrSt->isNonNull(RV).isConstrainedTrue()) {
+        os << " (assuming the call returns non-zero)";
+      }
+
+    }
+  }
 }
 
 namespace clang {
diff --git a/clang/lib/StaticAnalyzer/Core/RetainSummaryManager.cpp b/clang/lib/StaticAnalyzer/Core/RetainSummaryManager.cpp
index bc43b8f897e..2e40cc33381 100644
--- a/clang/lib/StaticAnalyzer/Core/RetainSummaryManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RetainSummaryManager.cpp
@@ -88,7 +88,9 @@ Optional<ObjKind> RetainSummaryManager::hasAnyEnabledAttrOf(const Decl *D,
       return None;
     K = ObjKind::ObjC;
   } else if (isOneOf<T, OSConsumedAttr, OSConsumesThisAttr,
-                     OSReturnsNotRetainedAttr, OSReturnsRetainedAttr>()) {
+                     OSReturnsNotRetainedAttr, OSReturnsRetainedAttr,
+                     OSReturnsRetainedOnZeroAttr,
+                     OSReturnsRetainedOnNonZeroAttr>()) {
     if (!TrackOSObjects)
       return None;
     K = ObjKind::OS;
@@ -522,6 +524,8 @@ static ArgEffect getStopTrackingHardEquivalent(ArgEffect E) {
   case IncRef:
   case UnretainedOutParameter:
   case RetainedOutParameter:
+  case RetainedOutParameterOnZero:
+  case RetainedOutParameterOnNonZero:
   case MayEscape:
   case StopTracking:
   case StopTrackingHard:
@@ -811,6 +815,29 @@ RetainSummaryManager::getRetEffectFromAnnotations(QualType RetTy,
   return None;
 }
 
+/// \return Whether the chain of typedefs starting from {@code QT}
+/// has a typedef with a given name {@code Name}.
+static bool hasTypedefNamed(QualType QT,
+                            StringRef Name) {
+  while (auto *T = dyn_cast<TypedefType>(QT)) {
+    const auto &Context = T->getDecl()->getASTContext();
+    if (T->getDecl()->getIdentifier() == &Context.Idents.get(Name))
+      return true;
+    QT = T->getDecl()->getUnderlyingType();
+  }
+  return false;
+}
+
+static QualType getCallableReturnType(const NamedDecl *ND) {
+  if (const auto *FD = dyn_cast<FunctionDecl>(ND)) {
+    return FD->getReturnType();
+  } else if (const auto *MD = dyn_cast<ObjCMethodDecl>(ND)) {
+    return MD->getReturnType();
+  } else {
+    llvm_unreachable("Unexpected decl");
+  }
+}
+
 bool RetainSummaryManager::applyParamAnnotationEffect(
     const ParmVarDecl *pd, unsigned parm_idx, const NamedDecl *FD,
     RetainSummaryTemplate &Template) {
@@ -820,21 +847,54 @@ bool RetainSummaryManager::applyParamAnnotationEffect(
                               GeneralizedConsumedAttr>(pd, QT)) {
     Template->addArg(AF, parm_idx, ArgEffect(DecRef, *K));
     return true;
-  } else if (auto K =
-                 hasAnyEnabledAttrOf<CFReturnsRetainedAttr,
-                                     GeneralizedReturnsRetainedAttr>(pd, QT)) {
-    Template->addArg(AF, parm_idx, ArgEffect(RetainedOutParameter, *K));
+  } else if (auto K = hasAnyEnabledAttrOf<
+                 CFReturnsRetainedAttr, OSReturnsRetainedAttr,
+                 OSReturnsRetainedOnNonZeroAttr, OSReturnsRetainedOnZeroAttr,
+                 GeneralizedReturnsRetainedAttr>(pd, QT)) {
+
+    // For OSObjects, we try to guess whether the object is created based
+    // on the return value.
+    if (K == ObjKind::OS) {
+      QualType QT = getCallableReturnType(FD);
+
+      bool HasRetainedOnZero = pd->hasAttr<OSReturnsRetainedOnZeroAttr>();
+      bool HasRetainedOnNonZero = pd->hasAttr<OSReturnsRetainedOnNonZeroAttr>();
+
+      // The usual convention is to create an object on non-zero return, but
+      // it's reverted if the typedef chain has a typedef kern_return_t,
+      // because kReturnSuccess constant is defined as zero.
+      // The convention can be overwritten by custom attributes.
+      bool SuccessOnZero =
+          HasRetainedOnZero ||
+          (hasTypedefNamed(QT, "kern_return_t") && !HasRetainedOnNonZero);
+      bool ShouldSplit = !QT.isNull() && !QT->isVoidType();
+      ArgEffectKind AK = RetainedOutParameter;
+      if (ShouldSplit && SuccessOnZero) {
+        AK = RetainedOutParameterOnZero;
+      } else if (ShouldSplit && (!SuccessOnZero || HasRetainedOnNonZero)) {
+        AK = RetainedOutParameterOnNonZero;
+      }
+      Template->addArg(AF, parm_idx, ArgEffect(AK, ObjKind::OS));
+    }
+
+    // For others:
+    // Do nothing. Retained out parameters will either point to a +1 reference
+    // or NULL, but the way you check for failure differs depending on the
+    // API. Consequently, we don't have a good way to track them yet.
     return true;
-  } else if (auto K = hasAnyEnabledAttrOf<CFReturnsNotRetainedAttr>(pd, QT)) {
+  } else if (auto K = hasAnyEnabledAttrOf<CFReturnsNotRetainedAttr,
+                                          OSReturnsNotRetainedAttr,
+                                          GeneralizedReturnsNotRetainedAttr>(
+                 pd, QT)) {
     Template->addArg(AF, parm_idx, ArgEffect(UnretainedOutParameter, *K));
     return true;
-  } else {
-    if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
-      for (const auto *OD : MD->overridden_methods()) {
-        const ParmVarDecl *OP = OD->parameters()[parm_idx];
-        if (applyParamAnnotationEffect(OP, parm_idx, OD, Template))
-          return true;
-      }
+  }
+
+  if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
+    for (const auto *OD : MD->overridden_methods()) {
+      const ParmVarDecl *OP = OD->parameters()[parm_idx];
+      if (applyParamAnnotationEffect(OP, parm_idx, OD, Template))
+        return true;
     }
   }