[MS ABI] Don't crash while mangling recursive lambdas

We might get into bad situations where we try to embed the signature of
an inner lambda into an outer lambda which cannot work: the inner lambda
wants to embed the name of the outer lambda!

Instead, omit the return type for lambdas.

This fixes PR26105.

N.B.  While we are here, make lambdas nested within functions use an
artificial scope so that they can get demangled.

llvm-svn: 258003

1 files changed, 12 insertions, 4 deletions

diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index 4a45f9e4051..0634319cbd1 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -160,14 +160,17 @@ public:
                              raw_ostream &Out) override;
   void mangleStringLiteral(const StringLiteral *SL, raw_ostream &Out) override;
   bool getNextDiscriminator(const NamedDecl *ND, unsigned &disc) {
-    // Lambda closure types are already numbered.
-    if (isLambda(ND))
-      return false;
-
     const DeclContext *DC = getEffectiveDeclContext(ND);
     if (!DC->isFunctionOrMethod())
       return false;
 
+    // Lambda closure types are already numbered, give out a phony number so
+    // that they demangle nicely.
+    if (isLambda(ND)) {
+      disc = 1;
+      return true;
+    }
+
     // Use the canonical number for externally visible decls.
     if (ND->isExternallyVisible()) {
       disc = getASTContext().getManglingNumber(ND);
@@ -1799,9 +1802,12 @@ void MicrosoftCXXNameMangler::mangleFunctionType(const FunctionType *T,
   SourceRange Range;
   if (D) Range = D->getSourceRange();
 
+  bool IsInLambda = false;
   bool IsStructor = false, HasThisQuals = ForceThisQuals, IsCtorClosure = false;
   CallingConv CC = T->getCallConv();
   if (const CXXMethodDecl *MD = dyn_cast_or_null<CXXMethodDecl>(D)) {
+    if (MD->getParent()->isLambda())
+      IsInLambda = true;
     if (MD->isInstance())
       HasThisQuals = true;
     if (isa<CXXDestructorDecl>(MD)) {
@@ -1875,6 +1881,8 @@ void MicrosoftCXXNameMangler::mangleFunctionType(const FunctionType *T,
              "shouldn't need to mangle __auto_type!");
       mangleSourceName(AT->isDecltypeAuto() ? "<decltype-auto>" : "<auto>");
       Out << '@';
+    } else if (IsInLambda) {
+      Out << '@';
     } else {
       if (ResultType->isVoidType())
         ResultType = ResultType.getUnqualifiedType();