clang-format: [JS] generator and async functions.

For generators, see:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Iterators_and_generators
async functions are not quite in the spec yet, but stage 3 and already widely used:
http://tc39.github.io/ecmascript-asyncawait/

Reviewers: djasper

Subscribers: klimek

Differential Revision: http://reviews.llvm.org/D19204

llvm-svn: 267368

4 files changed, 35 insertions, 11 deletions

diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 9cff34bc3df..bfcb8e562e3 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -479,7 +479,7 @@ unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State,
   // is common and should be formatted like a free-standing function.
   if (Style.Language != FormatStyle::LK_JavaScript ||
       Current.NestingLevel != 0 || !PreviousNonComment->is(tok::equal) ||
-      !Current.is(Keywords.kw_function))
+      !Current.isOneOf(Keywords.kw_async, Keywords.kw_function))
     State.Stack.back().NestedBlockIndent = State.Column;
 
   if (NextNonComment->isMemberAccess()) {
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index f15c9e998e6..d80bd0af5f0 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -535,13 +535,16 @@ struct AdditionalKeywords {
     kw_NS_ENUM = &IdentTable.get("NS_ENUM");
     kw_NS_OPTIONS = &IdentTable.get("NS_OPTIONS");
 
+    kw_async = &IdentTable.get("async");
+    kw_await = &IdentTable.get("await");
     kw_finally = &IdentTable.get("finally");
-    kw_function = &IdentTable.get("function");
     kw_from = &IdentTable.get("from");
+    kw_function = &IdentTable.get("function");
     kw_import = &IdentTable.get("import");
     kw_is = &IdentTable.get("is");
     kw_let = &IdentTable.get("let");
     kw_var = &IdentTable.get("var");
+    kw_yield = &IdentTable.get("yield");
 
     kw_abstract = &IdentTable.get("abstract");
     kw_assert = &IdentTable.get("assert");
@@ -582,13 +585,16 @@ struct AdditionalKeywords {
   IdentifierInfo *kw___except;
 
   // JavaScript keywords.
+  IdentifierInfo *kw_async;
+  IdentifierInfo *kw_await;
   IdentifierInfo *kw_finally;
-  IdentifierInfo *kw_function;
   IdentifierInfo *kw_from;
+  IdentifierInfo *kw_function;
   IdentifierInfo *kw_import;
   IdentifierInfo *kw_is;
   IdentifierInfo *kw_let;
   IdentifierInfo *kw_var;
+  IdentifierInfo *kw_yield;
 
   // Java keywords.
   IdentifierInfo *kw_abstract;
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 9c745399434..ceecb3d36cd 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -2078,6 +2078,9 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
   } else if (Style.Language == FormatStyle::LK_JavaScript) {
     if (Left.is(TT_JsFatArrow))
       return true;
+    if (Right.is(tok::star) &&
+        Left.isOneOf(Keywords.kw_function, Keywords.kw_yield))
+      return false;
     if (Left.isOneOf(Keywords.kw_let, Keywords.kw_var, Keywords.kw_in,
                      Keywords.kw_of) &&
         (!Left.Previous || !Left.Previous->is(tok::period)))
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 55eecf64e22..c6bf71adbf1 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -668,7 +668,8 @@ static bool mustBeJSIdent(const AdditionalKeywords &Keywords,
   // FIXME: This returns true for C/C++ keywords like 'struct'.
   return FormatTok->is(tok::identifier) &&
          (FormatTok->Tok.getIdentifierInfo() == nullptr ||
-          !FormatTok->isOneOf(Keywords.kw_in, Keywords.kw_of,
+          !FormatTok->isOneOf(Keywords.kw_in, Keywords.kw_of, Keywords.kw_async,
+                              Keywords.kw_await, Keywords.kw_yield,
                               Keywords.kw_finally, Keywords.kw_function,
                               Keywords.kw_import, Keywords.kw_is,
                               Keywords.kw_let, Keywords.kw_var,
@@ -687,7 +688,7 @@ static bool mustBeJSIdentOrValue(const AdditionalKeywords &Keywords,
 static bool isJSDeclOrStmt(const AdditionalKeywords &Keywords,
                            const FormatToken *FormatTok) {
   return FormatTok->isOneOf(
-      tok::kw_return,
+      tok::kw_return, Keywords.kw_yield,
       // conditionals
       tok::kw_if, tok::kw_else,
       // loops
@@ -698,7 +699,9 @@ static bool isJSDeclOrStmt(const AdditionalKeywords &Keywords,
       tok::kw_throw, tok::kw_try, tok::kw_catch, Keywords.kw_finally,
       // declaration
       tok::kw_const, tok::kw_class, Keywords.kw_var, Keywords.kw_let,
-      Keywords.kw_function);
+      Keywords.kw_async, Keywords.kw_function,
+      // import/export
+      Keywords.kw_import, tok::kw_export);
 }
 
 // readTokenWithJavaScriptASI reads the next token and terminates the current
@@ -1003,7 +1006,8 @@ void UnwrappedLineParser::parseStructuralElement() {
       // Parse function literal unless 'function' is the first token in a line
       // in which case this should be treated as a free-standing function.
       if (Style.Language == FormatStyle::LK_JavaScript &&
-          FormatTok->is(Keywords.kw_function) && Line->Tokens.size() > 0) {
+          FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function) &&
+          Line->Tokens.size() > 0) {
         tryToParseJSFunction();
         break;
       }
@@ -1189,8 +1193,16 @@ bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
 }
 
 void UnwrappedLineParser::tryToParseJSFunction() {
+  assert(FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function));
+  if (FormatTok->is(Keywords.kw_async))
+    nextToken();
+  // Consume "function".
   nextToken();
 
+  // Consume * (generator function).
+  if (FormatTok->is(tok::star))
+    nextToken();
+
   // Consume function name.
   if (FormatTok->is(tok::identifier))
     nextToken();
@@ -1235,7 +1247,7 @@ bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
   // replace this by using parseAssigmentExpression() inside.
   do {
     if (Style.Language == FormatStyle::LK_JavaScript) {
-      if (FormatTok->is(Keywords.kw_function)) {
+      if (FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function)) {
         tryToParseJSFunction();
         continue;
       }
@@ -1333,7 +1345,7 @@ void UnwrappedLineParser::parseParens() {
       break;
     case tok::identifier:
       if (Style.Language == FormatStyle::LK_JavaScript &&
-          FormatTok->is(Keywords.kw_function))
+          FormatTok->isOneOf(Keywords.kw_async, Keywords.kw_function))
         tryToParseJSFunction();
       else
         nextToken();
@@ -1904,8 +1916,11 @@ void UnwrappedLineParser::parseJavaScriptEs6ImportExport() {
   if (FormatTok->is(tok::kw_default))
     nextToken();
 
-  // Consume "function" and "default function", so that these get parsed as
-  // free-standing JS functions, i.e. do not require a trailing semicolon.
+  // Consume "async function", "function" and "default function", so that these
+  // get parsed as free-standing JS functions, i.e. do not require a trailing
+  // semicolon.
+  if (FormatTok->is(Keywords.kw_async))
+    nextToken();
   if (FormatTok->is(Keywords.kw_function)) {
     nextToken();
     return;