Testbed and skeleton of a new expression parser

LLVM's JIT is now the foundation of dynamic-compilation features for many languages. Clang also has low-level support for dynamic compilation (ASTImporter and ExternalASTSource, notably). How the compiler is set up for dynamic parsing is generally left up to individual clients, for example LLDB's C/C++/Objective-C expression parser and the ROOT project.

Although this arrangement offers external clients the flexibility to implement dynamic features as they see fit, the lack of an in-tree client means that subtle bugs can be introduced that cause regressions in the external clients but aren't caught by tests (or users) until much later. LLDB for example regularly encounters complicated ODR violation scenarios where it is not immediately clear who is at fault.

Other external clients (notably, Cling) rely on similar functionality, and another goal is to break this functionality up into composable parts so that any client can be built easily on top of Clang without requiring extensive additional code.

I propose that the parts required to build a simple expression parser be added to Clang.  Initially, I aim to have the following features:

- A piece that looks up external declarations from a variety of sources (e.g., from previous dynamic compilations, from modules, or from DWARF) and uses clear conflict resolution rules to reconcile differences, with easily understood errors. This functionality will be supported by in-tree tests.

- A piece that works hand in hand with the LLVM JIT to resolve the locations of external declarations so that e.g. variables can be redeclared and (for high-performance applications like DTrace) external variables can be accessed directly from the registers where they reside.

This commit adds a tester that parses a sequence of source files and then uses them as source data for an expression. External references are resolved using an ExternalASTSource that responds to name queries using an ASTImporter. This is the setup that LLDB uses, and the motivating reason for MinimalImport in ASTImporter.  When complete, this tester will implement the first of the above goals.

Differential Revision: https://reviews.llvm.org/D27180

llvm-svn: 290004

3 files changed, 344 insertions, 0 deletions

diff --git a/clang/tools/CMakeLists.txt b/clang/tools/CMakeLists.txt
index b8850e45468..b0c97f0f1e4 100644
--- a/clang/tools/CMakeLists.txt
+++ b/clang/tools/CMakeLists.txt
@@ -5,6 +5,7 @@ add_clang_subdirectory(driver)
 add_clang_subdirectory(clang-format)
 add_clang_subdirectory(clang-format-vs)
 add_clang_subdirectory(clang-fuzzer)
+add_clang_subdirectory(clang-import-test)
 add_clang_subdirectory(clang-offload-bundler)
 
 add_clang_subdirectory(c-index-test)
diff --git a/clang/tools/clang-import-test/CMakeLists.txt b/clang/tools/clang-import-test/CMakeLists.txt
new file mode 100644
index 00000000000..4d41a5674ae
--- /dev/null
+++ b/clang/tools/clang-import-test/CMakeLists.txt
@@ -0,0 +1,24 @@
+set(LLVM_LINK_COMPONENTS
+  support
+)
+
+if(NOT CLANG_BUILT_STANDALONE)
+  set(tablegen_deps intrinsics_gen)
+endif()
+
+add_clang_tool(clang-import-test
+  clang-import-test.cpp
+  DEPENDS
+  ${tablegen_deps}
+  )
+
+set(CLANG_IMPORT_TEST_LIB_DEPS
+  clangAST
+  clangBasic
+  clangCodeGen
+  clangFrontend
+  )
+
+target_link_libraries(clang-import-test
+  ${CLANG_IMPORT_TEST_LIB_DEPS}
+  )
diff --git a/clang/tools/clang-import-test/clang-import-test.cpp b/clang/tools/clang-import-test/clang-import-test.cpp
new file mode 100644
index 00000000000..931b46245bb
--- /dev/null
+++ b/clang/tools/clang-import-test/clang-import-test.cpp
@@ -0,0 +1,319 @@
+//===-- import-test.cpp - ASTImporter/ExternalASTSource testbed -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/ASTImporter.h"
+#include "clang/Basic/Builtins.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TargetOptions.h"
+#include "clang/CodeGen/ModuleBuilder.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/TextDiagnosticBuffer.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Parse/ParseAST.h"
+
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/Host.h"
+#include "llvm/Support/Signals.h"
+
+#include <memory>
+#include <string>
+
+using namespace clang;
+
+static llvm::cl::opt<std::string> Expression(
+    "expression", llvm::cl::Required,
+    llvm::cl::desc("Path to a file containing the expression to parse"));
+
+static llvm::cl::list<std::string>
+    Imports("import", llvm::cl::ZeroOrMore,
+            llvm::cl::desc("Path to a file containing declarations to import"));
+
+static llvm::cl::list<std::string>
+    ClangArgs("Xcc", llvm::cl::ZeroOrMore,
+              llvm::cl::desc("Argument to pass to the CompilerInvocation"),
+              llvm::cl::CommaSeparated);
+
+namespace init_convenience {
+class TestDiagnosticConsumer : public DiagnosticConsumer {
+private:
+  std::unique_ptr<TextDiagnosticBuffer> Passthrough;
+  const LangOptions *LangOpts = nullptr;
+
+public:
+  TestDiagnosticConsumer()
+      : Passthrough(llvm::make_unique<TextDiagnosticBuffer>()) {}
+
+  virtual void BeginSourceFile(const LangOptions &LangOpts,
+                               const Preprocessor *PP = nullptr) override {
+    this->LangOpts = &LangOpts;
+    return Passthrough->BeginSourceFile(LangOpts, PP);
+  }
+
+  virtual void EndSourceFile() override {
+    this->LangOpts = nullptr;
+    Passthrough->EndSourceFile();
+  }
+
+  virtual bool IncludeInDiagnosticCounts() const override {
+    return Passthrough->IncludeInDiagnosticCounts();
+  }
+
+private:
+  static void PrintSourceForLocation(const SourceLocation &Loc,
+                                     SourceManager &SM) {
+    const char *LocData = SM.getCharacterData(Loc, /*Invalid=*/nullptr);
+    unsigned LocColumn =
+        SM.getSpellingColumnNumber(Loc, /*Invalid=*/nullptr) - 1;
+    FileID FID = SM.getFileID(Loc);
+    llvm::MemoryBuffer *Buffer = SM.getBuffer(FID, Loc, /*Invalid=*/nullptr);
+
+    assert(LocData >= Buffer->getBufferStart() &&
+           LocData < Buffer->getBufferEnd());
+
+    const char *LineBegin = LocData - LocColumn;
+
+    assert(LineBegin >= Buffer->getBufferStart());
+
+    const char *LineEnd = nullptr;
+
+    for (LineEnd = LineBegin; *LineEnd != '\n' && *LineEnd != '\r' &&
+                              LineEnd < Buffer->getBufferEnd();
+         ++LineEnd)
+      ;
+
+    llvm::StringRef LineString(LineBegin, LineEnd - LineBegin);
+
+    llvm::errs() << LineString << '\n';
+    std::string Space(LocColumn, ' ');
+    llvm::errs() << Space.c_str() << '\n';
+  }
+
+  virtual void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                                const Diagnostic &Info) override {
+    if (Info.hasSourceManager() && LangOpts) {
+      SourceManager &SM = Info.getSourceManager();
+
+      if (Info.getLocation().isValid()) {
+        Info.getLocation().print(llvm::errs(), SM);
+        llvm::errs() << ": ";
+      }
+
+      SmallString<16> DiagText;
+      Info.FormatDiagnostic(DiagText);
+      llvm::errs() << DiagText << '\n';
+
+      if (Info.getLocation().isValid()) {
+        PrintSourceForLocation(Info.getLocation(), SM);
+      }
+
+      for (const CharSourceRange &Range : Info.getRanges()) {
+        bool Invalid = true;
+        StringRef Ref = Lexer::getSourceText(Range, SM, *LangOpts, &Invalid);
+        if (!Invalid) {
+          llvm::errs() << Ref.str() << '\n';
+        }
+      }
+    }
+    DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info);
+  }
+};
+
+std::unique_ptr<CompilerInstance>
+BuildCompilerInstance(ArrayRef<const char *> ClangArgv) {
+  auto Ins = llvm::make_unique<CompilerInstance>();
+  auto DC = llvm::make_unique<TestDiagnosticConsumer>();
+  const bool ShouldOwnClient = true;
+  Ins->createDiagnostics(DC.release(), ShouldOwnClient);
+
+  auto Inv = llvm::make_unique<CompilerInvocation>();
+
+  CompilerInvocation::CreateFromArgs(*Inv, ClangArgv.data(),
+                                     &ClangArgv.data()[ClangArgv.size()],
+                                     Ins->getDiagnostics());
+
+  Inv->getLangOpts()->CPlusPlus = true;
+  Inv->getLangOpts()->CPlusPlus11 = true;
+  Inv->getHeaderSearchOpts().UseLibcxx = true;
+  Inv->getLangOpts()->Bool = true;
+  Inv->getLangOpts()->WChar = true;
+  Inv->getLangOpts()->Blocks = true;
+  Inv->getLangOpts()->DebuggerSupport = true;
+  Inv->getLangOpts()->SpellChecking = false;
+  Inv->getLangOpts()->ThreadsafeStatics = false;
+  Inv->getLangOpts()->AccessControl = false;
+  Inv->getLangOpts()->DollarIdents = true;
+  Inv->getCodeGenOpts().setDebugInfo(codegenoptions::FullDebugInfo);
+  Inv->getTargetOpts().Triple = llvm::sys::getDefaultTargetTriple();
+
+  Ins->setInvocation(Inv.release());
+
+  TargetInfo *TI = TargetInfo::CreateTargetInfo(
+      Ins->getDiagnostics(), Ins->getInvocation().TargetOpts);
+  Ins->setTarget(TI);
+  Ins->getTarget().adjust(Ins->getLangOpts());
+  Ins->createFileManager();
+  Ins->createSourceManager(Ins->getFileManager());
+  Ins->createPreprocessor(TU_Complete);
+
+  return Ins;
+}
+
+std::unique_ptr<ASTContext>
+BuildASTContext(CompilerInstance &CI, SelectorTable &ST, Builtin::Context &BC) {
+  auto AST = llvm::make_unique<ASTContext>(
+      CI.getLangOpts(), CI.getSourceManager(),
+      CI.getPreprocessor().getIdentifierTable(), ST, BC);
+  AST->InitBuiltinTypes(CI.getTarget());
+  return AST;
+}
+
+std::unique_ptr<CodeGenerator> BuildCodeGen(CompilerInstance &CI,
+                                            llvm::LLVMContext &LLVMCtx) {
+  StringRef ModuleName("$__module");
+  return std::unique_ptr<CodeGenerator>(CreateLLVMCodeGen(
+      CI.getDiagnostics(), ModuleName, CI.getHeaderSearchOpts(),
+      CI.getPreprocessorOpts(), CI.getCodeGenOpts(), LLVMCtx));
+}
+} // end namespace
+
+namespace {
+class TestExternalASTSource : public ExternalASTSource {
+private:
+  llvm::ArrayRef<std::unique_ptr<CompilerInstance>> ImportCIs;
+  std::map<CompilerInstance *, std::unique_ptr<ASTImporter>> ForwardImporters;
+  std::map<CompilerInstance *, std::unique_ptr<ASTImporter>> ReverseImporters;
+
+public:
+  TestExternalASTSource(
+      CompilerInstance &ExpressionCI,
+      llvm::ArrayRef<std::unique_ptr<CompilerInstance>> ImportCIs)
+      : ImportCIs(ImportCIs) {
+    for (const std::unique_ptr<CompilerInstance> &ImportCI : ImportCIs) {
+      ForwardImporters[ImportCI.get()] = llvm::make_unique<ASTImporter>(
+          ExpressionCI.getASTContext(), ExpressionCI.getFileManager(),
+          ImportCI->getASTContext(), ImportCI->getFileManager(),
+          /*MinimalImport=*/true);
+      ReverseImporters[ImportCI.get()] = llvm::make_unique<ASTImporter>(
+          ImportCI->getASTContext(), ImportCI->getFileManager(),
+          ExpressionCI.getASTContext(), ExpressionCI.getFileManager(),
+          /*MinimalImport=*/true);
+    }
+  }
+
+  bool FindExternalVisibleDeclsByName(const DeclContext *DC,
+                                      DeclarationName Name) override {
+    llvm::SmallVector<NamedDecl *, 1> Decls;
+
+    if (isa<TranslationUnitDecl>(DC)) {
+      for (const std::unique_ptr<CompilerInstance> &I : ImportCIs) {
+        DeclarationName FromName = ReverseImporters[I.get()]->Import(Name);
+        DeclContextLookupResult Result =
+            I->getASTContext().getTranslationUnitDecl()->lookup(FromName);
+        for (NamedDecl *FromD : Result) {
+          NamedDecl *D =
+              llvm::cast<NamedDecl>(ForwardImporters[I.get()]->Import(FromD));
+          Decls.push_back(D);
+        }
+      }
+    }
+    if (Decls.empty()) {
+      return false;
+    } else {
+      SetExternalVisibleDeclsForName(DC, Name, Decls);
+      return true;
+    }
+  }
+};
+
+void AddExternalSource(
+    CompilerInstance &CI,
+    llvm::ArrayRef<std::unique_ptr<CompilerInstance>> Imports) {
+  ASTContext &AST = CI.getASTContext();
+  auto ES = llvm::make_unique<TestExternalASTSource>(CI, Imports);
+  AST.setExternalSource(ES.release());
+  AST.getTranslationUnitDecl()->setHasExternalVisibleStorage();
+}
+
+llvm::Error ParseSource(const std::string &Path, CompilerInstance &CI,
+                        CodeGenerator &CG) {
+  SourceManager &SM = CI.getSourceManager();
+  const FileEntry *FE = CI.getFileManager().getFile(Path);
+  if (!FE) {
+    return llvm::make_error<llvm::StringError>(
+        llvm::Twine("Couldn't open ", Path), std::error_code());
+  }
+  SM.setMainFileID(SM.createFileID(FE, SourceLocation(), SrcMgr::C_User));
+  ParseAST(CI.getPreprocessor(), &CG, CI.getASTContext());
+  return llvm::Error::success();
+}
+
+llvm::Expected<std::unique_ptr<CompilerInstance>>
+Parse(const std::string &Path,
+      llvm::ArrayRef<std::unique_ptr<CompilerInstance>> Imports) {
+  std::vector<const char *> ClangArgv(ClangArgs.size());
+  std::transform(ClangArgs.begin(), ClangArgs.end(), ClangArgv.begin(),
+                 [](const std::string &s) -> const char * { return s.data(); });
+  std::unique_ptr<CompilerInstance> CI =
+      init_convenience::BuildCompilerInstance(ClangArgv);
+  auto ST = llvm::make_unique<SelectorTable>();
+  auto BC = llvm::make_unique<Builtin::Context>();
+  std::unique_ptr<ASTContext> AST =
+      init_convenience::BuildASTContext(*CI, *ST, *BC);
+  CI->setASTContext(AST.release());
+  AddExternalSource(*CI, Imports);
+
+  auto LLVMCtx = llvm::make_unique<llvm::LLVMContext>();
+  std::unique_ptr<CodeGenerator> CG =
+      init_convenience::BuildCodeGen(*CI, *LLVMCtx);
+  CG->Initialize(CI->getASTContext());
+
+  CI->getDiagnosticClient().BeginSourceFile(CI->getLangOpts(),
+                                            &CI->getPreprocessor());
+  if (llvm::Error PE = ParseSource(Path, *CI, *CG)) {
+    return std::move(PE);
+  }
+  CI->getDiagnosticClient().EndSourceFile();
+  if (CI->getDiagnosticClient().getNumErrors()) {
+    return llvm::make_error<llvm::StringError>(
+        "Errors occured while parsing the expression.", std::error_code());
+  } else {
+    return std::move(CI);
+  }
+}
+} // end namespace
+
+int main(int argc, const char **argv) {
+  const bool DisableCrashReporting = true;
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0], DisableCrashReporting);
+  llvm::cl::ParseCommandLineOptions(argc, argv);
+  std::vector<std::unique_ptr<CompilerInstance>> ImportCIs;
+  for (auto I : Imports) {
+      llvm::Expected<std::unique_ptr<CompilerInstance>> ImportCI = Parse(I, {});
+      if (auto E = ImportCI.takeError()) {
+        llvm::errs() << llvm::toString(std::move(E));
+        exit(-1);
+      } else {
+        ImportCIs.push_back(std::move(*ImportCI));
+      }
+    }
+  llvm::Expected<std::unique_ptr<CompilerInstance>> ExpressionCI =
+      Parse(Expression, ImportCIs);
+  if (auto E = ExpressionCI.takeError()) {
+    llvm::errs() << llvm::toString(std::move(E));
+    exit(-1);
+  } else {
+    return 0;
+  }
+}