diff options
Diffstat (limited to 'clang/lib')
-rw-r--r-- | clang/lib/Frontend/ASTUnit.cpp | 7 | ||||
-rw-r--r-- | clang/lib/Frontend/CMakeLists.txt | 1 | ||||
-rw-r--r-- | clang/lib/Frontend/ChainedIncludesSource.cpp | 5 | ||||
-rw-r--r-- | clang/lib/Frontend/CompilerInstance.cpp | 14 | ||||
-rw-r--r-- | clang/lib/Frontend/CompilerInvocation.cpp | 52 | ||||
-rw-r--r-- | clang/lib/Frontend/FrontendActions.cpp | 30 | ||||
-rw-r--r-- | clang/lib/Frontend/TestModuleFileExtension.cpp | 121 | ||||
-rw-r--r-- | clang/lib/Frontend/TestModuleFileExtension.h | 71 | ||||
-rw-r--r-- | clang/lib/Serialization/ASTReader.cpp | 230 | ||||
-rw-r--r-- | clang/lib/Serialization/ASTWriter.cpp | 58 | ||||
-rw-r--r-- | clang/lib/Serialization/CMakeLists.txt | 1 | ||||
-rw-r--r-- | clang/lib/Serialization/GeneratePCH.cpp | 12 | ||||
-rw-r--r-- | clang/lib/Serialization/ModuleFileExtension.cpp | 22 |
13 files changed, 560 insertions, 64 deletions
diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp index d3d7185ea3a..6b89447305f 100644 --- a/clang/lib/Frontend/ASTUnit.cpp +++ b/clang/lib/Frontend/ASTUnit.cpp @@ -186,7 +186,7 @@ struct ASTUnit::ASTWriterData { llvm::BitstreamWriter Stream; ASTWriter Writer; - ASTWriterData() : Stream(Buffer), Writer(Stream) { } + ASTWriterData() : Stream(Buffer), Writer(Stream, { }) { } }; void ASTUnit::clearFileLevelDecls() { @@ -709,7 +709,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile( bool disableValid = false; if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION")) disableValid = true; - AST->Reader = new ASTReader(PP, Context, PCHContainerRdr, + AST->Reader = new ASTReader(PP, Context, PCHContainerRdr, { }, /*isysroot=*/"", /*DisableValidation=*/disableValid, AllowPCHWithCompilerErrors); @@ -927,6 +927,7 @@ public: const Preprocessor &PP, StringRef isysroot, raw_ostream *Out) : PCHGenerator(PP, "", nullptr, isysroot, std::make_shared<PCHBuffer>(), + ArrayRef<llvm::IntrusiveRefCntPtr<ModuleFileExtension>>(), /*AllowASTWithErrors=*/true), Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action), Out(Out) { @@ -2500,7 +2501,7 @@ bool ASTUnit::serialize(raw_ostream &OS) { SmallString<128> Buffer; llvm::BitstreamWriter Stream(Buffer); - ASTWriter Writer(Stream); + ASTWriter Writer(Stream, { }); return serializeUnit(Writer, Buffer, getSema(), hasErrors, OS); } diff --git a/clang/lib/Frontend/CMakeLists.txt b/clang/lib/Frontend/CMakeLists.txt index 9a3e459640a..af42a905b38 100644 --- a/clang/lib/Frontend/CMakeLists.txt +++ b/clang/lib/Frontend/CMakeLists.txt @@ -35,6 +35,7 @@ add_clang_library(clangFrontend PrintPreprocessedOutput.cpp SerializedDiagnosticPrinter.cpp SerializedDiagnosticReader.cpp + TestModuleFileExtension.cpp TextDiagnostic.cpp TextDiagnosticBuffer.cpp TextDiagnosticPrinter.cpp diff --git a/clang/lib/Frontend/ChainedIncludesSource.cpp b/clang/lib/Frontend/ChainedIncludesSource.cpp index d36162b8dab..1c1081fbe08 100644 --- a/clang/lib/Frontend/ChainedIncludesSource.cpp +++ b/clang/lib/Frontend/ChainedIncludesSource.cpp @@ -82,6 +82,7 @@ createASTReader(CompilerInstance &CI, StringRef pchFile, std::unique_ptr<ASTReader> Reader; Reader.reset(new ASTReader(PP, CI.getASTContext(), CI.getPCHContainerReader(), + /*Extensions=*/{ }, /*isysroot=*/"", /*DisableValidation=*/true)); for (unsigned ti = 0; ti < bufNames.size(); ++ti) { StringRef sr(bufNames[ti]); @@ -160,8 +161,10 @@ IntrusiveRefCntPtr<ExternalSemaSource> clang::createChainedIncludesSource( Clang->createASTContext(); auto Buffer = std::make_shared<PCHBuffer>(); + ArrayRef<llvm::IntrusiveRefCntPtr<ModuleFileExtension>> Extensions; auto consumer = llvm::make_unique<PCHGenerator>( - Clang->getPreprocessor(), "-", nullptr, /*isysroot=*/"", Buffer); + Clang->getPreprocessor(), "-", nullptr, /*isysroot=*/"", Buffer, + Extensions); Clang->getASTContext().setASTMutationListener( consumer->GetASTMutationListener()); Clang->setASTConsumer(std::move(consumer)); diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 8220a6e4925..c3f19a3a00d 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -408,7 +408,9 @@ void CompilerInstance::createPCHExternalASTSource( ModuleManager = createPCHExternalASTSource( Path, getHeaderSearchOpts().Sysroot, DisablePCHValidation, AllowPCHWithCompilerErrors, getPreprocessor(), getASTContext(), - getPCHContainerReader(), DeserializationListener, + getPCHContainerReader(), + getFrontendOpts().ModuleFileExtensions, + DeserializationListener, OwnDeserializationListener, Preamble, getFrontendOpts().UseGlobalModuleIndex); } @@ -417,15 +419,16 @@ IntrusiveRefCntPtr<ASTReader> CompilerInstance::createPCHExternalASTSource( StringRef Path, StringRef Sysroot, bool DisablePCHValidation, bool AllowPCHWithCompilerErrors, Preprocessor &PP, ASTContext &Context, const PCHContainerReader &PCHContainerRdr, + ArrayRef<IntrusiveRefCntPtr<ModuleFileExtension>> Extensions, void *DeserializationListener, bool OwnDeserializationListener, bool Preamble, bool UseGlobalModuleIndex) { HeaderSearchOptions &HSOpts = PP.getHeaderSearchInfo().getHeaderSearchOpts(); IntrusiveRefCntPtr<ASTReader> Reader(new ASTReader( - PP, Context, PCHContainerRdr, Sysroot.empty() ? "" : Sysroot.data(), - DisablePCHValidation, AllowPCHWithCompilerErrors, - /*AllowConfigurationMismatch*/ false, HSOpts.ModulesValidateSystemHeaders, - UseGlobalModuleIndex)); + PP, Context, PCHContainerRdr, Extensions, + Sysroot.empty() ? "" : Sysroot.data(), DisablePCHValidation, + AllowPCHWithCompilerErrors, /*AllowConfigurationMismatch*/ false, + HSOpts.ModulesValidateSystemHeaders, UseGlobalModuleIndex)); // We need the external source to be set up before we read the AST, because // eagerly-deserialized declarations may use it. @@ -1267,6 +1270,7 @@ void CompilerInstance::createModuleManager() { *FrontendTimerGroup); ModuleManager = new ASTReader( getPreprocessor(), getASTContext(), getPCHContainerReader(), + getFrontendOpts().ModuleFileExtensions, Sysroot.empty() ? "" : Sysroot.c_str(), PPOpts.DisablePCHValidation, /*AllowASTWithCompilerErrors=*/false, /*AllowConfigurationMismatch=*/false, diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 0c1c4eaefa9..468cc5746ce 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -7,6 +7,7 @@ // //===----------------------------------------------------------------------===// +#include "TestModuleFileExtension.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/Version.h" @@ -19,6 +20,7 @@ #include "clang/Frontend/Utils.h" #include "clang/Lex/HeaderSearchOptions.h" #include "clang/Serialization/ASTReader.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" @@ -832,6 +834,30 @@ static void ParseFileSystemArgs(FileSystemOptions &Opts, ArgList &Args) { Opts.WorkingDir = Args.getLastArgValue(OPT_working_directory); } +/// Parse the argument to the -ftest-module-file-extension +/// command-line argument. +/// +/// \returns true on error, false on success. +static bool parseTestModuleFileExtensionArg(StringRef Arg, + std::string &BlockName, + unsigned &MajorVersion, + unsigned &MinorVersion, + bool &Hashed, + std::string &UserInfo) { + SmallVector<StringRef, 5> Args; + Arg.split(Args, ':', 5); + if (Args.size() < 5) + return true; + + BlockName = Args[0]; + if (Args[1].getAsInteger(10, MajorVersion)) return true; + if (Args[2].getAsInteger(10, MinorVersion)) return true; + if (Args[3].getAsInteger(2, Hashed)) return true; + if (Args.size() > 4) + UserInfo = Args[4]; + return false; +} + static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, DiagnosticsEngine &Diags) { using namespace options; @@ -924,6 +950,26 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, if (A->getValue(0) == Opts.AddPluginActions[i]) Opts.AddPluginArgs[i].emplace_back(A->getValue(1)); + for (const std::string &Arg : + Args.getAllArgValues(OPT_ftest_module_file_extension_EQ)) { + std::string BlockName; + unsigned MajorVersion; + unsigned MinorVersion; + bool Hashed; + std::string UserInfo; + if (parseTestModuleFileExtensionArg(Arg, BlockName, MajorVersion, + MinorVersion, Hashed, UserInfo)) { + Diags.Report(diag::err_test_module_file_extension_format) << Arg; + + continue; + } + + // Add the testing module file extension. + Opts.ModuleFileExtensions.push_back( + new TestModuleFileExtension(BlockName, MajorVersion, MinorVersion, + Hashed, UserInfo)); + } + if (const Arg *A = Args.getLastArg(OPT_code_completion_at)) { Opts.CodeCompletionAt = ParsedSourceLocation::FromString(A->getValue()); @@ -2076,6 +2122,12 @@ std::string CompilerInvocation::getModuleHash() const { // Extend the signature with the user build path. code = hash_combine(code, hsOpts.ModuleUserBuildPath); + // Extend the signature with the module file extensions. + const FrontendOptions &frontendOpts = getFrontendOpts(); + for (auto ext : frontendOpts.ModuleFileExtensions) { + code = ext->hashExtension(code); + } + // Darwin-specific hack: if we have a sysroot, use the contents and // modification time of // $sysroot/System/Library/CoreServices/SystemVersion.plist diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 6ba4b134c38..865fb474020 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -91,7 +91,8 @@ GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { auto Buffer = std::make_shared<PCHBuffer>(); std::vector<std::unique_ptr<ASTConsumer>> Consumers; Consumers.push_back(llvm::make_unique<PCHGenerator>( - CI.getPreprocessor(), OutputFile, nullptr, Sysroot, Buffer)); + CI.getPreprocessor(), OutputFile, nullptr, Sysroot, + Buffer, CI.getFrontendOpts().ModuleFileExtensions)); Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator( CI, InFile, OutputFile, OS, Buffer)); @@ -133,10 +134,13 @@ GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI, auto Buffer = std::make_shared<PCHBuffer>(); std::vector<std::unique_ptr<ASTConsumer>> Consumers; + Consumers.push_back(llvm::make_unique<PCHGenerator>( - CI.getPreprocessor(), OutputFile, Module, Sysroot, Buffer, - /*AllowASTWithErrors*/false, - /*IncludeTimestamps*/+CI.getFrontendOpts().BuildingImplicitModule)); + CI.getPreprocessor(), OutputFile, Module, Sysroot, + Buffer, CI.getFrontendOpts().ModuleFileExtensions, + /*AllowASTWithErrors=*/false, + /*IncludeTimestamps=*/ + +CI.getFrontendOpts().BuildingImplicitModule)); Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator( CI, InFile, OutputFile, OS, Buffer)); return llvm::make_unique<MultiplexConsumer>(std::move(Consumers)); @@ -421,6 +425,7 @@ void VerifyPCHAction::ExecuteAction() { const std::string &Sysroot = CI.getHeaderSearchOpts().Sysroot; std::unique_ptr<ASTReader> Reader(new ASTReader( CI.getPreprocessor(), CI.getASTContext(), CI.getPCHContainerReader(), + CI.getFrontendOpts().ModuleFileExtensions, Sysroot.empty() ? "" : Sysroot.c_str(), /*DisableValidation*/ false, /*AllowPCHWithCompilerErrors*/ false, @@ -564,6 +569,20 @@ namespace { } return false; } + + /// Indicates that a particular module file extension has been read. + void readModuleFileExtension( + const ModuleFileExtensionMetadata &Metadata) override { + Out.indent(2) << "Module file extension '" + << Metadata.BlockName << "' " << Metadata.MajorVersion + << "." << Metadata.MinorVersion; + if (!Metadata.UserInfo.empty()) { + Out << ": "; + Out.write_escaped(Metadata.UserInfo); + } + + Out << "\n"; + } #undef DUMP_BOOLEAN }; } @@ -583,7 +602,8 @@ void DumpModuleInfoAction::ExecuteAction() { DumpModuleInfoListener Listener(Out); ASTReader::readASTFileControlBlock( getCurrentFile(), getCompilerInstance().getFileManager(), - getCompilerInstance().getPCHContainerReader(), Listener); + getCompilerInstance().getPCHContainerReader(), + /*FindModuleFileExtensions=*/true, Listener); } //===----------------------------------------------------------------------===// diff --git a/clang/lib/Frontend/TestModuleFileExtension.cpp b/clang/lib/Frontend/TestModuleFileExtension.cpp new file mode 100644 index 00000000000..e17dc3c1ba6 --- /dev/null +++ b/clang/lib/Frontend/TestModuleFileExtension.cpp @@ -0,0 +1,121 @@ +//===-- TestModuleFileExtension.cpp - Module Extension Tester -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "TestModuleFileExtension.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Serialization/ASTReader.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/Bitcode/BitstreamWriter.h" +#include "llvm/Support/raw_ostream.h" +using namespace clang; +using namespace clang::serialization; + +TestModuleFileExtension::Writer::~Writer() { } + +void TestModuleFileExtension::Writer::writeExtensionContents( + llvm::BitstreamWriter &Stream) { + using namespace llvm; + + // Write an abbreviation for this record. + BitCodeAbbrev *Abv = new llvm::BitCodeAbbrev(); + Abv->Add(BitCodeAbbrevOp(FIRST_EXTENSION_RECORD_ID)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of characters + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // message + auto Abbrev = Stream.EmitAbbrev(Abv); + + // Write a message into the extension block. + SmallString<64> Message; + { + auto Ext = static_cast<TestModuleFileExtension *>(getExtension()); + raw_svector_ostream OS(Message); + OS << "Hello from " << Ext->BlockName << " v" << Ext->MajorVersion << "." + << Ext->MinorVersion; + } + SmallVector<uint64_t, 4> Record; + Record.push_back(FIRST_EXTENSION_RECORD_ID); + Record.push_back(Message.size()); + Stream.EmitRecordWithBlob(Abbrev, Record, Message); +} + +TestModuleFileExtension::Reader::Reader(ModuleFileExtension *Ext, + const llvm::BitstreamCursor &InStream) + : ModuleFileExtensionReader(Ext), Stream(InStream) +{ + // Read the extension block. + SmallVector<uint64_t, 4> Record; + while (true) { + llvm::BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + switch (Entry.Kind) { + case llvm::BitstreamEntry::SubBlock: + case llvm::BitstreamEntry::EndBlock: + case llvm::BitstreamEntry::Error: + return; + + case llvm::BitstreamEntry::Record: + break; + } + + Record.clear(); + StringRef Blob; + unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); + switch (RecCode) { + case FIRST_EXTENSION_RECORD_ID: { + StringRef Message = Blob.substr(0, Record[0]); + fprintf(stderr, "Read extension block message: %s\n", + Message.str().c_str()); + break; + } + } + } +} + +TestModuleFileExtension::Reader::~Reader() { } + +TestModuleFileExtension::~TestModuleFileExtension() { } + +ModuleFileExtensionMetadata +TestModuleFileExtension::getExtensionMetadata() const { + return { BlockName, MajorVersion, MinorVersion, UserInfo }; +} + +llvm::hash_code TestModuleFileExtension::hashExtension( + llvm::hash_code Code) const { + if (Hashed) { + Code = llvm::hash_combine(Code, BlockName); + Code = llvm::hash_combine(Code, MajorVersion); + Code = llvm::hash_combine(Code, MinorVersion); + Code = llvm::hash_combine(Code, UserInfo); + } + + return Code; +} + +std::unique_ptr<ModuleFileExtensionWriter> +TestModuleFileExtension::createExtensionWriter(ASTWriter &) { + return std::unique_ptr<ModuleFileExtensionWriter>(new Writer(this)); +} + +std::unique_ptr<ModuleFileExtensionReader> +TestModuleFileExtension::createExtensionReader( + const ModuleFileExtensionMetadata &Metadata, + ASTReader &Reader, serialization::ModuleFile &Mod, + const llvm::BitstreamCursor &Stream) +{ + assert(Metadata.BlockName == BlockName && "Wrong block name"); + if (std::make_pair(Metadata.MajorVersion, Metadata.MinorVersion) != + std::make_pair(MajorVersion, MinorVersion)) { + Reader.getDiags().Report(Mod.ImportLoc, + diag::err_test_module_file_extension_version) + << BlockName << Metadata.MajorVersion << Metadata.MinorVersion + << MajorVersion << MinorVersion; + return nullptr; + } + + return std::unique_ptr<ModuleFileExtensionReader>( + new TestModuleFileExtension::Reader(this, Stream)); +} diff --git a/clang/lib/Frontend/TestModuleFileExtension.h b/clang/lib/Frontend/TestModuleFileExtension.h new file mode 100644 index 00000000000..2a7245463cf --- /dev/null +++ b/clang/lib/Frontend/TestModuleFileExtension.h @@ -0,0 +1,71 @@ +//===-- TestModuleFileExtension.h - Module Extension Tester -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H +#define LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H + +#include "clang/Serialization/ModuleFileExtension.h" +#include "clang/Basic/LLVM.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Bitcode/BitstreamReader.h" +#include <string> + +namespace clang { + +/// A module file extension used for testing purposes. +class TestModuleFileExtension : public ModuleFileExtension { + std::string BlockName; + unsigned MajorVersion; + unsigned MinorVersion; + bool Hashed; + std::string UserInfo; + + class Writer : public ModuleFileExtensionWriter { + public: + Writer(ModuleFileExtension *Ext) : ModuleFileExtensionWriter(Ext) { } + ~Writer() override; + + void writeExtensionContents(llvm::BitstreamWriter &Stream) override; + }; + + class Reader : public ModuleFileExtensionReader { + llvm::BitstreamCursor Stream; + + public: + ~Reader() override; + + Reader(ModuleFileExtension *Ext, const llvm::BitstreamCursor &InStream); + }; + +public: + TestModuleFileExtension(StringRef BlockName, + unsigned MajorVersion, + unsigned MinorVersion, + bool Hashed, + StringRef UserInfo) + : BlockName(BlockName), + MajorVersion(MajorVersion), MinorVersion(MinorVersion), + Hashed(Hashed), UserInfo(UserInfo) { } + ~TestModuleFileExtension() override; + + ModuleFileExtensionMetadata getExtensionMetadata() const override; + + llvm::hash_code hashExtension(llvm::hash_code Code) const override; + + std::unique_ptr<ModuleFileExtensionWriter> + createExtensionWriter(ASTWriter &Writer) override; + + std::unique_ptr<ModuleFileExtensionReader> + createExtensionReader(const ModuleFileExtensionMetadata &Metadata, + ASTReader &Reader, serialization::ModuleFile &Mod, + const llvm::BitstreamCursor &Stream) override; +}; + +} // end namespace clang + +#endif // LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index c54f659f09f..c09e97c1584 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -160,6 +160,12 @@ bool ChainedASTReaderListener::visitInputFile(StringRef Filename, return Continue; } +void ChainedASTReaderListener::readModuleFileExtension( + const ModuleFileExtensionMetadata &Metadata) { + First->readModuleFileExtension(Metadata); + Second->readModuleFileExtension(Metadata); +} + //===----------------------------------------------------------------------===// // PCH validator implementation //===----------------------------------------------------------------------===// @@ -3423,6 +3429,36 @@ static void updateModuleTimestamp(ModuleFile &MF) { OS << "Timestamp file\n"; } +/// \brief Given a cursor at the start of an AST file, scan ahead and drop the +/// cursor into the start of the given block ID, returning false on success and +/// true on failure. +static bool SkipCursorToBlock(BitstreamCursor &Cursor, unsigned BlockID) { + while (1) { + llvm::BitstreamEntry Entry = Cursor.advance(); + switch (Entry.Kind) { + case llvm::BitstreamEntry::Error: + case llvm::BitstreamEntry::EndBlock: + return true; + + case llvm::BitstreamEntry::Record: + // Ignore top-level records. + Cursor.skipRecord(Entry.ID); + break; + + case llvm::BitstreamEntry::SubBlock: + if (Entry.ID == BlockID) { + if (Cursor.EnterSubBlock(BlockID)) + return true; + // Found it! + return false; + } + + if (Cursor.SkipBlock()) + return true; + } + } +} + ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName, ModuleKind Type, SourceLocation ImportLoc, @@ -3480,6 +3516,12 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName, if (ASTReadResult Result = ReadASTBlock(F, ClientLoadCapabilities)) return Result; + // Read the extension blocks. + while (!SkipCursorToBlock(F.Stream, EXTENSION_BLOCK_ID)) { + if (ASTReadResult Result = ReadExtensionBlock(F)) + return Result; + } + // Once read, set the ModuleFile bit base offset and update the size in // bits of all files we've seen. F.GlobalBitOffset = TotalModulesSizeInBits; @@ -3738,14 +3780,13 @@ ASTReader::ReadASTCore(StringRef FileName, // This is used for compatibility with older PCH formats. bool HaveReadControlBlock = false; - while (1) { llvm::BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { case llvm::BitstreamEntry::Error: - case llvm::BitstreamEntry::EndBlock: case llvm::BitstreamEntry::Record: + case llvm::BitstreamEntry::EndBlock: Error("invalid record at top-level of AST file"); return Failure; @@ -3800,6 +3841,79 @@ ASTReader::ReadASTCore(StringRef FileName, break; } } + + return Success; +} + +/// Parse a record and blob containing module file extension metadata. +static bool parseModuleFileExtensionMetadata( + const SmallVectorImpl<uint64_t> &Record, + StringRef Blob, + ModuleFileExtensionMetadata &Metadata) { + if (Record.size() < 4) return true; + + Metadata.MajorVersion = Record[0]; + Metadata.MinorVersion = Record[1]; + + unsigned BlockNameLen = Record[2]; + unsigned UserInfoLen = Record[3]; + + if (BlockNameLen + UserInfoLen > Blob.size()) return true; + + Metadata.BlockName = std::string(Blob.data(), Blob.data() + BlockNameLen); + Metadata.UserInfo = std::string(Blob.data() + BlockNameLen, + Blob.data() + BlockNameLen + UserInfoLen); + return false; +} + +ASTReader::ASTReadResult ASTReader::ReadExtensionBlock(ModuleFile &F) { + BitstreamCursor &Stream = F.Stream; + + RecordData Record; + while (true) { + llvm::BitstreamEntry Entry = Stream.advance(); + switch (Entry.Kind) { + case llvm::BitstreamEntry::SubBlock: + if (Stream.SkipBlock()) + return Failure; + + continue; + + case llvm::BitstreamEntry::EndBlock: + return Success; + + case llvm::BitstreamEntry::Error: + return HadErrors; + + case llvm::BitstreamEntry::Record: + break; + } + + Record.clear(); + StringRef Blob; + unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); + switch (RecCode) { + case EXTENSION_METADATA: { + ModuleFileExtensionMetadata Metadata; + if (parseModuleFileExtensionMetadata(Record, Blob, Metadata)) + return Failure; + + // Find a module file extension with this block name. + auto Known = ModuleFileExtensions.find(Metadata.BlockName); + if (Known == ModuleFileExtensions.end()) break; + + // Form a reader. + if (auto Reader = Known->second->createExtensionReader(Metadata, *this, + F, Stream)) { + F.ExtensionReaders.push_back(std::move(Reader)); + } + + break; + } + } + } + + return Success; } void ASTReader::InitializeContext() { @@ -3940,36 +4054,6 @@ void ASTReader::finalizeForWriting() { // Nothing to do for now. } -/// \brief Given a cursor at the start of an AST file, scan ahead and drop the -/// cursor into the start of the given block ID, returning false on success and -/// true on failure. -static bool SkipCursorToBlock(BitstreamCursor &Cursor, unsigned BlockID) { - while (1) { - llvm::BitstreamEntry Entry = Cursor.advance(); - switch (Entry.Kind) { - case llvm::BitstreamEntry::Error: - case llvm::BitstreamEntry::EndBlock: - return true; - - case llvm::BitstreamEntry::Record: - // Ignore top-level records. - Cursor.skipRecord(Entry.ID); - break; - - case llvm::BitstreamEntry::SubBlock: - if (Entry.ID == BlockID) { - if (Cursor.EnterSubBlock(BlockID)) - return true; - // Found it! - return false; - } - - if (Cursor.SkipBlock()) - return true; - } - } -} - /// \brief Reads and return the signature record from \p StreamFile's control /// block, or else returns 0. static ASTFileSignature readASTFileSignature(llvm::BitstreamReader &StreamFile){ @@ -4097,6 +4181,7 @@ namespace { bool ASTReader::readASTFileControlBlock( StringRef Filename, FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, + bool FindModuleFileExtensions, ASTReaderListener &Listener) { // Open the AST file. // FIXME: This allows use of the VFS; we do not allow use of the @@ -4126,7 +4211,8 @@ bool ASTReader::readASTFileControlBlock( RecordData Record; std::string ModuleDir; - while (1) { + bool DoneWithControlBlock = false; + while (!DoneWithControlBlock) { llvm::BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { @@ -4159,7 +4245,8 @@ bool ASTReader::readASTFileControlBlock( } case llvm::BitstreamEntry::EndBlock: - return false; + DoneWithControlBlock = true; + break; case llvm::BitstreamEntry::Error: return true; @@ -4168,6 +4255,8 @@ bool ASTReader::readASTFileControlBlock( break; } + if (DoneWithControlBlock) break; + Record.clear(); StringRef Blob; unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); @@ -4251,6 +4340,50 @@ bool ASTReader::readASTFileControlBlock( break; } } + + // Look for module file extension blocks, if requested. + if (FindModuleFileExtensions) { + while (!SkipCursorToBlock(Stream, EXTENSION_BLOCK_ID)) { + bool DoneWithExtensionBlock = false; + while (!DoneWithExtensionBlock) { + llvm::BitstreamEntry Entry = Stream.advance(); + + switch (Entry.Kind) { + case llvm::BitstreamEntry::SubBlock: + if (Stream.SkipBlock()) + return true; + + continue; + + case llvm::BitstreamEntry::EndBlock: + DoneWithExtensionBlock = true; + continue; + + case llvm::BitstreamEntry::Error: + return true; + + case llvm::BitstreamEntry::Record: + break; + } + + Record.clear(); + StringRef Blob; + unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); + switch (RecCode) { + case EXTENSION_METADATA: { + ModuleFileExtensionMetadata Metadata; + if (parseModuleFileExtensionMetadata(Record, Blob, Metadata)) + return true; + + Listener.readModuleFileExtension(Metadata); + break; + } + } + } + } + } + + return false; } bool ASTReader::isAcceptableASTFile( @@ -4261,6 +4394,7 @@ bool ASTReader::isAcceptableASTFile( SimplePCHValidator validator(LangOpts, TargetOpts, PPOpts, ExistingModuleCachePath, FileMgr); return !readASTFileControlBlock(Filename, FileMgr, PCHContainerRdr, + /*FindModuleFileExtensions=*/false, validator); } @@ -8483,13 +8617,15 @@ void ASTReader::pushExternalDeclIntoScope(NamedDecl *D, DeclarationName Name) { } } -ASTReader::ASTReader(Preprocessor &PP, ASTContext &Context, - const PCHContainerReader &PCHContainerRdr, - StringRef isysroot, bool DisableValidation, - bool AllowASTWithCompilerErrors, - bool AllowConfigurationMismatch, bool ValidateSystemInputs, - bool UseGlobalIndex, - std::unique_ptr<llvm::Timer> ReadTimer) +ASTReader::ASTReader( + Preprocessor &PP, ASTContext &Context, + const PCHContainerReader &PCHContainerRdr, + ArrayRef<IntrusiveRefCntPtr<ModuleFileExtension>> Extensions, + StringRef isysroot, bool DisableValidation, + bool AllowASTWithCompilerErrors, + bool AllowConfigurationMismatch, bool ValidateSystemInputs, + bool UseGlobalIndex, + std::unique_ptr<llvm::Timer> ReadTimer) : Listener(new PCHValidator(PP, *this)), DeserializationListener(nullptr), OwnsDeserializationListener(false), SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()), PCHContainerRdr(PCHContainerRdr), @@ -8513,6 +8649,18 @@ ASTReader::ASTReader(Preprocessor &PP, ASTContext &Context, TotalModulesSizeInBits(0), NumCurrentElementsDeserializing(0), PassingDeclsToConsumer(false), ReadingKind(Read_None) { SourceMgr.setExternalSLocEntrySource(this); + + for (const auto &Ext : Extensions) { + auto BlockName = Ext->getExtensionMetadata().BlockName; + auto Known = ModuleFileExtensions.find(BlockName); + if (Known != ModuleFileExtensions.end()) { + Diags.Report(diag::warn_duplicate_module_file_extension) + << BlockName; + continue; + } + + ModuleFileExtensions.insert({BlockName, Ext}); + } } ASTReader::~ASTReader() { diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 01e9f220f97..c5a9755fed3 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "clang/Serialization/ASTWriter.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "ASTCommon.h" #include "ASTReaderInternals.h" #include "MultiOnDiskHashTable.h" @@ -1094,7 +1095,11 @@ void ASTWriter::WriteBlockInfoBlock() { RECORD(PPD_MACRO_EXPANSION); RECORD(PPD_MACRO_DEFINITION); RECORD(PPD_INCLUSION_DIRECTIVE); - + + // Decls and Types block. + BLOCK(EXTENSION_BLOCK); + RECORD(EXTENSION_METADATA); + #undef RECORD #undef BLOCK Stream.ExitBlock(); @@ -3876,6 +3881,40 @@ void ASTWriter::WriteOptimizePragmaOptions(Sema &SemaRef) { Stream.EmitRecord(OPTIMIZE_PRAGMA_OPTIONS, Record); } +void ASTWriter::WriteModuleFileExtension(ModuleFileExtensionWriter &Writer) { + // Enter the extension block. + Stream.EnterSubblock(EXTENSION_BLOCK_ID, 4); + + // Emit the metadata record abbreviation. + llvm::BitCodeAbbrev *Abv = new llvm::BitCodeAbbrev(); + Abv->Add(llvm::BitCodeAbbrevOp(EXTENSION_METADATA)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob)); + unsigned Abbrev = Stream.EmitAbbrev(Abv); + + // Emit the metadata record. + RecordData Record; + auto Metadata = Writer.getExtension()->getExtensionMetadata(); + Record.push_back(EXTENSION_METADATA); + Record.push_back(Metadata.MajorVersion); + Record.push_back(Metadata.MinorVersion); + Record.push_back(Metadata.BlockName.size()); + Record.push_back(Metadata.UserInfo.size()); + SmallString<64> Buffer; + Buffer += Metadata.BlockName; + Buffer += Metadata.UserInfo; + Stream.EmitRecordWithBlob(Abbrev, Record, Buffer); + + // Emit the contents of the extension block. + Writer.writeExtensionContents(Stream); + + // Exit the extension block. + Stream.ExitBlock(); +} + //===----------------------------------------------------------------------===// // General Serialization Routines //===----------------------------------------------------------------------===// @@ -3979,7 +4018,10 @@ void ASTWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) { SelectorOffsets[ID - FirstSelectorID] = Offset; } -ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, bool IncludeTimestamps) +ASTWriter::ASTWriter( + llvm::BitstreamWriter &Stream, + ArrayRef<llvm::IntrusiveRefCntPtr<ModuleFileExtension>> Extensions, + bool IncludeTimestamps) : Stream(Stream), Context(nullptr), PP(nullptr), Chain(nullptr), WritingModule(nullptr), IncludeTimestamps(IncludeTimestamps), WritingAST(false), DoneWritingDeclsAndTypes(false), @@ -3999,7 +4041,12 @@ ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, bool IncludeTimestamps) DeclVarAbbrev(0), DeclFieldAbbrev(0), DeclEnumAbbrev(0), DeclObjCIvarAbbrev(0), DeclCXXMethodAbbrev(0), DeclRefExprAbbrev(0), CharacterLiteralAbbrev(0), IntegerLiteralAbbrev(0), - ExprImplicitCastAbbrev(0) {} + ExprImplicitCastAbbrev(0) { + for (const auto &Ext : Extensions) { + if (auto Writer = Ext->createExtensionWriter(*this)) + ModuleFileExtensionWriters.push_back(std::move(Writer)); + } +} ASTWriter::~ASTWriter() { llvm::DeleteContainerSeconds(FileDeclIDs); @@ -4396,7 +4443,6 @@ uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, WriteCXXCtorInitializersOffsets(); WriteFileDeclIDsMap(); WriteSourceManagerBlock(Context.getSourceManager(), PP); - WriteComments(); WritePreprocessor(PP, isModule); WriteHeaderSearch(PP.getHeaderSearchInfo()); @@ -4526,6 +4572,10 @@ uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, Stream.EmitRecord(STATISTICS, Record); Stream.ExitBlock(); + // Write the module file extension blocks. + for (const auto &ExtWriter : ModuleFileExtensionWriters) + WriteModuleFileExtension(*ExtWriter); + return Signature; } diff --git a/clang/lib/Serialization/CMakeLists.txt b/clang/lib/Serialization/CMakeLists.txt index d885db22975..95b33c388c5 100644 --- a/clang/lib/Serialization/CMakeLists.txt +++ b/clang/lib/Serialization/CMakeLists.txt @@ -15,6 +15,7 @@ add_clang_library(clangSerialization GeneratePCH.cpp GlobalModuleIndex.cpp Module.cpp + ModuleFileExtension.cpp ModuleManager.cpp ADDITIONAL_HEADERS diff --git a/clang/lib/Serialization/GeneratePCH.cpp b/clang/lib/Serialization/GeneratePCH.cpp index 9de2fdb75a0..4a2255ab6d3 100644 --- a/clang/lib/Serialization/GeneratePCH.cpp +++ b/clang/lib/Serialization/GeneratePCH.cpp @@ -23,13 +23,15 @@ using namespace clang; -PCHGenerator::PCHGenerator(const Preprocessor &PP, StringRef OutputFile, - clang::Module *Module, StringRef isysroot, - std::shared_ptr<PCHBuffer> Buffer, - bool AllowASTWithErrors, bool IncludeTimestamps) +PCHGenerator::PCHGenerator( + const Preprocessor &PP, StringRef OutputFile, + clang::Module *Module, StringRef isysroot, + std::shared_ptr<PCHBuffer> Buffer, + ArrayRef<llvm::IntrusiveRefCntPtr<ModuleFileExtension>> Extensions, + bool AllowASTWithErrors, bool IncludeTimestamps) : PP(PP), OutputFile(OutputFile), Module(Module), isysroot(isysroot.str()), SemaPtr(nullptr), Buffer(Buffer), Stream(Buffer->Data), - Writer(Stream, IncludeTimestamps), + Writer(Stream, Extensions, IncludeTimestamps), AllowASTWithErrors(AllowASTWithErrors) { Buffer->IsComplete = false; } diff --git a/clang/lib/Serialization/ModuleFileExtension.cpp b/clang/lib/Serialization/ModuleFileExtension.cpp new file mode 100644 index 00000000000..81dcfd60ce8 --- /dev/null +++ b/clang/lib/Serialization/ModuleFileExtension.cpp @@ -0,0 +1,22 @@ +//===-- ModuleFileExtension.cpp - Module File Extensions ------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "clang/Serialization/ModuleFileExtension.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/Support/raw_ostream.h" +using namespace clang; + +ModuleFileExtension::~ModuleFileExtension() { } + +llvm::hash_code ModuleFileExtension::hashExtension(llvm::hash_code Code) const { + return Code; +} + +ModuleFileExtensionWriter::~ModuleFileExtensionWriter() { } + +ModuleFileExtensionReader::~ModuleFileExtensionReader() { } |