summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lld/ELF/Config.h3
-rw-r--r--lld/ELF/Driver.cpp34
-rw-r--r--lld/ELF/InputFiles.cpp17
-rw-r--r--lld/ELF/InputFiles.h7
-rw-r--r--lld/ELF/LTO.cpp94
-rw-r--r--lld/ELF/LTO.h4
-rw-r--r--lld/ELF/SymbolTable.cpp1
-rw-r--r--lld/test/ELF/lto/Inputs/thinlto_empty.ll2
-rw-r--r--lld/test/ELF/lto/thinlto.ll67
-rw-r--r--lld/test/ELF/lto/thinlto_prefix_replace.ll23
10 files changed, 221 insertions, 31 deletions
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index f58931a84a6..1223f180050 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -94,6 +94,8 @@ struct Configuration {
llvm::StringRef SoName;
llvm::StringRef Sysroot;
llvm::StringRef ThinLTOCacheDir;
+ llvm::StringRef ThinLTOIndexOnlyObjectsFile;
+ llvm::StringRef ThinLTOPrefixReplace;
std::string Rpath;
std::vector<VersionDefinition> VersionDefinitions;
std::vector<llvm::StringRef> AuxiliaryList;
@@ -156,6 +158,7 @@ struct Configuration {
bool SysvHash = false;
bool Target1Rel;
bool Trace;
+ bool ThinLTOIndexOnly;
bool UndefinedVersion;
bool WarnBackrefs;
bool WarnCommon;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index bcf4eebb4b1..144f1d5befc 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -777,27 +777,37 @@ void LinkerDriver::readConfigs(opt::InputArgList &Args) {
// Parse LTO plugin-related options for compatibility with gold.
for (auto *Arg : Args.filtered(OPT_plugin_opt)) {
StringRef S = Arg->getValue();
- if (S == "disable-verify")
+ if (S == "disable-verify") {
Config->DisableVerify = true;
- else if (S == "save-temps")
+ } else if (S == "save-temps") {
Config->SaveTemps = true;
- else if (S.startswith("O"))
+ } else if (S.startswith("O")) {
Config->LTOO = parseInt(S.substr(1), Arg);
- else if (S.startswith("lto-partitions="))
+ } else if (S.startswith("lto-partitions=")) {
Config->LTOPartitions = parseInt(S.substr(15), Arg);
- else if (S.startswith("jobs="))
+ } else if (S.startswith("jobs=")) {
Config->ThinLTOJobs = parseInt(S.substr(5), Arg);
- else if (S.startswith("mcpu="))
+ } else if (S.startswith("mcpu=")) {
parseClangOption(Saver.save("-" + S), Arg->getSpelling());
- else if (S == "new-pass-manager")
+ } else if (S == "new-pass-manager") {
Config->LTONewPassManager = true;
- else if (S == "debug-pass-manager")
+ } else if (S == "debug-pass-manager") {
Config->LTODebugPassManager = true;
- else if (S.startswith("sample-profile="))
- Config->LTOSampleProfile = S.substr(strlen("sample-profile="));
- else if (!S.startswith("/") && !S.startswith("-fresolution=") &&
- !S.startswith("-pass-through=") && !S.startswith("thinlto"))
+ } else if (S.startswith("sample-profile=")) {
+ Config->LTOSampleProfile = S.substr(15);
+ } else if (S == "thinlto-index-only") {
+ Config->ThinLTOIndexOnly = true;
+ } else if (S.startswith("thinlto-index-only=")) {
+ Config->ThinLTOIndexOnly = true;
+ Config->ThinLTOIndexOnlyObjectsFile = S.substr(19);
+ } else if (S.startswith("thinlto-prefix-replace=")) {
+ Config->ThinLTOPrefixReplace = S.substr(23);
+ if (!Config->ThinLTOPrefixReplace.contains(';'))
+ error("thinlto-prefix-replace expects 'old;new' format");
+ } else if (!S.startswith("/") && !S.startswith("-fresolution=") &&
+ !S.startswith("-pass-through=") && !S.startswith("thinlto")) {
parseClangOption(S, Arg->getSpelling());
+ }
}
// Parse -mllvm options.
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index f74a058079b..5f9b1f4a38c 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -42,6 +42,7 @@ bool InputFile::IsInGroup;
uint32_t InputFile::NextGroupId;
std::vector<BinaryFile *> elf::BinaryFiles;
std::vector<BitcodeFile *> elf::BitcodeFiles;
+std::vector<LazyObjFile *> elf::LazyObjFiles;
std::vector<InputFile *> elf::ObjectFiles;
std::vector<InputFile *> elf::SharedFiles;
@@ -1024,9 +1025,10 @@ BitcodeFile::BitcodeFile(MemoryBufferRef MB, StringRef ArchiveName,
// this causes a collision which result in only one of the objects being
// taken into consideration at LTO time (which very likely causes undefined
// symbols later in the link stage).
- MemoryBufferRef MBRef(MB.getBuffer(),
- Saver.save(ArchiveName + MB.getBufferIdentifier() +
- utostr(OffsetInArchive)));
+ MemoryBufferRef MBRef(
+ MB.getBuffer(),
+ Saver.save(ArchiveName + MB.getBufferIdentifier() +
+ (ArchiveName.empty() ? "" : utostr(OffsetInArchive))));
Obj = CHECK(lto::InputFile::create(MBRef), this);
Triple T(Obj->getTargetTriple());
@@ -1128,11 +1130,6 @@ void BinaryFile::parse() {
Data.size(), 0, STB_GLOBAL, nullptr, nullptr);
}
-static bool isBitcode(MemoryBufferRef MB) {
- using namespace sys::fs;
- return identify_magic(MB.getBuffer()) == file_magic::bitcode;
-}
-
InputFile *elf::createObjectFile(MemoryBufferRef MB, StringRef ArchiveName,
uint64_t OffsetInArchive) {
if (isBitcode(MB))
@@ -1168,9 +1165,9 @@ InputFile *elf::createSharedFile(MemoryBufferRef MB, StringRef DefaultSoName) {
}
MemoryBufferRef LazyObjFile::getBuffer() {
- if (Seen)
+ if (AddedToLink)
return MemoryBufferRef();
- Seen = true;
+ AddedToLink = true;
return MB;
}
diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h
index 09bc34b2887..dbd464394a2 100644
--- a/lld/ELF/InputFiles.h
+++ b/lld/ELF/InputFiles.h
@@ -259,11 +259,11 @@ public:
template <class ELFT> void parse();
MemoryBufferRef getBuffer();
InputFile *fetch();
+ bool AddedToLink = false;
private:
template <class ELFT> void addElfSymbols();
- bool Seen = false;
uint64_t OffsetInArchive;
};
@@ -351,8 +351,13 @@ InputFile *createObjectFile(MemoryBufferRef MB, StringRef ArchiveName = "",
uint64_t OffsetInArchive = 0);
InputFile *createSharedFile(MemoryBufferRef MB, StringRef DefaultSoName);
+inline bool isBitcode(MemoryBufferRef MB) {
+ return identify_magic(MB.getBuffer()) == llvm::file_magic::bitcode;
+}
+
extern std::vector<BinaryFile *> BinaryFiles;
extern std::vector<BitcodeFile *> BitcodeFiles;
+extern std::vector<LazyObjFile *> LazyObjFiles;
extern std::vector<InputFile *> ObjectFiles;
extern std::vector<InputFile *> SharedFiles;
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 861f73c4329..c778d6ccae4 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -20,6 +20,8 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/LTO/Caching.h"
#include "llvm/LTO/Config.h"
@@ -29,7 +31,6 @@
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cstddef>
#include <memory>
@@ -66,7 +67,57 @@ static void checkError(Error E) {
[&](ErrorInfoBase &EIB) { error(EIB.message()); });
}
-static std::unique_ptr<lto::LTO> createLTO() {
+// With the ThinLTOIndexOnly option, only the thin link is performed, and will
+// generate index files for the ThinLTO backends in a distributed build system.
+// The distributed build system may expect that index files are created for all
+// input bitcode objects provided to the linker for the thin link. However,
+// index files will not normally be created for input bitcode objects that
+// either aren't selected by the linker (i.e. in a static library and not
+// needed), or because they don't have a summary. Therefore we need to create
+// empty dummy index file outputs in those cases.
+// If SkipModule is true then .thinlto.bc should contain just
+// SkipModuleByDistributedBackend flag which requests distributed backend
+// to skip the compilation of the corresponding module and produce an empty
+// object file.
+static void writeEmptyDistributedBuildOutputs(const std::string &ModulePath,
+ const std::string &OldPrefix,
+ const std::string &NewPrefix,
+ bool SkipModule) {
+ std::string NewModulePath =
+ lto::getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
+ std::error_code EC;
+
+ raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
+ sys::fs::OpenFlags::F_None);
+ if (EC)
+ error("failed to write " + NewModulePath + ".thinlto.bc" + ": " +
+ EC.message());
+
+ if (SkipModule) {
+ ModuleSummaryIndex Index(false);
+ Index.setSkipModuleByDistributedBackend();
+ WriteIndexToFile(Index, OS);
+ }
+}
+
+// Creates and returns output stream with a list of object files for final
+// linking of distributed ThinLTO.
+static std::unique_ptr<raw_fd_ostream> createLinkedObjectsFile() {
+ if (Config->ThinLTOIndexOnlyObjectsFile.empty())
+ return nullptr;
+ std::error_code EC;
+ auto LinkedObjectsFile = llvm::make_unique<raw_fd_ostream>(
+ Config->ThinLTOIndexOnlyObjectsFile, EC, sys::fs::OpenFlags::F_None);
+ if (EC)
+ error("cannot create " + Config->ThinLTOIndexOnlyObjectsFile + ": " +
+ EC.message());
+ return LinkedObjectsFile;
+}
+
+// Creates instance of LTO.
+// LinkedObjectsFile is an output stream to write the list of object files for
+// the final ThinLTO linking. Can be nullptr.
+static std::unique_ptr<lto::LTO> createLTO(raw_fd_ostream *LinkedObjectsFile) {
lto::Config Conf;
// LLD supports the new relocations.
@@ -105,6 +156,13 @@ static std::unique_ptr<lto::LTO> createLTO() {
if (Config->ThinLTOJobs != -1u)
Backend = lto::createInProcessThinBackend(Config->ThinLTOJobs);
+ if (Config->ThinLTOIndexOnly) {
+ std::string OldPrefix, NewPrefix;
+ std::tie(OldPrefix, NewPrefix) = Config->ThinLTOPrefixReplace.split(';');
+ Backend = lto::createWriteIndexesThinBackend(OldPrefix, NewPrefix, true,
+ LinkedObjectsFile, nullptr);
+ }
+
Conf.SampleProfile = Config->LTOSampleProfile;
Conf.UseNewPM = Config->LTONewPassManager;
Conf.DebugPassManager = Config->LTODebugPassManager;
@@ -113,7 +171,9 @@ static std::unique_ptr<lto::LTO> createLTO() {
Config->LTOPartitions);
}
-BitcodeCompiler::BitcodeCompiler() : LTOObj(createLTO()) {
+BitcodeCompiler::BitcodeCompiler() {
+ LinkedObjects = createLinkedObjectsFile();
+ LTOObj = createLTO(LinkedObjects.get());
for (Symbol *Sym : Symtab->getSymbols()) {
StringRef Name = Sym->getName();
for (StringRef Prefix : {"__start_", "__stop_"})
@@ -131,6 +191,13 @@ static void undefine(Symbol *S) {
void BitcodeCompiler::add(BitcodeFile &F) {
lto::InputFile &Obj = *F.Obj;
+
+ std::string OldPrefix, NewPrefix;
+ std::tie(OldPrefix, NewPrefix) = Config->ThinLTOPrefixReplace.split(';');
+
+ // Create the empty files which, if indexed, will be overwritten later.
+ writeEmptyDistributedBuildOutputs(Obj.getName(), OldPrefix, NewPrefix, false);
+
unsigned SymNum = 0;
std::vector<Symbol *> Syms = F.getSymbols();
std::vector<lto::SymbolResolution> Resols(Syms.size());
@@ -188,6 +255,12 @@ std::vector<InputFile *> BitcodeCompiler::compile() {
Buff.resize(MaxTasks);
Files.resize(MaxTasks);
+ // If LazyObjFile has not been added to link, emit empty index files
+ if (Config->ThinLTOIndexOnly)
+ for (LazyObjFile *F : LazyObjFiles)
+ if (!F->AddedToLink && isBitcode(F->MB))
+ addLazyObjFile(F);
+
// The --thinlto-cache-dir option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
@@ -222,9 +295,24 @@ std::vector<InputFile *> BitcodeCompiler::compile() {
Ret.push_back(Obj);
}
+ // ThinLTO with index only option is required to generate only the index
+ // files. After that, we exit from linker and ThinLTO backend runs in a
+ // distributed environment.
+ if (Config->ThinLTOIndexOnly)
+ exit(0);
+
for (std::unique_ptr<MemoryBuffer> &File : Files)
if (File)
Ret.push_back(createObjectFile(*File));
return Ret;
}
+
+// For lazy object files not added to link, adds empty index files
+void BitcodeCompiler::addLazyObjFile(LazyObjFile *File) {
+ StringRef Identifier = File->getBuffer().getBufferIdentifier();
+ std::string OldPrefix, NewPrefix;
+ std::tie(OldPrefix, NewPrefix) = Config->ThinLTOPrefixReplace.split(';');
+ writeEmptyDistributedBuildOutputs(Identifier, OldPrefix, NewPrefix,
+ /* SkipModule */ true);
+}
diff --git a/lld/ELF/LTO.h b/lld/ELF/LTO.h
index 223af507a97..a491514333b 100644
--- a/lld/ELF/LTO.h
+++ b/lld/ELF/LTO.h
@@ -24,6 +24,7 @@
#include "lld/Common/LLVM.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <vector>
@@ -38,6 +39,7 @@ namespace elf {
class BitcodeFile;
class InputFile;
+class LazyObjFile;
class BitcodeCompiler {
public:
@@ -46,12 +48,14 @@ public:
void add(BitcodeFile &F);
std::vector<InputFile *> compile();
+ void addLazyObjFile(LazyObjFile *File);
private:
std::unique_ptr<llvm::lto::LTO> LTOObj;
std::vector<SmallString<0>> Buff;
std::vector<std::unique_ptr<MemoryBuffer>> Files;
llvm::DenseSet<StringRef> UsedStartStop;
+ std::unique_ptr<llvm::raw_fd_ostream> LinkedObjects;
};
} // namespace elf
} // namespace lld
diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp
index 7a688b72f21..e18810c2a01 100644
--- a/lld/ELF/SymbolTable.cpp
+++ b/lld/ELF/SymbolTable.cpp
@@ -82,6 +82,7 @@ template <class ELFT> void SymbolTable::addFile(InputFile *File) {
// Lazy object file
if (auto *F = dyn_cast<LazyObjFile>(File)) {
+ LazyObjFiles.push_back(F);
F->parse<ELFT>();
return;
}
diff --git a/lld/test/ELF/lto/Inputs/thinlto_empty.ll b/lld/test/ELF/lto/Inputs/thinlto_empty.ll
new file mode 100644
index 00000000000..a3c99cdfe77
--- /dev/null
+++ b/lld/test/ELF/lto/Inputs/thinlto_empty.ll
@@ -0,0 +1,2 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
diff --git a/lld/test/ELF/lto/thinlto.ll b/lld/test/ELF/lto/thinlto.ll
index 37d2b88131f..97b23b7df7d 100644
--- a/lld/test/ELF/lto/thinlto.ll
+++ b/lld/test/ELF/lto/thinlto.ll
@@ -1,7 +1,37 @@
; REQUIRES: x86
+
+; First ensure that the ThinLTO handling in lld handles
+; bitcode without summary sections gracefully and generates index file.
+; RUN: llvm-as %s -o %t.o
+; RUN: llvm-as %p/Inputs/thinlto.ll -o %t2.o
+; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t2.o -o %t3
+; RUN: ls %t2.o.thinlto.bc
+; RUN: not test -e %t3
+; RUN: ld.lld -m elf_x86_64 -shared %t.o %t2.o -o %t4
+; RUN: llvm-nm %t4 | FileCheck %s --check-prefix=NM
+
; Basic ThinLTO tests.
; RUN: opt -module-summary %s -o %t.o
; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o
+; RUN: opt -module-summary %p/Inputs/thinlto_empty.ll -o %t4.o
+
+; Ensure lld generates an index and not a binary if requested.
+; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t2.o -o %t3
+; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
+; RUN: llvm-bcanalyzer -dump %t2.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND2
+; RUN: not test -e %t3
+
+; Ensure lld generates an index even if the file is wrapped in --start-lib/--end-lib
+; RUN: rm -f %t2.o.thinlto.bc
+; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t4.o --start-lib %t2.o --end-lib -o %t5
+; RUN: ls %t2.o.thinlto.bc
+
+; Ensure lld generates error if unable to write to index file
+; RUN: rm -f %t4.o.thinlto.bc
+; RUN: touch %t4.o.thinlto.bc
+; RUN: chmod 400 %t4.o.thinlto.bc
+; RUN: ld.lld -m elf_x86_64 --plugin-opt=thinlto-index-only -shared %t.o %t4.o -o %t5 2>&1 | FileCheck %s --check-prefix=ERR
+; ERR: failed to write {{.*}}4.o.thinlto.bc: Permission denied
; First force single-threaded mode
; RUN: rm -f %t.lto.o %t1.lto.o
@@ -15,16 +45,43 @@
; RUN: llvm-nm %t21.lto.o | FileCheck %s --check-prefix=NM1
; RUN: llvm-nm %t22.lto.o | FileCheck %s --check-prefix=NM2
-; NM1: T f
-; NM1-NOT: U g
-
-; NM2: T g
-
; Then check without --thinlto-jobs (which currently default to hardware_concurrency)
; We just check that we don't crash or fail (as it's not sure which tests are
; stable on the final output file itself.
; RUN: ld.lld -shared %t.o %t2.o -o %t2
+; NM: T f
+; NM1: T f
+; NM1-NOT: U g
+; NM2: T g
+
+; The backend index for this module contains summaries from itself and
+; Inputs/thinlto.ll, as it imports from the latter.
+; BACKEND1: <MODULE_STRTAB_BLOCK
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}/thinlto.ll.tmp{{.*}}.o'
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}/thinlto.ll.tmp{{.*}}.o'
+; BACKEND1-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND1: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND1: <VERSION
+; BACKEND1: <FLAGS
+; BACKEND1: <VALUE_GUID op0={{1|2}} op1={{-3706093650706652785|-5300342847281564238}}
+; BACKEND1: <VALUE_GUID op0={{1|2}} op1={{-3706093650706652785|-5300342847281564238}}
+; BACKEND1: <COMBINED
+; BACKEND1: <COMBINED
+; BACKEND1: </GLOBALVAL_SUMMARY_BLOCK
+
+; The backend index for Input/thinlto.ll contains summaries from itself only,
+; as it does not import anything.
+; BACKEND2: <MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <ENTRY {{.*}} record string = '{{.*}}/thinlto.ll.tmp2.o'
+; BACKEND2-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND2-NEXT: <VERSION
+; BACKEND2-NEXT: <FLAGS
+; BACKEND2-NEXT: <VALUE_GUID op0=1 op1=-5300342847281564238
+; BACKEND2-NEXT: <COMBINED
+; BACKEND2-NEXT: </GLOBALVAL_SUMMARY_BLOCK
+
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
diff --git a/lld/test/ELF/lto/thinlto_prefix_replace.ll b/lld/test/ELF/lto/thinlto_prefix_replace.ll
new file mode 100644
index 00000000000..31b4402c173
--- /dev/null
+++ b/lld/test/ELF/lto/thinlto_prefix_replace.ll
@@ -0,0 +1,23 @@
+; REQUIRES: x86
+; Check that changing the output path via thinlto-prefix-replace works
+; RUN: mkdir -p %t/oldpath
+; RUN: opt -module-summary %s -o %t/oldpath/thinlto_prefix_replace.o
+
+; Ensure that there is no existing file at the new path, so we properly
+; test the creation of the new file there.
+; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
+; RUN: ld.lld --plugin-opt=thinlto-index-only --plugin-opt=thinlto-prefix-replace="%t/oldpath/;%t/newpath/" -shared %t/oldpath/thinlto_prefix_replace.o -o %t/thinlto_prefix_replace
+; RUN: ls %t/newpath/thinlto_prefix_replace.o.thinlto.bc
+
+; Ensure that lld generates error if prefix replace option does not have 'old;new' format
+; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
+; RUN: not ld.lld --plugin-opt=thinlto-index-only --plugin-opt=thinlto-prefix-replace="%t/oldpath/:%t/newpath/" -shared %t/oldpath/thinlto_prefix_replace.o -o %t/thinlto_prefix_replace 2>&1 | FileCheck %s --check-prefix=ERR
+; ERR: thinlto-prefix-replace expects 'old;new' format
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f() {
+entry:
+ ret void
+}
OpenPOWER on IntegriCloud