From 8dbe3629a09a754464f1420ce3059676c986090a Mon Sep 17 00:00:00 2001 From: Zachary Turner Date: Fri, 27 May 2016 01:54:44 +0000 Subject: [codeview,pdb] Try really hard to conserve memory when reading. PDBs can be extremely large. We're already mapping the entire PDB into the process's address space, but to make matters worse the blocks of the PDB are not arranged contiguously. So, when we have something like an array or a string embedded into the stream, we have to make a copy. Since it's convenient to use traditional data structures to iterate and manipulate these records, we need the memory to be contiguous. As a result of this, we were using roughly twice as much memory as the file size of the PDB, because every stream was copied out and re-stitched together contiguously. This patch addresses this by improving the MappedBlockStream to allocate from a BumpPtrAllocator only when a read requires a discontiguous read. Furthermore, it introduces some data structures backed by a stream which can iterate over both fixed and variable length records of a PDB. Since everything is backed by a stream and not a buffer, we can read almost everything from the PDB with zero copies. Differential Revision: http://reviews.llvm.org/D20654 Reviewed By: ruiu llvm-svn: 270951 --- llvm/lib/DebugInfo/CodeView/ByteStream.cpp | 27 +++++------- llvm/lib/DebugInfo/CodeView/StreamReader.cpp | 66 +++++++++++++++++++++++----- 2 files changed, 64 insertions(+), 29 deletions(-) (limited to 'llvm/lib/DebugInfo/CodeView') diff --git a/llvm/lib/DebugInfo/CodeView/ByteStream.cpp b/llvm/lib/DebugInfo/CodeView/ByteStream.cpp index 879343afbfd..1ea976b6a26 100644 --- a/llvm/lib/DebugInfo/CodeView/ByteStream.cpp +++ b/llvm/lib/DebugInfo/CodeView/ByteStream.cpp @@ -17,31 +17,24 @@ using namespace llvm::codeview; ByteStream::ByteStream() {} -ByteStream::ByteStream(MutableArrayRef Bytes) { initialize(Bytes); } +ByteStream::ByteStream(MutableArrayRef Data) : Data(Data) {} -ByteStream::ByteStream(uint32_t Length) { initialize(Length); } - -ByteStream::~ByteStream() { reset(); } +ByteStream::~ByteStream() {} void ByteStream::reset() { Ownership.reset(); Data = MutableArrayRef(); } -void ByteStream::initialize(MutableArrayRef Bytes) { - reset(); - Data = Bytes; -} - -void ByteStream::initialize(uint32_t Length) { +void ByteStream::load(uint32_t Length) { reset(); if (Length > 0) Data = MutableArrayRef(new uint8_t[Length], Length); Ownership.reset(Data.data()); } -Error ByteStream::initialize(StreamReader &Reader, uint32_t Length) { - initialize(Length); +Error ByteStream::load(StreamReader &Reader, uint32_t Length) { + load(Length); auto EC = Reader.readBytes(Data); if (EC) reset(); @@ -52,15 +45,15 @@ Error ByteStream::readBytes(uint32_t Offset, MutableArrayRef Buffer) const { if (Data.size() < Buffer.size() + Offset) return make_error(cv_error_code::insufficient_buffer); - ::memcpy(Buffer.data(), Data.data() + Offset, Buffer.size()); + ::memcpy(Buffer.data() + Offset, Data.data(), Buffer.size()); return Error::success(); } -Error ByteStream::getArrayRef(uint32_t Offset, ArrayRef &Buffer, - uint32_t Length) const { - if (Data.size() < Length + Offset) +Error ByteStream::readBytes(uint32_t Offset, uint32_t Size, + ArrayRef &Buffer) const { + if (Data.size() < Buffer.size() + Offset) return make_error(cv_error_code::insufficient_buffer); - Buffer = Data.slice(Offset, Length); + Buffer = Data.slice(Offset, Size); return Error::success(); } diff --git a/llvm/lib/DebugInfo/CodeView/StreamReader.cpp b/llvm/lib/DebugInfo/CodeView/StreamReader.cpp index 8a2926bc883..64985bfd0e2 100644 --- a/llvm/lib/DebugInfo/CodeView/StreamReader.cpp +++ b/llvm/lib/DebugInfo/CodeView/StreamReader.cpp @@ -10,12 +10,20 @@ #include "llvm/DebugInfo/CodeView/StreamReader.h" #include "llvm/DebugInfo/CodeView/CodeViewError.h" +#include "llvm/DebugInfo/CodeView/StreamRef.h" using namespace llvm; using namespace llvm::codeview; StreamReader::StreamReader(const StreamInterface &S) : Stream(S), Offset(0) {} +Error StreamReader::readBytes(uint32_t Size, ArrayRef &Buffer) { + if (auto EC = Stream.readBytes(Offset, Size, Buffer)) + return EC; + Offset += Size; + return Error::success(); +} + Error StreamReader::readBytes(MutableArrayRef Buffer) { if (auto EC = Stream.readBytes(Offset, Buffer)) return EC; @@ -23,29 +31,63 @@ Error StreamReader::readBytes(MutableArrayRef Buffer) { return Error::success(); } +Error StreamReader::readInteger(uint16_t &Dest) { + const support::ulittle16_t *P; + if (auto EC = readObject(P)) + return EC; + Dest = *P; + return Error::success(); +} + Error StreamReader::readInteger(uint32_t &Dest) { - support::ulittle32_t P; - if (auto EC = readObject(&P)) + const support::ulittle32_t *P; + if (auto EC = readObject(P)) return EC; - Dest = P; + Dest = *P; return Error::success(); } -Error StreamReader::readZeroString(std::string &Dest) { - Dest.clear(); - char C; +Error StreamReader::readZeroString(StringRef &Dest) { + uint32_t Length = 0; + // First compute the length of the string by reading 1 byte at a time. + uint32_t OriginalOffset = getOffset(); + const char *C; do { - if (auto EC = readObject(&C)) + if (auto EC = readObject(C)) return EC; - if (C != '\0') - Dest.push_back(C); - } while (C != '\0'); + if (*C != '\0') + ++Length; + } while (*C != '\0'); + // Now go back and request a reference for that many bytes. + uint32_t NewOffset = getOffset(); + setOffset(OriginalOffset); + + ArrayRef Data; + if (auto EC = readBytes(Length, Data)) + return EC; + Dest = StringRef(reinterpret_cast(Data.begin()), Data.size()); + + // Now set the offset back to where it was after we calculated the length. + setOffset(NewOffset); return Error::success(); } -Error StreamReader::getArrayRef(ArrayRef &Array, uint32_t Length) { - if (auto EC = Stream.getArrayRef(Offset, Array, Length)) +Error StreamReader::readFixedString(StringRef &Dest, uint32_t Length) { + ArrayRef Bytes; + if (auto EC = readBytes(Length, Bytes)) return EC; + Dest = StringRef(reinterpret_cast(Bytes.begin()), Bytes.size()); + return Error::success(); +} + +Error StreamReader::readStreamRef(StreamRef &Ref) { + return readStreamRef(Ref, bytesRemaining()); +} + +Error StreamReader::readStreamRef(StreamRef &Ref, uint32_t Length) { + if (bytesRemaining() < Length) + return make_error(cv_error_code::insufficient_buffer); + Ref = StreamRef(Stream, Offset, Length); Offset += Length; return Error::success(); } -- cgit v1.2.3