diff options
Diffstat (limited to 'compiler-rt/lib')
-rw-r--r-- | compiler-rt/lib/profile/InstrProfData.inc | 4 | ||||
-rw-r--r-- | compiler-rt/lib/profile/InstrProfiling.h | 46 | ||||
-rw-r--r-- | compiler-rt/lib/profile/InstrProfilingBuffer.c | 74 | ||||
-rw-r--r-- | compiler-rt/lib/profile/InstrProfilingFile.c | 139 | ||||
-rw-r--r-- | compiler-rt/lib/profile/InstrProfilingPort.h | 10 | ||||
-rw-r--r-- | compiler-rt/lib/profile/InstrProfilingRuntime.cpp | 3 | ||||
-rw-r--r-- | compiler-rt/lib/profile/InstrProfilingWriter.c | 22 |
7 files changed, 287 insertions, 11 deletions
diff --git a/compiler-rt/lib/profile/InstrProfData.inc b/compiler-rt/lib/profile/InstrProfData.inc index 7078af5f4cf..99f41d8fef0 100644 --- a/compiler-rt/lib/profile/InstrProfData.inc +++ b/compiler-rt/lib/profile/InstrProfData.inc @@ -130,7 +130,9 @@ INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) @@ -628,7 +630,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 4 +#define INSTR_PROF_RAW_VERSION 5 /* Indexed profile format version (start from 1). */ #define INSTR_PROF_INDEX_VERSION 5 /* Coverage mapping format vresion (start from 0). */ diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index ffc4396169d..78dfc675efd 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -39,6 +39,22 @@ typedef struct ValueProfNode { } ValueProfNode; /*! + * \brief Return 1 if profile counters are continuously synced to the raw + * profile via an mmap(). This is in contrast to the default mode, in which + * the raw profile is written out at program exit time. + */ +int __llvm_profile_is_continuous_mode_enabled(void); + +/*! + * \brief Enable continuous mode. + * + * See \ref __llvm_profile_is_continuous_mode_enabled. The behavior is undefined + * if continuous mode is already enabled, or if it cannot be enable due to + * conflicting options. + */ +void __llvm_profile_enable_continuous_mode(void); + +/*! * \brief Get number of bytes necessary to pad the argument to eight * byte boundary. */ @@ -159,6 +175,12 @@ int __llvm_orderfile_dump(void); * Note: There may be multiple copies of the profile runtime (one for each * instrumented image/DSO). This API only modifies the filename within the * copy of the runtime available to the calling image. + * + * Warning: This is a no-op if continuous mode (\ref + * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is + * that in continuous mode, profile counters are mmap()'d to the profile at + * program initialization time. Support for transferring the mmap'd profile + * counts to a new file has not been implemented. */ void __llvm_profile_set_filename(const char *Name); @@ -181,6 +203,12 @@ void __llvm_profile_set_filename(const char *Name); * Note: There may be multiple copies of the profile runtime (one for each * instrumented image/DSO). This API only modifies the file object within the * copy of the runtime available to the calling image. + * + * Warning: This is a no-op if continuous mode (\ref + * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is + * that in continuous mode, profile counters are mmap()'d to the profile at + * program initialization time. Support for transferring the mmap'd profile + * counts to a new file has not been implemented. */ void __llvm_profile_set_file_object(FILE *File, int EnableMerge); @@ -223,6 +251,24 @@ uint64_t __llvm_profile_get_version(void); uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, const __llvm_profile_data *End); +/* ! \brief Given the sizes of the data and counter information, return the + * number of padding bytes before and after the counters, and after the names, + * in the raw profile. + * + * Note: In this context, "size" means "number of entries", i.e. the first two + * arguments must be the result of __llvm_profile_get_data_size() and of + * (__llvm_profile_end_counters() - __llvm_profile_begin_counters()) resp. + * + * Note: When mmap() mode is disabled, no padding bytes before/after counters + * are needed. However, in mmap() mode, the counter section in the raw profile + * must be page-aligned: this API computes the number of padding bytes + * needed to achieve that. + */ +void __llvm_profile_get_padding_sizes_for_counters( + uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t *PaddingBytesAfterNames); + /*! * \brief Set the flag that profile data has been dumped to the file. * This is useful for users to disable dumping profile data to the file for diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index 5bdeb8e3280..089ff5a0153 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -8,6 +8,27 @@ #include "InstrProfiling.h" #include "InstrProfilingInternal.h" +#include "InstrProfilingPort.h" + +/* When continuous mode is enabled (%c), this parameter is set to 1. This is + * incompatible with the in-process merging mode. Lifting this restriction + * may be complicated, as merging mode requires a lock on the profile, and + * mmap() mode would require that lock to be held for the entire process + * lifetime. + * + * This parameter is defined here in InstrProfilingBuffer.o, instead of in + * InstrProfilingFile.o, to sequester all libc-dependent code in + * InstrProfilingFile.o. The test `instrprof-without-libc` will break if this + * layering is violated. */ +static int ContinuouslySyncProfile = 0; + +COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) { + return ContinuouslySyncProfile; +} + +COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) { + ContinuouslySyncProfile = 1; +} COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer(void) { @@ -30,6 +51,41 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, sizeof(__llvm_profile_data); } +/// Calculate the number of padding bytes needed to add to \p Offset in order +/// for (\p Offset + Padding) to be page-aligned. +static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset, + unsigned PageSize) { + uint64_t OffsetModPage = Offset % PageSize; + if (OffsetModPage > 0) + return PageSize - OffsetModPage; + return 0; +} + +COMPILER_RT_VISIBILITY +void __llvm_profile_get_padding_sizes_for_counters( + uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t *PaddingBytesAfterNames) { + if (!__llvm_profile_is_continuous_mode_enabled()) { + *PaddingBytesBeforeCounters = 0; + *PaddingBytesAfterCounters = 0; + *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); + return; + } + + // In continuous mode, the file offsets for headers and for the start of + // counter sections need to be page-aligned. + unsigned PageSize = getpagesize(); + uint64_t DataSizeInBytes = DataSize * sizeof(__llvm_profile_data); + uint64_t CountersSizeInBytes = CountersSize * sizeof(uint64_t); + *PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign( + sizeof(__llvm_profile_header) + DataSizeInBytes, PageSize); + *PaddingBytesAfterCounters = + calculateBytesNeededToPageAlign(CountersSizeInBytes, PageSize); + *PaddingBytesAfterNames = + calculateBytesNeededToPageAlign(NamesSize, PageSize); +} + COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, @@ -37,11 +93,21 @@ uint64_t __llvm_profile_get_size_for_buffer_internal( const char *NamesBegin, const char *NamesEnd) { /* Match logic in __llvm_profile_write_buffer(). */ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); - const uint8_t Padding = __llvm_profile_get_num_padding_bytes(NamesSize); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + uint64_t CountersSize = CountersEnd - CountersBegin; + + /* Determine how much padding is needed before/after the counters and after + * the names. */ + uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, + PaddingBytesAfterNames; + __llvm_profile_get_padding_sizes_for_counters( + DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + return sizeof(__llvm_profile_header) + - (__llvm_profile_get_data_size(DataBegin, DataEnd) * - sizeof(__llvm_profile_data)) + - (CountersEnd - CountersBegin) * sizeof(uint64_t) + NamesSize + Padding; + (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters + + (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters + + NamesSize + PaddingBytesAfterNames; } COMPILER_RT_VISIBILITY diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c index 1b253c3e865..e2253de8eda 100644 --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -32,6 +32,7 @@ #include "InstrProfiling.h" #include "InstrProfilingInternal.h" +#include "InstrProfilingPort.h" #include "InstrProfilingUtil.h" /* From where is profile name specified. @@ -100,6 +101,12 @@ static void setProfileFile(FILE *File) { ProfileFile = File; } COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File, int EnableMerge) { + if (__llvm_profile_is_continuous_mode_enabled()) { + PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported, because " + "continuous sync mode (%%c) is enabled", + fileno(File)); + return; + } setProfileFile(File); setProfileMergeRequested(EnableMerge); } @@ -347,6 +354,15 @@ static void truncateCurrentFile(void) { if (lprofCurFilename.MergePoolSize) return; + /* Only create the profile directory and truncate an existing profile once. + * In continuous mode, this is necessary, as the profile is written-to by the + * runtime initializer. */ + const char *lprofInitOnceEnv = "__LLVM_PROFILE_RT_INIT_ONCE"; + int initialized = getenv(lprofInitOnceEnv) != NULL; + if (initialized) + return; + setenv(lprofInitOnceEnv, lprofInitOnceEnv, 1); + createProfileDir(Filename); /* Truncate the file. Later we'll reopen and append. */ @@ -356,6 +372,99 @@ static void truncateCurrentFile(void) { fclose(File); } +static void initializeProfileForContinuousMode(void) { +#if defined(__Fuchsia__) || defined(_WIN32) + PROF_ERR("%s\n", "Continuous mode not yet supported on Fuchsia or Windows."); +#else // defined(__Fuchsia__) || defined(_WIN32) + if (!__llvm_profile_is_continuous_mode_enabled()) + return; + + /* Get the sizes of various profile data sections. Taken from + * __llvm_profile_get_size_for_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const uint64_t *CountersBegin = __llvm_profile_begin_counters(); + const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *NamesBegin = __llvm_profile_begin_names(); + const char *NamesEnd = __llvm_profile_end_names(); + const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + uint64_t CountersSize = CountersEnd - CountersBegin; + + /* Check that the counter and data sections in this image are page-aligned. */ + unsigned PageSize = getpagesize(); + if ((intptr_t)CountersBegin % PageSize != 0) { + PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n", + CountersBegin, PageSize); + return; + } + if ((intptr_t)DataBegin % PageSize != 0) { + PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", + DataBegin, PageSize); + return; + } + + /* Open the raw profile in append mode. */ + int Length = getCurFilenameLength(); + char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + const char *Filename = getCurFilename(FilenameBuf, 0); + if (!Filename) + return; + FILE *File = fopen(Filename, "a+b"); + if (!File) + return; + + int Fileno = fileno(File); + + /* Check that the offset within the file is page-aligned. */ + off_t CurrentFileOffset = ftello(File); + off_t OffsetModPage = CurrentFileOffset % PageSize; + if (OffsetModPage != 0) { + PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not" + "page-aligned. CurrentFileOffset = %lld, pagesz = %u.\n", + CurrentFileOffset, PageSize); + return; + } + + /* Determine how much padding is needed before/after the counters and after + * the names. */ + uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, + PaddingBytesAfterNames; + __llvm_profile_get_padding_sizes_for_counters( + DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + + uint64_t PageAlignedCountersLength = + (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters; + uint64_t FileOffsetToCounters = + CurrentFileOffset + sizeof(__llvm_profile_header) + + (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters; + + /* Write the partial profile. This grows the file to a point where the mmap() + * can succeed. Leak the file handle, as the file should stay open. */ + setProfileFile(File); + int rc = writeFile(Filename); + if (rc) + PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); + setProfileFile(NULL); + + uint64_t *CounterMmap = (uint64_t *)mmap( + (void *)CountersBegin, PageAlignedCountersLength, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToCounters); + if (CounterMmap != CountersBegin) { + PROF_ERR( + "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" + " - CountersBegin: %p\n" + " - PageAlignedCountersLength: %llu\n" + " - Fileno: %d\n" + " - FileOffsetToCounters: %llu\n", + strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno, + FileOffsetToCounters); + return; + } +#endif // defined(__Fuchsia__) || defined(_WIN32) +} + static const char *DefaultProfileName = "default.profraw"; static void resetFilenameToDefault(void) { if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) { @@ -419,12 +528,33 @@ static int parseFilenamePattern(const char *FilenamePat, FilenamePat); return -1; } + } else if (FilenamePat[I] == 'c') { + if (__llvm_profile_is_continuous_mode_enabled()) { + PROF_WARN("%%c specifier can only be specified once in %s.\n", + FilenamePat); + return -1; + } + if (MergingEnabled) { + PROF_WARN("%%c specifier can not be used with profile merging (%%m) " + "in %s.\n", + FilenamePat); + return -1; + } + + __llvm_profile_enable_continuous_mode(); + I++; /* advance to 'c' */ } else if (containsMergeSpecifier(FilenamePat, I)) { if (MergingEnabled) { PROF_WARN("%%m specifier can only be specified once in %s.\n", FilenamePat); return -1; } + if (__llvm_profile_is_continuous_mode_enabled()) { + PROF_WARN("%%c specifier can not be used with profile merging (%%m) " + "in %s.\n", + FilenamePat); + return -1; + } MergingEnabled = 1; if (FilenamePat[I] == 'm') lprofCurFilename.MergePoolSize = 1; @@ -447,6 +577,7 @@ static void parseAndSetFilename(const char *FilenamePat, const char *OldFilenamePat = lprofCurFilename.FilenamePat; ProfileNameSpecifier OldPNS = lprofCurFilename.PNS; + /* The old profile name specifier takes precedence over the old one. */ if (PNS < OldPNS) return; @@ -475,6 +606,7 @@ static void parseAndSetFilename(const char *FilenamePat, } truncateCurrentFile(); + initializeProfileForContinuousMode(); } /* Return buffer length that is required to store the current profile @@ -511,7 +643,8 @@ static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) { return 0; if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts || - lprofCurFilename.MergePoolSize)) { + lprofCurFilename.MergePoolSize || + __llvm_profile_is_continuous_mode_enabled())) { if (!ForceUseBuf) return lprofCurFilename.FilenamePat; @@ -646,6 +779,8 @@ void __llvm_profile_initialize_file(void) { */ COMPILER_RT_VISIBILITY void __llvm_profile_set_filename(const char *FilenamePat) { + if (__llvm_profile_is_continuous_mode_enabled()) + return; parseAndSetFilename(FilenamePat, PNS_runtime_api, 1); } @@ -660,7 +795,7 @@ int __llvm_profile_write_file(void) { char *FilenameBuf; int PDeathSig = 0; - if (lprofProfileDumped()) { + if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) { PROF_NOTE("Profile data not written to file: %s.\n", "already written"); return 0; } diff --git a/compiler-rt/lib/profile/InstrProfilingPort.h b/compiler-rt/lib/profile/InstrProfilingPort.h index da5b5c0f8bb..9462cf1a240 100644 --- a/compiler-rt/lib/profile/InstrProfilingPort.h +++ b/compiler-rt/lib/profile/InstrProfilingPort.h @@ -99,6 +99,16 @@ (((ch) == DIR_SEPARATOR) || ((ch) == DIR_SEPARATOR_2)) #endif /* DIR_SEPARATOR_2 */ +#if defined(_WIN32) +static inline size_t getpagesize() { + SYSTEM_INFO S; + GetNativeSystemInfo(&S); + return S.dwPageSize; +} +#else /* defined(_WIN32) */ +#include <unistd.h> +#endif /* defined(_WIN32) */ + #define PROF_ERR(Format, ...) \ fprintf(stderr, "LLVM Profile Error: " Format, __VA_ARGS__); diff --git a/compiler-rt/lib/profile/InstrProfilingRuntime.cpp b/compiler-rt/lib/profile/InstrProfilingRuntime.cpp index 679186ef830..5dff09d7063 100644 --- a/compiler-rt/lib/profile/InstrProfilingRuntime.cpp +++ b/compiler-rt/lib/profile/InstrProfilingRuntime.cpp @@ -19,8 +19,9 @@ namespace { class RegisterRuntime { public: RegisterRuntime() { - __llvm_profile_register_write_file_atexit(); __llvm_profile_initialize_file(); + if (!__llvm_profile_is_continuous_mode_enabled()) + __llvm_profile_register_write_file_atexit(); } }; diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c index d910cbb8f2f..ae9e1fa6ac1 100644 --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -14,6 +14,7 @@ #include "InstrProfiling.h" #include "InstrProfilingInternal.h" +#include "InstrProfilingPort.h" #define INSTR_PROF_VALUE_PROF_DATA #include "InstrProfData.inc" @@ -257,10 +258,11 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); const uint64_t CountersSize = CountersEnd - CountersBegin; const uint64_t NamesSize = NamesEnd - NamesBegin; - const uint64_t Padding = __llvm_profile_get_num_padding_bytes(NamesSize); /* Enough zeroes for padding. */ - const char Zeroes[sizeof(uint64_t)] = {0}; + unsigned PageSize = getpagesize(); + char Zeroes[PageSize]; + memset(Zeroes, 0, PageSize); /* Create the header. */ __llvm_profile_header Header; @@ -268,6 +270,14 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, if (!DataSize) return 0; + /* Determine how much padding is needed before/after the counters and after + * the names. */ + uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, + PaddingBytesAfterNames; + __llvm_profile_get_padding_sizes_for_counters( + DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + /* Initialize header structure. */ #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; #include "InstrProfData.inc" @@ -276,11 +286,17 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, ProfDataIOVec IOVec[] = { {&Header, sizeof(__llvm_profile_header), 1}, {DataBegin, sizeof(__llvm_profile_data), DataSize}, + {Zeroes, sizeof(uint8_t), PaddingBytesBeforeCounters}, {CountersBegin, sizeof(uint64_t), CountersSize}, + {Zeroes, sizeof(uint8_t), PaddingBytesAfterCounters}, {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize}, - {Zeroes, sizeof(uint8_t), Padding}}; + {Zeroes, sizeof(uint8_t), PaddingBytesAfterNames}}; if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) return -1; + /* Value profiling is not yet supported in continuous mode. */ + if (__llvm_profile_is_continuous_mode_enabled()) + return 0; + return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); } |