diff options
| -rw-r--r-- | compiler-rt/lib/profile/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | compiler-rt/lib/profile/InstrProfiling.c | 219 | ||||
| -rw-r--r-- | compiler-rt/lib/profile/InstrProfilingValue.c | 237 | ||||
| -rw-r--r-- | compiler-rt/make/platform/clang_darwin.mk | 2 | ||||
| -rw-r--r-- | compiler-rt/make/platform/clang_linux.mk | 2 | ||||
| -rw-r--r-- | compiler-rt/test/profile/instrprof-without-libc.c | 5 |
6 files changed, 245 insertions, 221 deletions
diff --git a/compiler-rt/lib/profile/CMakeLists.txt b/compiler-rt/lib/profile/CMakeLists.txt index 0e726d83b01..1b10ade0eee 100644 --- a/compiler-rt/lib/profile/CMakeLists.txt +++ b/compiler-rt/lib/profile/CMakeLists.txt @@ -27,6 +27,7 @@ add_custom_target(profile) set(PROFILE_SOURCES GCDAProfiling.c InstrProfiling.c + InstrProfilingValue.c InstrProfilingBuffer.c InstrProfilingFile.c InstrProfilingWriter.c diff --git a/compiler-rt/lib/profile/InstrProfiling.c b/compiler-rt/lib/profile/InstrProfiling.c index f5a5022d27a..e6be7e722df 100644 --- a/compiler-rt/lib/profile/InstrProfiling.c +++ b/compiler-rt/lib/profile/InstrProfiling.c @@ -14,28 +14,8 @@ #include <stdlib.h> #include <string.h> #define INSTR_PROF_VALUE_PROF_DATA -#define INSTR_PROF_COMMON_API_IMPL #include "InstrProfData.inc" -#define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory"); -#define PROF_OOM_RETURN(Msg) \ - { \ - PROF_OOM(Msg) \ - return 0; \ - } - -#if COMPILER_RT_HAS_ATOMICS != 1 -LLVM_LIBRARY_VISIBILITY -uint32_t BoolCmpXchg(void **Ptr, void *OldV, void *NewV) { - void *R = *Ptr; - if (R == OldV) { - *Ptr = NewV; - return 1; - } - return 0; -} -#endif - char *(*GetEnvHook)(const char *) = 0; LLVM_LIBRARY_VISIBILITY uint64_t __llvm_profile_get_magic(void) { @@ -86,202 +66,3 @@ LLVM_LIBRARY_VISIBILITY void __llvm_profile_reset_counters(void) { } } -/* This method is only used in value profiler mock testing. */ -LLVM_LIBRARY_VISIBILITY void -__llvm_profile_set_num_value_sites(__llvm_profile_data *Data, - uint32_t ValueKind, uint16_t NumValueSites) { - *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites; -} - -/* This method is only used in value profiler mock testing. */ -LLVM_LIBRARY_VISIBILITY const __llvm_profile_data * -__llvm_profile_iterate_data(const __llvm_profile_data *Data) { - return Data + 1; -} - -/* This method is only used in value profiler mock testing. */ -LLVM_LIBRARY_VISIBILITY void * -__llvm_get_function_addr(const __llvm_profile_data *Data) { - return Data->FunctionPointer; -} - -/* Allocate an array that holds the pointers to the linked lists of - * value profile counter nodes. The number of element of the array - * is the total number of value profile sites instrumented. Returns - * 0 if allocation fails. - */ - -static int allocateValueProfileCounters(__llvm_profile_data *Data) { - uint64_t NumVSites = 0; - uint32_t VKI; - for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) - NumVSites += Data->NumValueSites[VKI]; - - ValueProfNode **Mem = - (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *)); - if (!Mem) - return 0; - if (!BOOL_CMPXCHG(&Data->Values, 0, Mem)) { - free(Mem); - return 0; - } - return 1; -} - -static void deallocateValueProfileCounters(__llvm_profile_data *Data) { - uint64_t NumVSites = 0, I; - uint32_t VKI; - if (!Data->Values) - return; - for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) - NumVSites += Data->NumValueSites[VKI]; - for (I = 0; I < NumVSites; I++) { - ValueProfNode *Node = ((ValueProfNode **)Data->Values)[I]; - while (Node) { - ValueProfNode *Next = Node->Next; - free(Node); - Node = Next; - } - } - free(Data->Values); -} - -LLVM_LIBRARY_VISIBILITY void -__llvm_profile_instrument_target(uint64_t TargetValue, void *Data, - uint32_t CounterIndex) { - - __llvm_profile_data *PData = (__llvm_profile_data *)Data; - if (!PData) - return; - - if (!PData->Values) { - if (!allocateValueProfileCounters(PData)) - return; - } - - ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values; - ValueProfNode *PrevVNode = NULL; - ValueProfNode *CurrentVNode = ValueCounters[CounterIndex]; - - uint8_t VDataCount = 0; - while (CurrentVNode) { - if (TargetValue == CurrentVNode->VData.Value) { - CurrentVNode->VData.Count++; - return; - } - PrevVNode = CurrentVNode; - CurrentVNode = CurrentVNode->Next; - ++VDataCount; - } - - if (VDataCount >= UCHAR_MAX) - return; - - CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode)); - if (!CurrentVNode) - return; - - CurrentVNode->VData.Value = TargetValue; - CurrentVNode->VData.Count++; - - uint32_t Success = 0; - if (!ValueCounters[CounterIndex]) - Success = BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode); - else if (PrevVNode && !PrevVNode->Next) - Success = BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode); - - if (!Success) { - free(CurrentVNode); - return; - } -} - -/* For multi-threaded programs, while the profile is being dumped, other - threads may still be updating the value profile data and creating new - value entries. To accommadate this, we need to add extra bytes to the - data buffer. The size of the extra space is controlled by an environment - variable. */ -static unsigned getVprofExtraBytes() { - const char *ExtraStr = - GetEnvHook ? GetEnvHook("LLVM_VALUE_PROF_BUFFER_EXTRA") : 0; - if (!ExtraStr || !ExtraStr[0]) - return 1024; - return (unsigned)atoi(ExtraStr); -} - -/* Extract the value profile data info from the runtime. */ -#define DEF_VALUE_RECORD(R, NS, V) \ - ValueProfRuntimeRecord R; \ - if (initializeValueProfRuntimeRecord(&R, NS, V)) \ - PROF_OOM_RETURN("Failed to write value profile data "); - -#define DTOR_VALUE_RECORD(R) finalizeValueProfRuntimeRecord(&R); - -LLVM_LIBRARY_VISIBILITY uint64_t -__llvm_profile_gather_value_data(uint8_t **VDataArray) { - size_t S = 0, RealSize = 0, BufferCapacity = 0, Extra = 0; - __llvm_profile_data *I; - if (!VDataArray) - PROF_OOM_RETURN("Failed to write value profile data "); - - const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); - - /* - * Compute the total Size of the buffer to hold ValueProfData - * structures for functions with value profile data. - */ - for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) { - - DEF_VALUE_RECORD(R, I->NumValueSites, I->Values); - - /* Compute the size of ValueProfData from this runtime record. */ - if (getNumValueKindsRT(&R) != 0) - S += getValueProfDataSizeRT(&R); - - DTOR_VALUE_RECORD(R); - } - /* No value sites or no value profile data is collected. */ - if (!S) - return 0; - - Extra = getVprofExtraBytes(); - BufferCapacity = S + Extra; - *VDataArray = calloc(BufferCapacity, sizeof(uint8_t)); - if (!*VDataArray) - PROF_OOM_RETURN("Failed to write value profile data "); - - ValueProfData *VD = (ValueProfData *)(*VDataArray); - /* - * Extract value profile data and write into ValueProfData structure - * one by one. Note that new value profile data added to any value - * site (from another thread) after the ValueProfRuntimeRecord is - * initialized (when the profile data snapshot is taken) won't be - * collected. This is not a problem as those dropped value will have - * very low taken count. - */ - for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) { - DEF_VALUE_RECORD(R, I->NumValueSites, I->Values); - if (getNumValueKindsRT(&R) == 0) - continue; - - /* Record R has taken a snapshot of the VP data at this point. Newly - added VP data for this function will be dropped. */ - /* Check if there is enough space. */ - if (BufferCapacity - RealSize < getValueProfDataSizeRT(&R)) { - PROF_ERR("Value profile data is dropped :%s \n", - "Out of buffer space. Use environment " - " LLVM_VALUE_PROF_BUFFER_EXTRA to allocate more"); - I->Values = 0; - } - - serializeValueProfDataFromRT(&R, VD); - deallocateValueProfileCounters(I); - I->Values = VD; - RealSize += VD->TotalSize; - VD = (ValueProfData *)((char *)VD + VD->TotalSize); - DTOR_VALUE_RECORD(R); - } - - return RealSize; -} diff --git a/compiler-rt/lib/profile/InstrProfilingValue.c b/compiler-rt/lib/profile/InstrProfilingValue.c new file mode 100644 index 00000000000..eafa99bedf2 --- /dev/null +++ b/compiler-rt/lib/profile/InstrProfilingValue.c @@ -0,0 +1,237 @@ +/*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\ +|* +|* The LLVM Compiler Infrastructure +|* +|* This file is distributed under the University of Illinois Open Source +|* License. See LICENSE.TXT for details. +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#define INSTR_PROF_VALUE_PROF_DATA +#define INSTR_PROF_COMMON_API_IMPL +#include "InstrProfData.inc" + +#define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory"); +#define PROF_OOM_RETURN(Msg) \ + { \ + PROF_OOM(Msg) \ + return 0; \ + } + +#if COMPILER_RT_HAS_ATOMICS != 1 +LLVM_LIBRARY_VISIBILITY +uint32_t BoolCmpXchg(void **Ptr, void *OldV, void *NewV) { + void *R = *Ptr; + if (R == OldV) { + *Ptr = NewV; + return 1; + } + return 0; +} +#endif + +/* This method is only used in value profiler mock testing. */ +LLVM_LIBRARY_VISIBILITY void +__llvm_profile_set_num_value_sites(__llvm_profile_data *Data, + uint32_t ValueKind, uint16_t NumValueSites) { + *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites; +} + +/* This method is only used in value profiler mock testing. */ +LLVM_LIBRARY_VISIBILITY const __llvm_profile_data * +__llvm_profile_iterate_data(const __llvm_profile_data *Data) { + return Data + 1; +} + +/* This method is only used in value profiler mock testing. */ +LLVM_LIBRARY_VISIBILITY void * +__llvm_get_function_addr(const __llvm_profile_data *Data) { + return Data->FunctionPointer; +} + +/* Allocate an array that holds the pointers to the linked lists of + * value profile counter nodes. The number of element of the array + * is the total number of value profile sites instrumented. Returns + * 0 if allocation fails. + */ + +static int allocateValueProfileCounters(__llvm_profile_data *Data) { + uint64_t NumVSites = 0; + uint32_t VKI; + for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) + NumVSites += Data->NumValueSites[VKI]; + + ValueProfNode **Mem = + (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *)); + if (!Mem) + return 0; + if (!BOOL_CMPXCHG(&Data->Values, 0, Mem)) { + free(Mem); + return 0; + } + return 1; +} + +static void deallocateValueProfileCounters(__llvm_profile_data *Data) { + uint64_t NumVSites = 0, I; + uint32_t VKI; + if (!Data->Values) + return; + for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) + NumVSites += Data->NumValueSites[VKI]; + for (I = 0; I < NumVSites; I++) { + ValueProfNode *Node = ((ValueProfNode **)Data->Values)[I]; + while (Node) { + ValueProfNode *Next = Node->Next; + free(Node); + Node = Next; + } + } + free(Data->Values); +} + +LLVM_LIBRARY_VISIBILITY void +__llvm_profile_instrument_target(uint64_t TargetValue, void *Data, + uint32_t CounterIndex) { + + __llvm_profile_data *PData = (__llvm_profile_data *)Data; + if (!PData) + return; + + if (!PData->Values) { + if (!allocateValueProfileCounters(PData)) + return; + } + + ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values; + ValueProfNode *PrevVNode = NULL; + ValueProfNode *CurrentVNode = ValueCounters[CounterIndex]; + + uint8_t VDataCount = 0; + while (CurrentVNode) { + if (TargetValue == CurrentVNode->VData.Value) { + CurrentVNode->VData.Count++; + return; + } + PrevVNode = CurrentVNode; + CurrentVNode = CurrentVNode->Next; + ++VDataCount; + } + + if (VDataCount >= UCHAR_MAX) + return; + + CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode)); + if (!CurrentVNode) + return; + + CurrentVNode->VData.Value = TargetValue; + CurrentVNode->VData.Count++; + + uint32_t Success = 0; + if (!ValueCounters[CounterIndex]) + Success = BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode); + else if (PrevVNode && !PrevVNode->Next) + Success = BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode); + + if (!Success) { + free(CurrentVNode); + return; + } +} + +/* For multi-threaded programs, while the profile is being dumped, other + threads may still be updating the value profile data and creating new + value entries. To accommadate this, we need to add extra bytes to the + data buffer. The size of the extra space is controlled by an environment + variable. */ +static unsigned getVprofExtraBytes() { + const char *ExtraStr = + GetEnvHook ? GetEnvHook("LLVM_VALUE_PROF_BUFFER_EXTRA") : 0; + if (!ExtraStr || !ExtraStr[0]) + return 1024; + return (unsigned)atoi(ExtraStr); +} + +/* Extract the value profile data info from the runtime. */ +#define DEF_VALUE_RECORD(R, NS, V) \ + ValueProfRuntimeRecord R; \ + if (initializeValueProfRuntimeRecord(&R, NS, V)) \ + PROF_OOM_RETURN("Failed to write value profile data "); + +#define DTOR_VALUE_RECORD(R) finalizeValueProfRuntimeRecord(&R); + +LLVM_LIBRARY_VISIBILITY uint64_t +__llvm_profile_gather_value_data(uint8_t **VDataArray) { + size_t S = 0, RealSize = 0, BufferCapacity = 0, Extra = 0; + __llvm_profile_data *I; + if (!VDataArray) + PROF_OOM_RETURN("Failed to write value profile data "); + + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + + /* + * Compute the total Size of the buffer to hold ValueProfData + * structures for functions with value profile data. + */ + for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) { + + DEF_VALUE_RECORD(R, I->NumValueSites, I->Values); + + /* Compute the size of ValueProfData from this runtime record. */ + if (getNumValueKindsRT(&R) != 0) + S += getValueProfDataSizeRT(&R); + + DTOR_VALUE_RECORD(R); + } + /* No value sites or no value profile data is collected. */ + if (!S) + return 0; + + Extra = getVprofExtraBytes(); + BufferCapacity = S + Extra; + *VDataArray = calloc(BufferCapacity, sizeof(uint8_t)); + if (!*VDataArray) + PROF_OOM_RETURN("Failed to write value profile data "); + + ValueProfData *VD = (ValueProfData *)(*VDataArray); + /* + * Extract value profile data and write into ValueProfData structure + * one by one. Note that new value profile data added to any value + * site (from another thread) after the ValueProfRuntimeRecord is + * initialized (when the profile data snapshot is taken) won't be + * collected. This is not a problem as those dropped value will have + * very low taken count. + */ + for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) { + DEF_VALUE_RECORD(R, I->NumValueSites, I->Values); + if (getNumValueKindsRT(&R) == 0) + continue; + + /* Record R has taken a snapshot of the VP data at this point. Newly + added VP data for this function will be dropped. */ + /* Check if there is enough space. */ + if (BufferCapacity - RealSize < getValueProfDataSizeRT(&R)) { + PROF_ERR("Value profile data is dropped :%s \n", + "Out of buffer space. Use environment " + " LLVM_VALUE_PROF_BUFFER_EXTRA to allocate more"); + I->Values = 0; + } + + serializeValueProfDataFromRT(&R, VD); + deallocateValueProfileCounters(I); + I->Values = VD; + RealSize += VD->TotalSize; + VD = (ValueProfData *)((char *)VD + VD->TotalSize); + DTOR_VALUE_RECORD(R); + } + + return RealSize; +} diff --git a/compiler-rt/make/platform/clang_darwin.mk b/compiler-rt/make/platform/clang_darwin.mk index f85ced5cdda..9944481d8be 100644 --- a/compiler-rt/make/platform/clang_darwin.mk +++ b/compiler-rt/make/platform/clang_darwin.mk @@ -277,7 +277,7 @@ FUNCTIONS.osx := mulosi4 mulodi4 muloti4 $(ATOMIC_FUNCTIONS) $(FP16_FUNCTIONS) FUNCTIONS.profile_osx := GCDAProfiling InstrProfiling InstrProfilingBuffer \ InstrProfilingFile InstrProfilingPlatformDarwin \ InstrProfilingRuntime InstrProfilingUtil \ - InstrProfilingWriter + InstrProfilingWriter InstrProfilingValue FUNCTIONS.profile_ios := $(FUNCTIONS.profile_osx) FUNCTIONS.asan_osx_dynamic := $(AsanFunctions) $(AsanCXXFunctions) \ diff --git a/compiler-rt/make/platform/clang_linux.mk b/compiler-rt/make/platform/clang_linux.mk index 80c22006ca8..bf5ee4a928f 100644 --- a/compiler-rt/make/platform/clang_linux.mk +++ b/compiler-rt/make/platform/clang_linux.mk @@ -79,7 +79,7 @@ FUNCTIONS.builtins-x86_64 := $(CommonFunctions) $(ArchFunctions.x86_64) FUNCTIONS.profile-i386 := GCDAProfiling InstrProfiling InstrProfilingBuffer \ InstrProfilingFile InstrProfilingPlatformOther \ InstrProfilingRuntime InstrProfilingUtil \ - InstrProfilingWriter + InstrProfilingWriter InstrProfilingValue FUNCTIONS.profile-x86_64 := $(FUNCTIONS.profile-i386) # Always use optimized variants. diff --git a/compiler-rt/test/profile/instrprof-without-libc.c b/compiler-rt/test/profile/instrprof-without-libc.c index f23149186cf..eb0a76ded39 100644 --- a/compiler-rt/test/profile/instrprof-without-libc.c +++ b/compiler-rt/test/profile/instrprof-without-libc.c @@ -58,4 +58,9 @@ int main(int argc, const char *argv[]) { // CHECK-SYMBOLS-NOT: _getenv // CHECK-SYMBOLS-NOT: getenv // CHECK-SYMBOLS-NOT: _malloc +// CHECK-SYMBOLS-NOT: malloc +// CHECK-SYMBOLS-NOT: _calloc +// CHECK-SYMBOLS-NOT: calloc +// CHECK-SYMBOLS-NOT: _free +// CHECK-SYMBOLS-NOT: free // CHECK-SYMBOLS-NOT: _open |
