Implement sized deallocation for std::allocator and friends.

Summary:
C++14 sized deallocation is disabled by default due to ABI concerns. However, when a user manually enables it then libc++ should take advantage of it since sized deallocation can provide a significant performance win depending on the underlying malloc implementation. (Note that libc++'s definitions of sized delete don't do anything special yet, but users are free to provide their own).

This patch updates __libcpp_deallocate to selectively call sized operator delete when it's available. `__libcpp_deallocate_unsized` should be used when the size of the allocation is unknown.

On Apple this patch makes no attempt to determine if the sized operator delete is unavailable, only that the language feature is enabled. This could cause a compile error when using `std::allocator`, but the same compile error would occur whenever the user calls `new`, so I don't think it's a problem.

Reviewers: ldionne, mclow.lists

Reviewed By: ldionne

Subscribers: rsmith, ckennelly, libcxx-commits, christof

Differential Revision: https://reviews.llvm.org/D53120

llvm-svn: 345214

1 files changed, 88 insertions, 13 deletions

diff --git a/libcxx/include/new b/libcxx/include/new
index 09d5162aeab..412d51bd027 100644
--- a/libcxx/include/new
+++ b/libcxx/include/new
@@ -104,12 +104,17 @@ void  operator delete[](void* ptr, void*) noexcept;
 #pragma GCC system_header
 #endif
 
-#if !defined(_LIBCPP_BUILDING_LIBRARY) && _LIBCPP_STD_VER < 14
+#if !defined(__cpp_sized_deallocation) || __cpp_sized_deallocation  < 201309L
+#define _LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION
+#endif
+
+#if !defined(_LIBCPP_BUILDING_LIBRARY) && _LIBCPP_STD_VER < 14 && \
+    defined(_LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION)
 # define _LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION
 #endif
 
 #if defined(_LIBCPP_HAS_NO_LIBRARY_SIZED_DEALLOCATION) || \
-    !defined(__cpp_sized_deallocation) || __cpp_sized_deallocation < 201309
+    defined(_LIBCPP_HAS_NO_LANGUAGE_SIZED_DEALLOCATION)
 # define _LIBCPP_HAS_NO_SIZED_DEALLOCATION
 #endif
 
@@ -255,24 +260,94 @@ inline _LIBCPP_INLINE_VISIBILITY void *__libcpp_allocate(size_t __size, size_t _
 #endif
 }
 
-inline _LIBCPP_INLINE_VISIBILITY void __libcpp_deallocate(void* __ptr, size_t __align) {
+struct _DeallocateCaller {
+  static inline _LIBCPP_INLINE_VISIBILITY
+  void __do_deallocate_handle_size_align(void *__ptr, size_t __size, size_t __align) {
+#if defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
+    ((void)__align);
+    return __do_deallocate_handle_size(__ptr, __size);
+#else
+    if (__is_overaligned_for_new(__align)) {
+      const align_val_t __align_val = static_cast<align_val_t>(__align);
+      return __do_deallocate_handle_size(__ptr, __size, __align_val);
+    } else {
+      return __do_deallocate_handle_size(__ptr, __size);
+    }
+#endif
+  }
+
+  static inline _LIBCPP_INLINE_VISIBILITY
+  void __do_deallocate_handle_align(void *__ptr, size_t __align) {
+#if defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
+    ((void)__align);
+    return __do_call(__ptr);
+#else
+    if (__is_overaligned_for_new(__align)) {
+      const align_val_t __align_val = static_cast<align_val_t>(__align);
+      return __do_call(__ptr, __align_val);
+    } else {
+      return __do_call(__ptr);
+    }
+#endif
+  }
+
+ private:
+  static inline void __do_deallocate_handle_size(void *__ptr, size_t __size) {
+#ifdef _LIBCPP_HAS_NO_SIZED_DEALLOCATION
+    ((void)__size);
+    return __do_call(__ptr);
+#else
+    return __do_call(__ptr, __size);
+#endif
+  }
+
 #ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
-  if (__is_overaligned_for_new(__align)) {
-    const align_val_t __align_val = static_cast<align_val_t>(__align);
-# ifdef _LIBCPP_HAS_NO_BUILTIN_OVERLOADED_OPERATOR_NEW_DELETE
-    return ::operator delete(__ptr, __align_val);
-# else
-    return __builtin_operator_delete(__ptr, __align_val);
-# endif
+  static inline void __do_deallocate_handle_size(void *__ptr, size_t __size, align_val_t __align) {
+#ifdef _LIBCPP_HAS_NO_SIZED_DEALLOCATION
+    ((void)__size);
+    return __do_call(__ptr, __align);
+#else
+    return __do_call(__ptr, __size, __align);
+#endif
   }
+#endif
+
+private:
+  template <class _A1, class _A2>
+  static inline void __do_call(void *__ptr, _A1 __a1, _A2 __a2) {
+#if defined(_LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE) || \
+    defined(_LIBCPP_HAS_NO_BUILTIN_OVERLOADED_OPERATOR_NEW_DELETE)
+    return ::operator delete(__ptr, __a1, __a2);
 #else
-  ((void)__align);
+    return __builtin_operator_delete(__ptr, __a1, __a2);
 #endif
+  }
+
+  template <class _A1>
+  static inline void __do_call(void *__ptr, _A1 __a1) {
+#if defined(_LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE) || \
+    defined(_LIBCPP_HAS_NO_BUILTIN_OVERLOADED_OPERATOR_NEW_DELETE)
+    return ::operator delete(__ptr, __a1);
+#else
+    return __builtin_operator_delete(__ptr, __a1);
+#endif
+  }
+
+  static inline void __do_call(void *__ptr) {
 #ifdef _LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE
-  return ::operator delete(__ptr);
+    return ::operator delete(__ptr);
 #else
-  return __builtin_operator_delete(__ptr);
+    return __builtin_operator_delete(__ptr);
 #endif
+  }
+};
+
+inline _LIBCPP_INLINE_VISIBILITY void __libcpp_deallocate(void* __ptr, size_t __size, size_t __align) {
+  _DeallocateCaller::__do_deallocate_handle_size_align(__ptr, __size, __align);
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void __libcpp_deallocate_unsized(void* __ptr, size_t __align) {
+  _DeallocateCaller::__do_deallocate_handle_align(__ptr, __align);
 }
 
 #ifdef _LIBCPP_BAD_ARRAY_LENGTH_DEFINED