Make lcm/gcd work better in edge cases. Fixes a UBSAN failure.

llvm-svn: 294779

1 files changed, 17 insertions, 12 deletions

diff --git a/libcxx/include/numeric b/libcxx/include/numeric
index ff2853e917a..8f25146938a 100644
--- a/libcxx/include/numeric
+++ b/libcxx/include/numeric
@@ -201,18 +201,23 @@ iota(_ForwardIterator __first, _ForwardIterator __last, _Tp __value_)
 
 
 #if _LIBCPP_STD_VER > 14
-template <typename _Tp, bool _IsSigned = is_signed<_Tp>::value> struct __abs;
+template <typename _Result, typename _Source, bool _IsSigned = is_signed<_Source>::value> struct __abs;
 
-template <typename _Tp>
-struct __abs<_Tp, true> {
+template <typename _Result, typename _Source>
+struct __abs<_Result, _Source, true> {
     _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY
-    _Tp operator()(_Tp __t) const noexcept { return __t >= 0 ? __t : -__t; }
+    _Result operator()(_Source __t) const noexcept
+    {
+    if (__t >= 0) return __t;
+    if (__t == numeric_limits<_Source>::min()) return -static_cast<_Result>(__t);
+    return -__t;
+    }
 };
 
-template <typename _Tp>
-struct __abs<_Tp, false> {
+template <typename _Result, typename _Source>
+struct __abs<_Result, _Source, false> {
     _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY
-    _Tp operator()(_Tp __t) const noexcept { return __t; }
+    _Result operator()(_Source __t) const noexcept { return __t; }
 };
 
 
@@ -220,7 +225,7 @@ template<class _Tp>
 _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY
 _Tp __gcd(_Tp __m, _Tp __n)
 {
-    static_assert((!is_signed<_Tp>::value), "" );
+    static_assert((!is_signed<_Tp>::value), "");
     return __n == 0 ? __m : __gcd<_Tp>(__n, __m % __n);
 }
 
@@ -235,8 +240,8 @@ gcd(_Tp __m, _Up __n)
     static_assert((!is_same<typename remove_cv<_Up>::type, bool>::value), "Second argument to gcd cannot be bool" );
     using _Rp = common_type_t<_Tp,_Up>;
     using _Wp = make_unsigned_t<_Rp>;
-    return static_cast<_Rp>(__gcd(static_cast<_Wp>(__abs<_Tp>()(__m)),
-                                  static_cast<_Wp>(__abs<_Up>()(__n))));
+    return static_cast<_Rp>(__gcd(static_cast<_Wp>(__abs<_Rp, _Tp>()(__m)),
+                                  static_cast<_Wp>(__abs<_Rp, _Up>()(__n))));
 }
 
 template<class _Tp, class _Up>
@@ -251,8 +256,8 @@ lcm(_Tp __m, _Up __n)
         return 0;
 
     using _Rp = common_type_t<_Tp,_Up>;
-    _Rp __val1 = __abs<_Tp>()(__m) / gcd(__m,__n);
-    _Up __val2 = __abs<_Up>()(__n);
+    _Rp __val1 = __abs<_Rp, _Tp>()(__m) / gcd(__m, __n);
+    _Rp __val2 = __abs<_Rp, _Up>()(__n);
     _LIBCPP_ASSERT((numeric_limits<_Rp>::max() / __val1 > __val2), "Overflow in lcm");
     return __val1 * __val2;
 }