1 //===-- Double-precision sincos function ----------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "src/math/sincos.h" 10 #include "hdr/errno_macros.h" 11 #include "src/__support/FPUtil/FEnvImpl.h" 12 #include "src/__support/FPUtil/FPBits.h" 13 #include "src/__support/FPUtil/double_double.h" 14 #include "src/__support/FPUtil/dyadic_float.h" 15 #include "src/__support/FPUtil/except_value_utils.h" 16 #include "src/__support/FPUtil/multiply_add.h" 17 #include "src/__support/FPUtil/rounding_mode.h" 18 #include "src/__support/common.h" 19 #include "src/__support/macros/config.h" 20 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY 21 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA 22 #include "src/math/generic/range_reduction_double_common.h" 23 #include "src/math/generic/sincos_eval.h" 24 25 #ifdef LIBC_TARGET_CPU_HAS_FMA 26 #include "range_reduction_double_fma.h" 27 #else 28 #include "range_reduction_double_nofma.h" 29 #endif // LIBC_TARGET_CPU_HAS_FMA 30 31 namespace LIBC_NAMESPACE_DECL { 32 33 using DoubleDouble = fputil::DoubleDouble; 34 using Float128 = typename fputil::DyadicFloat<128>; 35 36 LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) { 37 using FPBits = typename fputil::FPBits<double>; 38 FPBits xbits(x); 39 40 uint16_t x_e = xbits.get_biased_exponent(); 41 42 DoubleDouble y; 43 unsigned k; 44 LargeRangeReduction range_reduction_large{}; 45 46 // |x| < 2^16 47 if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT)) { 48 // |x| < 2^-7 49 if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 7)) { 50 // |x| < 2^-27 51 if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 27)) { 52 // Signed zeros. 53 if (LIBC_UNLIKELY(x == 0.0)) { 54 *sin_x = x; 55 *cos_x = 1.0; 56 return; 57 } 58 59 // For |x| < 2^-27, max(|sin(x) - x|, |cos(x) - 1|) < ulp(x)/2. 60 #ifdef LIBC_TARGET_CPU_HAS_FMA 61 *sin_x = fputil::multiply_add(x, -0x1.0p-54, x); 62 *cos_x = fputil::multiply_add(x, -x, 1.0); 63 #else 64 *cos_x = fputil::round_result_slightly_down(1.0); 65 66 if (LIBC_UNLIKELY(x_e < 4)) { 67 int rounding_mode = fputil::quick_get_round(); 68 if (rounding_mode == FE_TOWARDZERO || 69 (xbits.sign() == Sign::POS && rounding_mode == FE_DOWNWARD) || 70 (xbits.sign() == Sign::NEG && rounding_mode == FE_UPWARD)) 71 *sin_x = FPBits(xbits.uintval() - 1).get_val(); 72 } 73 *sin_x = fputil::multiply_add(x, -0x1.0p-54, x); 74 #endif // LIBC_TARGET_CPU_HAS_FMA 75 return; 76 } 77 // No range reduction needed. 78 k = 0; 79 y.lo = 0.0; 80 y.hi = x; 81 } else { 82 // Small range reduction. 83 k = range_reduction_small(x, y); 84 } 85 } else { 86 // Inf or NaN 87 if (LIBC_UNLIKELY(x_e > 2 * FPBits::EXP_BIAS)) { 88 // sin(+-Inf) = NaN 89 if (xbits.get_mantissa() == 0) { 90 fputil::set_errno_if_required(EDOM); 91 fputil::raise_except_if_required(FE_INVALID); 92 } 93 *sin_x = *cos_x = x + FPBits::quiet_nan().get_val(); 94 return; 95 } 96 97 // Large range reduction. 98 k = range_reduction_large.fast(x, y); 99 } 100 101 DoubleDouble sin_y, cos_y; 102 103 [[maybe_unused]] double err = generic::sincos_eval(y, sin_y, cos_y); 104 105 // Look up sin(k * pi/128) and cos(k * pi/128) 106 #ifdef LIBC_MATH_HAS_SMALL_TABLES 107 // Memory saving versions. Use 65-entry table. __anon36eff5f40102(unsigned kk) 108 auto get_idx_dd = [](unsigned kk) -> DoubleDouble { 109 unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63); 110 DoubleDouble ans = SIN_K_PI_OVER_128[idx]; 111 if (kk & 128) { 112 ans.hi = -ans.hi; 113 ans.lo = -ans.lo; 114 } 115 return ans; 116 }; 117 DoubleDouble sin_k = get_idx_dd(k); 118 DoubleDouble cos_k = get_idx_dd(k + 64); 119 #else 120 // Fast look up version, but needs 256-entry table. 121 // cos(k * pi/128) = sin(k * pi/128 + pi/2) = sin((k + 64) * pi/128). 122 DoubleDouble sin_k = SIN_K_PI_OVER_128[k & 255]; 123 DoubleDouble cos_k = SIN_K_PI_OVER_128[(k + 64) & 255]; 124 #endif // LIBC_MATH_HAS_SMALL_TABLES 125 126 DoubleDouble msin_k{-sin_k.lo, -sin_k.hi}; 127 128 // After range reduction, k = round(x * 128 / pi) and y = x - k * (pi / 128). 129 // So k is an integer and -pi / 256 <= y <= pi / 256. 130 // Then sin(x) = sin((k * pi/128 + y) 131 // = sin(y) * cos(k*pi/128) + cos(y) * sin(k*pi/128) 132 DoubleDouble sin_k_cos_y = fputil::quick_mult(cos_y, sin_k); 133 DoubleDouble cos_k_sin_y = fputil::quick_mult(sin_y, cos_k); 134 // cos(x) = cos((k * pi/128 + y) 135 // = cos(y) * cos(k*pi/128) - sin(y) * sin(k*pi/128) 136 DoubleDouble cos_k_cos_y = fputil::quick_mult(cos_y, cos_k); 137 DoubleDouble msin_k_sin_y = fputil::quick_mult(sin_y, msin_k); 138 139 DoubleDouble sin_dd = 140 fputil::exact_add<false>(sin_k_cos_y.hi, cos_k_sin_y.hi); 141 DoubleDouble cos_dd = 142 fputil::exact_add<false>(cos_k_cos_y.hi, msin_k_sin_y.hi); 143 sin_dd.lo += sin_k_cos_y.lo + cos_k_sin_y.lo; 144 cos_dd.lo += msin_k_sin_y.lo + cos_k_cos_y.lo; 145 146 #ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS 147 *sin_x = sin_dd.hi + sin_dd.lo; 148 *cos_x = cos_dd.hi + cos_dd.lo; 149 return; 150 #else 151 // Accurate test and pass for correctly rounded implementation. 152 153 double sin_lp = sin_dd.lo + err; 154 double sin_lm = sin_dd.lo - err; 155 double cos_lp = cos_dd.lo + err; 156 double cos_lm = cos_dd.lo - err; 157 158 double sin_upper = sin_dd.hi + sin_lp; 159 double sin_lower = sin_dd.hi + sin_lm; 160 double cos_upper = cos_dd.hi + cos_lp; 161 double cos_lower = cos_dd.hi + cos_lm; 162 163 // Ziv's rounding test. 164 if (LIBC_LIKELY(sin_upper == sin_lower && cos_upper == cos_lower)) { 165 *sin_x = sin_upper; 166 *cos_x = cos_upper; 167 return; 168 } 169 170 Float128 u_f128, sin_u, cos_u; 171 if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT)) 172 u_f128 = range_reduction_small_f128(x); 173 else 174 u_f128 = range_reduction_large.accurate(); 175 176 generic::sincos_eval(u_f128, sin_u, cos_u); 177 __anon36eff5f40202(unsigned kk) 178 auto get_sin_k = [](unsigned kk) -> Float128 { 179 unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63); 180 Float128 ans = SIN_K_PI_OVER_128_F128[idx]; 181 if (kk & 128) 182 ans.sign = Sign::NEG; 183 return ans; 184 }; 185 186 // cos(k * pi/128) = sin(k * pi/128 + pi/2) = sin((k + 64) * pi/128). 187 Float128 sin_k_f128 = get_sin_k(k); 188 Float128 cos_k_f128 = get_sin_k(k + 64); 189 Float128 msin_k_f128 = get_sin_k(k + 128); 190 191 // TODO: Add assertion if Ziv's accuracy tests fail in debug mode. 192 // https://github.com/llvm/llvm-project/issues/96452. 193 194 if (sin_upper == sin_lower) 195 *sin_x = sin_upper; 196 else 197 // sin(x) = sin((k * pi/128 + u) 198 // = sin(u) * cos(k*pi/128) + cos(u) * sin(k*pi/128) 199 *sin_x = static_cast<double>( 200 fputil::quick_add(fputil::quick_mul(sin_k_f128, cos_u), 201 fputil::quick_mul(cos_k_f128, sin_u))); 202 203 if (cos_upper == cos_lower) 204 *cos_x = cos_upper; 205 else 206 // cos(x) = cos((k * pi/128 + u) 207 // = cos(u) * cos(k*pi/128) - sin(u) * sin(k*pi/128) 208 *cos_x = static_cast<double>( 209 fputil::quick_add(fputil::quick_mul(cos_k_f128, cos_u), 210 fputil::quick_mul(msin_k_f128, sin_u))); 211 212 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS 213 } 214 215 } // namespace LIBC_NAMESPACE_DECL 216