NumericAL
A header-only C++20 lexer built on REAL
Loading...
Searching...
No Matches
elementary.hpp
Go to the documentation of this file.
1
8#ifndef NUMERICAL_ELEMENTARY_HPP
9#define NUMERICAL_ELEMENTARY_HPP
10
11#include <algorithm>
12#include <array>
13#include <bit>
14#include <cmath>
15#include <cstddef>
16#include <cstdint>
17#include <limits>
18
19#include "simd.hpp"
20
21#if defined(__aarch64__) && defined(__GNUC__)
22# define NUMERICAL_EXP_NEON // table_exp of two lanes in NEON (tested with #ifdef only)
23# include <arm_neon.h>
24#endif
25
26namespace numerical::detail {
27
29 inline constexpr double exp_inv_ln2_128 {0x1.71547652b82fep+7};
30
33 inline constexpr double exp_ln2_128_high {0x1.62e42fee00000p-8};
34
36 inline constexpr double exp_ln2_128_low {0x1.a39ef35793c76p-40};
37
39 inline constexpr double exp_normal_bound {-0x1.6232bdd7abcd2p+9};
40
44 constexpr std::array<double, 256> exp_table {
45 0x1.0000000000000p+0, 0x0.0p+0,
46 0x1.0163da9fb3335p+0, 0x1.b61299ab8cdb7p-54,
47 0x1.02c9a3e778061p+0, -0x1.19083535b085dp-56,
48 0x1.04315e86e7f85p+0, -0x1.0a31c1977c96ep-54,
49 0x1.059b0d3158574p+0, 0x1.d73e2a475b465p-55,
50 0x1.0706b29ddf6dep+0, -0x1.c91dfe2b13c27p-55,
51 0x1.0874518759bc8p+0, 0x1.186be4bb284ffp-57,
52 0x1.09e3ecac6f383p+0, 0x1.1487818316136p-54,
53 0x1.0b5586cf9890fp+0, 0x1.8a62e4adc610bp-54,
54 0x1.0cc922b7247f7p+0, 0x1.01edc16e24f71p-54,
55 0x1.0e3ec32d3d1a2p+0, 0x1.03a1727c57b53p-59,
56 0x1.0fb66affed31bp+0, -0x1.b9bedc44ebd7bp-57,
57 0x1.11301d0125b51p+0, -0x1.6c51039449b3ap-54,
58 0x1.12abdc06c31ccp+0, -0x1.1b514b36ca5c7p-58,
59 0x1.1429aaea92de0p+0, -0x1.32fbf9af1369ep-54,
60 0x1.15a98c8a58e51p+0, 0x1.2406ab9eeab0ap-55,
61 0x1.172b83c7d517bp+0, -0x1.19041b9d78a76p-55,
62 0x1.18af9388c8deap+0, -0x1.11023d1970f6cp-54,
63 0x1.1a35beb6fcb75p+0, 0x1.e5b4c7b4968e4p-55,
64 0x1.1bbe084045cd4p+0, -0x1.95386352ef607p-54,
65 0x1.1d4873168b9aap+0, 0x1.e016e00a2643cp-54,
66 0x1.1ed5022fcd91dp+0, -0x1.1df98027bb78cp-54,
67 0x1.2063b88628cd6p+0, 0x1.dc775814a8495p-55,
68 0x1.21f49917ddc96p+0, 0x1.2a97e9494a5eep-55,
69 0x1.2387a6e756238p+0, 0x1.9b07eb6c70573p-54,
70 0x1.251ce4fb2a63fp+0, 0x1.ac155bef4f4a4p-55,
71 0x1.26b4565e27cddp+0, 0x1.2bd339940e9d9p-55,
72 0x1.284dfe1f56381p+0, -0x1.a4c3a8c3f0d7ep-54,
73 0x1.29e9df51fdee1p+0, 0x1.612e8afad1255p-55,
74 0x1.2b87fd0dad990p+0, -0x1.10adcd6381aa4p-59,
75 0x1.2d285a6e4030bp+0, 0x1.0024754db41d5p-54,
76 0x1.2ecafa93e2f56p+0, 0x1.1ca0f45d52383p-56,
77 0x1.306fe0a31b715p+0, 0x1.6f46ad23182e4p-55,
78 0x1.32170fc4cd831p+0, 0x1.a9ce78e18047cp-55,
79 0x1.33c08b26416ffp+0, 0x1.32721843659a6p-54,
80 0x1.356c55f929ff1p+0, -0x1.b5cee5c4e4628p-55,
81 0x1.371a7373aa9cbp+0, -0x1.63aeabf42eae2p-54,
82 0x1.38cae6d05d866p+0, -0x1.e958d3c9904bdp-54,
83 0x1.3a7db34e59ff7p+0, -0x1.5e436d661f5e3p-56,
84 0x1.3c32dc313a8e5p+0, -0x1.efff8375d29c3p-54,
85 0x1.3dea64c123422p+0, 0x1.ada0911f09ebcp-55,
86 0x1.3fa4504ac801cp+0, -0x1.7d023f956f9f3p-54,
87 0x1.4160a21f72e2ap+0, -0x1.ef3691c309278p-58,
88 0x1.431f5d950a897p+0, -0x1.1c7dde35f7999p-55,
89 0x1.44e086061892dp+0, 0x1.89b7a04ef80d0p-59,
90 0x1.46a41ed1d0057p+0, 0x1.c944bd1648a76p-54,
91 0x1.486a2b5c13cd0p+0, 0x1.3c1a3b69062f0p-56,
92 0x1.4a32af0d7d3dep+0, 0x1.9cb62f3d1be56p-54,
93 0x1.4bfdad5362a27p+0, 0x1.d4397afec42e2p-56,
94 0x1.4dcb299fddd0dp+0, 0x1.8ecdbbc6a7833p-54,
95 0x1.4f9b2769d2ca7p+0, -0x1.4b309d25957e3p-54,
96 0x1.516daa2cf6642p+0, -0x1.f768569bd93efp-55,
97 0x1.5342b569d4f82p+0, -0x1.07abe1db13cadp-55,
98 0x1.551a4ca5d920fp+0, -0x1.d689cefede59bp-55,
99 0x1.56f4736b527dap+0, 0x1.9bb2c011d93adp-54,
100 0x1.58d12d497c7fdp+0, 0x1.295e15b9a1de8p-55,
101 0x1.5ab07dd485429p+0, 0x1.6324c054647adp-54,
102 0x1.5c9268a5946b7p+0, 0x1.c4b1b816986a2p-60,
103 0x1.5e76f15ad2148p+0, 0x1.ba6f93080e65ep-54,
104 0x1.605e1b976dc09p+0, -0x1.3e2429b56de47p-54,
105 0x1.6247eb03a5585p+0, -0x1.383c17e40b497p-54,
106 0x1.6434634ccc320p+0, -0x1.c483c759d8933p-55,
107 0x1.6623882552225p+0, -0x1.bb60987591c34p-54,
108 0x1.68155d44ca973p+0, 0x1.038ae44f73e65p-57,
109 0x1.6a09e667f3bcdp+0, -0x1.bdd3413b26456p-54,
110 0x1.6c012750bdabfp+0, -0x1.2895667ff0b0dp-56,
111 0x1.6dfb23c651a2fp+0, -0x1.bbe3a683c88abp-57,
112 0x1.6ff7df9519484p+0, -0x1.83c0f25860ef6p-55,
113 0x1.71f75e8ec5f74p+0, -0x1.16e4786887a99p-55,
114 0x1.73f9a48a58174p+0, -0x1.0a8d96c65d53cp-54,
115 0x1.75feb564267c9p+0, -0x1.0245957316dd3p-54,
116 0x1.780694fde5d3fp+0, 0x1.866b80a02162dp-54,
117 0x1.7a11473eb0187p+0, -0x1.41577ee04992fp-55,
118 0x1.7c1ed0130c132p+0, 0x1.f124cd1164dd6p-54,
119 0x1.7e2f336cf4e62p+0, 0x1.05d02ba15797ep-56,
120 0x1.80427543e1a12p+0, -0x1.27c86626d972bp-54,
121 0x1.82589994cce13p+0, -0x1.d4c1dd41532d8p-54,
122 0x1.8471a4623c7adp+0, -0x1.8d684a341cdfbp-55,
123 0x1.868d99b4492edp+0, -0x1.fc6f89bd4f6bap-54,
124 0x1.88ac7d98a6699p+0, 0x1.994c2f37cb53ap-54,
125 0x1.8ace5422aa0dbp+0, 0x1.6e9f156864b27p-54,
126 0x1.8cf3216b5448cp+0, -0x1.0d55e32e9e3aap-56,
127 0x1.8f1ae99157736p+0, 0x1.5cc13a2e3976cp-55,
128 0x1.9145b0b91ffc6p+0, -0x1.dd6792e582524p-54,
129 0x1.93737b0cdc5e5p+0, -0x1.75fc781b57ebcp-57,
130 0x1.95a44cbc8520fp+0, -0x1.64b7c96a5f039p-56,
131 0x1.97d829fde4e50p+0, -0x1.d185b7c1b85d1p-54,
132 0x1.9a0f170ca07bap+0, -0x1.173bd91cee632p-54,
133 0x1.9c49182a3f090p+0, 0x1.c7c46b071f2bep-56,
134 0x1.9e86319e32323p+0, 0x1.824ca78e64c6ep-56,
135 0x1.a0c667b5de565p+0, -0x1.359495d1cd533p-54,
136 0x1.a309bec4a2d33p+0, 0x1.6305c7ddc36abp-54,
137 0x1.a5503b23e255dp+0, -0x1.d2f6edb8d41e1p-54,
138 0x1.a799e1330b358p+0, 0x1.bcb7ecac563c7p-54,
139 0x1.a9e6b5579fdbfp+0, 0x1.0fac90ef7fd31p-54,
140 0x1.ac36bbfd3f37ap+0, -0x1.f9234cae76cd0p-55,
141 0x1.ae89f995ad3adp+0, 0x1.7a1cd345dcc81p-54,
142 0x1.b0e07298db666p+0, -0x1.bdef54c80e425p-54,
143 0x1.b33a2b84f15fbp+0, -0x1.2805e3084d708p-57,
144 0x1.b59728de5593ap+0, -0x1.c71dfbbba6de3p-54,
145 0x1.b7f76f2fb5e47p+0, -0x1.5584f7e54ac3bp-56,
146 0x1.ba5b030a1064ap+0, -0x1.efcd30e54292ep-54,
147 0x1.bcc1e904bc1d2p+0, 0x1.23dd07a2d9e84p-55,
148 0x1.bf2c25bd71e09p+0, -0x1.efdca3f6b9c73p-54,
149 0x1.c199bdd85529cp+0, 0x1.11065895048ddp-55,
150 0x1.c40ab5fffd07ap+0, 0x1.b4537e083c60ap-54,
151 0x1.c67f12e57d14bp+0, 0x1.2884dff483cadp-54,
152 0x1.c8f6d9406e7b5p+0, 0x1.1acbc48805c44p-56,
153 0x1.cb720dcef9069p+0, 0x1.503cbd1e949dbp-56,
154 0x1.cdf0b555dc3fap+0, -0x1.dd83b53829d72p-55,
155 0x1.d072d4a07897cp+0, -0x1.cbc3743797a9cp-54,
156 0x1.d2f87080d89f2p+0, -0x1.d487b719d8578p-54,
157 0x1.d5818dcfba487p+0, 0x1.2ed02d75b3707p-55,
158 0x1.d80e316c98398p+0, -0x1.11ec18beddfe8p-54,
159 0x1.da9e603db3285p+0, 0x1.c2300696db532p-54,
160 0x1.dd321f301b460p+0, 0x1.2da5778f018c3p-54,
161 0x1.dfc97337b9b5fp+0, -0x1.1a5cd4f184b5cp-54,
162 0x1.e264614f5a129p+0, -0x1.7b627817a1496p-54,
163 0x1.e502ee78b3ff6p+0, 0x1.39e8980a9cc8fp-55,
164 0x1.e7a51fbc74c83p+0, 0x1.2d522ca0c8de2p-54,
165 0x1.ea4afa2a490dap+0, -0x1.e9c23179c2893p-54,
166 0x1.ecf482d8e67f1p+0, -0x1.c93f3b411ad8cp-54,
167 0x1.efa1bee615a27p+0, 0x1.dc7f486a4b6b0p-54,
168 0x1.f252b376bba97p+0, 0x1.3a1a5bf0d8e43p-54,
169 0x1.f50765b6e4540p+0, 0x1.9d3e12dd8a18bp-54,
170 0x1.f7bfdad9cbe14p+0, -0x1.dbb12d006350ap-54,
171 0x1.fa7c1819e90d8p+0, 0x1.74853f3a5931ep-55,
172 0x1.fd3c22b8f71f1p+0, 0x1.2eb74966579e7p-57,
173 };
174
181 double if_true,
182 double if_false) noexcept
183 {
184 const std::uint64_t mask {std::uint64_t {0} - static_cast<std::uint64_t>(condition)};
185 return std::bit_cast<double>((std::bit_cast<std::uint64_t>(if_true) & mask) |
186 (std::bit_cast<std::uint64_t>(if_false) & ~mask));
187 }
188
195 NUMERICAL_ALWAYS_INLINE double table_exp(double x) noexcept
196 {
197 constexpr double shifter {0x1.8p52}; // n in the low bits of x·128/ln 2 + shifter, rounded to an integer
198 const double clamped {x > 710.0 ? 710.0 : x}; // past it, the result overflows all the same
199 const double z {std::fma(clamped, exp_inv_ln2_128, shifter)};
200 const auto n {std::bit_cast<std::int64_t>(z) - std::bit_cast<std::int64_t>(shifter)};
201 const double k {z - shifter};
202 const double r {std::fma(-k, exp_ln2_128_low, std::fma(-k, exp_ln2_128_high, clamped))};
203 const double r2 {r * r};
204 double p {std::fma(r, 1.0 / 720.0, 1.0 / 120.0)};
205 p = std::fma(r, p, 1.0 / 24.0);
206 p = std::fma(r, p, 1.0 / 6.0);
207 p = std::fma(r, p, 0.5);
208 p = std::fma(r2, p, r); // exp(r) − 1
209 const auto j {static_cast<std::size_t>(static_cast<std::uint64_t>(n) & 127U)}; // n mod 128
210 const double head {exp_table[2 * j]};
211 const double tail {exp_table[(2 * j) + 1]};
212 const double scaled {head + std::fma(head + tail, p, tail)};
213 // 2^e in two factors, each a normal double for e from −1023 to 1024 (the bound and the clamp): 2^1024 overflows
214 const std::int64_t e {(n - static_cast<std::int64_t>(j)) / 128}; // exact: n − j is a multiple of 128
215 const std::int64_t half {e / 2};
216 const double first {std::bit_cast<double>(static_cast<std::uint64_t>(half + 1023) << 52U)};
217 const double second {std::bit_cast<double>(static_cast<std::uint64_t>(e - half + 1023) << 52U)};
218 const double result {(scaled * first) * second};
219 return blend(std::isnan(x), x, result);
220 }
221
224 double b,
225 double& sum,
226 double& error) noexcept
227 {
228 sum = a + b;
229 const double b_part {sum - a};
230 error = (a - (sum - b_part)) + (b - b_part);
231 }
232
240 double& sum,
241 double& small) noexcept
242 {
243 constexpr double shifter {0x1.8p52};
244 const double z {std::fma(clamped, exp_inv_ln2_128, shifter)};
245 const auto n {std::bit_cast<std::int64_t>(z) - std::bit_cast<std::int64_t>(shifter)};
246 const double k {z - shifter};
247 const double r_head {std::fma(-k, exp_ln2_128_high, clamped)}; // exact, as table_exp's first step
248 const double r {std::fma(-k, exp_ln2_128_low, r_head)};
249 double q {std::fma(r, 1.0 / 5040.0, 1.0 / 720.0)};
250 q = std::fma(r, q, 1.0 / 120.0);
251 q = std::fma(r, q, 1.0 / 24.0);
252 q = std::fma(r, q, 1.0 / 6.0);
253 q = std::fma(r, q, 0.5);
254 // every product beside a sum fused explicitly, or used by no sum, so that no compiler's contraction rounds the
255 // scalar call and the vector loop two ways
256 const double square {r * r};
257 const double rest {square * q}; // exp(r) − 1 − r
258 const double p {std::fma(square, q, r)}; // exp(r) − 1
259 const double r_rest {std::fma(-k, exp_ln2_128_low, rest)}; // r − r_head + exp(r) − 1 − r
260 const auto j {static_cast<std::size_t>(static_cast<std::uint64_t>(n) & 127U)};
261 const double head {exp_table[2 * j]};
262 const double tail {exp_table[(2 * j) + 1]};
263 const std::int64_t e {(n - static_cast<std::int64_t>(j)) / 128}; // −58 to 1024
264 // 2^−e, a normal double: past 2^−1022, where it is far under an ulp of the head, 2^−1022 in its place
265 const double unit {std::bit_cast<double>(static_cast<std::uint64_t>(1023 - (e > 1022 ? 1022 : e)) << 52U)};
266 double lead {};
267 double lead_err {};
269 const double product {head * r_head};
270 const double prod_err {std::fma(head, r_head, -product)};
271 double sum_err {};
273 small = lead_err + sum_err + prod_err + std::fma(head, r_rest, std::fma(tail, p, tail));
274 return e;
275 }
276
284 NUMERICAL_ALWAYS_INLINE double table_expm1(double x) noexcept
285 {
286 double sum {};
287 double small {};
288 // both clamps by masks: GCC makes two `?:` clamps one branch, and GCC 13 even this one, under which it vectorizes
289 // no loop (expm1 of 10^6 doubles 20 ms, 3.3 by masks)
290 const double above {blend(x > 710.0, 710.0, x)};
291 const std::int64_t e {expm1_parts(blend(above < -40.0, -40.0, above), sum, small)};
292 const std::int64_t half {e / 2};
293 const double first {std::bit_cast<double>(static_cast<std::uint64_t>(half + 1023) << 52U)};
294 const double second {std::bit_cast<double>(static_cast<std::uint64_t>(e - half + 1023) << 52U)};
295 const double result {((sum + small) * first) * second};
296 return blend(std::fabs(x) >= 0x1p-54, result, x); // NaN fails the comparison
297 }
298
305 NUMERICAL_ALWAYS_INLINE double table_tanh(double x) noexcept
306 {
307 const double magnitude {std::fabs(x)};
308 double sum {};
309 double small {};
310 // the clamp by a mask: GCC 13 makes a `?:` here a branch, under which it vectorizes no loop (12.5 ms, 4.4)
311 const std::int64_t e {expm1_parts(2.0 * blend(magnitude > 22.0, 22.0, magnitude), sum, small)}; // 0 to 64
312 double high {};
313 double low {};
314 two_sum(sum, small, high, low);
315 const double scale {std::bit_cast<double>(static_cast<std::uint64_t>(e + 1023) << 52U)};
316 const double t {high * scale}; // t's tail low · scale, exact
317 double d_high {};
318 double d_err {};
319 two_sum(2.0, t, d_high, d_err);
320 const double d_low {std::fma(low, scale, d_err)};
321 const double inverse {1.0 / d_high};
322 const double quotient {t * inverse};
323 const double residual {std::fma(-quotient, d_low, std::fma(low, scale, std::fma(-quotient, d_high, t)))};
324 const double result {std::fma(residual, inverse, quotient)};
325 return blend(magnitude >= 0x1p-27, std::copysign(result, x), x); // NaN fails the comparison
326 }
327
336 double& d_high,
337 double& d_low,
338 double& u_high,
339 double& u_low) noexcept
340 {
341 double sum {};
342 double small {};
343 const std::int64_t e {expm1_parts(magnitude > 711.0 ? 711.0 : magnitude, sum, small)}; // 0 to 1026
344 double high {};
345 double low {};
346 two_sum(sum, small, high, low);
347 const std::int64_t kept {std::min<std::int64_t>(e, 64)};
348 const double scale {std::bit_cast<double>(static_cast<std::uint64_t>(kept + 1023) << 52U)};
349 double d_err {};
350 two_sum(1.0, high * scale, d_high, d_err); // high · scale and low · scale exact
351 d_low = std::fma(low, scale, d_err);
352 u_high = 1.0 / d_high;
353 u_low = std::fma(-u_high, d_low, std::fma(-u_high, d_high, 1.0)) * u_high;
354 return std::bit_cast<double>(static_cast<std::uint64_t>(e - kept + 1022) << 52U); // 2^(e − kept − 1)
355 }
356
362 NUMERICAL_ALWAYS_INLINE double table_sinh(double x) noexcept
363 {
364 const double magnitude {std::fabs(x)};
365 double d_high {};
366 double d_low {};
367 double u_high {};
368 double u_low {};
369 const double power {hyperbolic_parts(magnitude, d_high, d_low, u_high, u_low)};
370 double lead {};
371 double lead_err {};
373 const double result {(lead + ((lead_err + d_low) - u_low)) * power};
374 return blend(magnitude >= 0x1p-27, std::copysign(result, x), x); // NaN fails the comparison
375 }
376
381 NUMERICAL_ALWAYS_INLINE double table_cosh(double x) noexcept
382 {
383 double d_high {};
384 double d_low {};
385 double u_high {};
386 double u_low {};
387 const double power {hyperbolic_parts(std::fabs(x), d_high, d_low, u_high, u_low)};
388 double lead {};
389 double lead_err {};
391 const double result {(lead + ((lead_err + d_low) + u_low)) * power};
392 return blend(std::isnan(x), x, result);
393 }
394
396 inline constexpr double exp2_ln2 {0x1.62e42fefa39efp-1};
397
404 NUMERICAL_ALWAYS_INLINE double table_exp2(double x) noexcept
405 {
406 constexpr double shifter {0x1.8p52};
407 const double clamped {x > 1025.0 ? 1025.0 : x}; // past 1024, the result overflows all the same
408 const double z {std::fma(clamped, 128.0, shifter)};
409 const auto n {std::bit_cast<std::int64_t>(z) - std::bit_cast<std::int64_t>(shifter)};
410 const double k {z - shifter};
411 const double r {std::fma(k, -0x1p-7, clamped) * exp2_ln2}; // f exact, f · ln 2 rounded once
412 const double r2 {r * r};
413 double p {std::fma(r, 1.0 / 720.0, 1.0 / 120.0)};
414 p = std::fma(r, p, 1.0 / 24.0);
415 p = std::fma(r, p, 1.0 / 6.0);
416 p = std::fma(r, p, 0.5);
417 p = std::fma(r2, p, r);
418 const auto j {static_cast<std::size_t>(static_cast<std::uint64_t>(n) & 127U)};
419 const double head {exp_table[2 * j]};
420 const double tail {exp_table[(2 * j) + 1]};
421 const double scaled {head + std::fma(head + tail, p, tail)};
422 const std::int64_t e {(n - static_cast<std::int64_t>(j)) / 128};
423 const std::int64_t half {e / 2};
424 const double first {std::bit_cast<double>(static_cast<std::uint64_t>(half + 1023) << 52U)};
425 const double second {std::bit_cast<double>(static_cast<std::uint64_t>(e - half + 1023) << 52U)};
426 const double result {(scaled * first) * second};
427 return blend(std::isnan(x), x, result);
428 }
429
431 inline constexpr double atan_half_pi_high {0x1.921fb54442d18p+0};
432
434 inline constexpr double atan_half_pi_low {0x1.1a62633145c07p-54};
435
438 constexpr std::array<double, 130> atan_table {
439 0.0, 0.0,
440 0x1.fff555bbb729bp-7, -0x1.220c39d4dff50p-61,
441 0x1.ffd55bba97625p-6, -0x1.5ec431444912cp-60,
442 0x1.7fb818430da2ap-5, -0x1.86ef8f794f105p-63,
443 0x1.ff55bb72cfdeap-5, -0x1.c934d86d23f1dp-60,
444 0x1.3f59f0e7c559dp-4, 0x1.ac4ce285df847p-58,
445 0x1.7ee182602f10fp-4, -0x1.cfb654c0c3d98p-58,
446 0x1.be39ebe6f07c3p-4, 0x1.f7b8f29a05987p-58,
447 0x1.fd5ba9aac2f6ep-4, -0x1.cd37686760c17p-59,
448 0x1.1e1fafb043727p-3, -0x1.b485914dacf8cp-59,
449 0x1.3d6eee8c6626cp-3, 0x1.61a3b0ce9281bp-57,
450 0x1.5c9811e3ec26ap-3, -0x1.054ab2c010f3dp-58,
451 0x1.7b97b4bce5b02p-3, 0x1.347b0b4f881cap-58,
452 0x1.9a6a8e96c8626p-3, 0x1.cf601e7b4348ep-59,
453 0x1.b90d7529260a2p-3, 0x1.17b10d2e0e5abp-61,
454 0x1.d77d5df205736p-3, 0x1.c648d1534597ep-57,
455 0x1.f5b75f92c80ddp-3, 0x1.8ab6e3cf7afbdp-57,
456 0x1.09dc597d86362p-2, 0x1.62e47390cb865p-56,
457 0x1.18bf5a30bf178p-2, 0x1.30ca4748b1bf9p-57,
458 0x1.278372057ef46p-2, -0x1.077cdd36dfc81p-56,
459 0x1.362773707ebccp-2, -0x1.963a544b672d8p-57,
460 0x1.44aa436c2af0ap-2, -0x1.5d5e43c55b3bap-56,
461 0x1.530ad9951cd4ap-2, -0x1.2566480884082p-57,
462 0x1.614840309cfe2p-2, -0x1.a725715711f00p-56,
463 0x1.6f61941e4def1p-2, -0x1.c63aae6f6e918p-56,
464 0x1.7d5604b63b3f7p-2, 0x1.69c885c2b249ap-56,
465 0x1.8b24d394a1b25p-2, 0x1.b6d0ba3748fa8p-56,
466 0x1.98cd5454d6b18p-2, 0x1.9e6c988fd0a77p-56,
467 0x1.a64eec3cc23fdp-2, -0x1.24dec1b50b7ffp-56,
468 0x1.b3a911da65c6cp-2, 0x1.ae187b1ca5040p-56,
469 0x1.c0db4c94ec9f0p-2, -0x1.cc1ce70934c34p-56,
470 0x1.cde53432c1351p-2, -0x1.a2cfa4418f1adp-56,
471 0x1.dac670561bb4fp-2, 0x1.a2b7f222f65e2p-56,
472 0x1.e77eb7f175a34p-2, 0x1.0e53dc1bf3435p-56,
473 0x1.f40dd0b541418p-2, -0x1.a3992dc382a23p-57,
474 0x1.0039c73c1a40cp-1, -0x1.b32c949c9d593p-55,
475 0x1.0657e94db30d0p-1, -0x1.d5b495f6349e6p-56,
476 0x1.0c6145b5b43dap-1, 0x1.974fa13b5404fp-58,
477 0x1.1255d9bfbd2a9p-1, -0x1.2bdaee1c0ee35p-58,
478 0x1.1835a88be7c13p-1, 0x1.c621cec00c301p-55,
479 0x1.1e00babdefeb4p-1, -0x1.928df287a668fp-58,
480 0x1.23b71e2cc9e6ap-1, 0x1.c421c9f38224ep-57,
481 0x1.2958e59308e31p-1, -0x1.09e73b0c6c087p-56,
482 0x1.2ee628406cbcap-1, 0x1.c5d5e9ff0cf8dp-55,
483 0x1.345f01cce37bbp-1, 0x1.1021137c71102p-55,
484 0x1.39c391cd4171ap-1, -0x1.2304331d8bf46p-55,
485 0x1.3f13fb89e96f4p-1, 0x1.ecf8b492644f0p-56,
486 0x1.445065b795b56p-1, -0x1.f76d0163f79c8p-56,
487 0x1.4978fa3269ee1p-1, 0x1.2419a87f2a458p-56,
488 0x1.4e8de5bb6ec04p-1, 0x1.4a33dbeb3796cp-55,
489 0x1.538f57b89061fp-1, -0x1.1bb74abda520cp-55,
490 0x1.587d81f732fbbp-1, -0x1.5e5c9d8c5a950p-56,
491 0x1.5d58987169b18p-1, 0x1.0028e4bc5e7cap-57,
492 0x1.6220d115d7b8ep-1, -0x1.2b785350ee8c1p-57,
493 0x1.66d663923e087p-1, -0x1.6ea6febe8bbbap-56,
494 0x1.6b798920b3d99p-1, -0x1.a80386188c50ep-55,
495 0x1.700a7c5784634p-1, -0x1.8c34d25aadef6p-56,
496 0x1.748978fba8e0fp-1, 0x1.7b2a6165884a1p-59,
497 0x1.78f6bbd5d315ep-1, 0x1.406a089803740p-55,
498 0x1.7d528289fa093p-1, 0x1.560821e2f3aa9p-55,
499 0x1.819d0b7158a4dp-1, -0x1.bf76229d3b917p-56,
500 0x1.85d69576cc2c5p-1, 0x1.6b66e7fc8b8c3p-57,
501 0x1.89ff5ff57f1f8p-1, -0x1.55b9a5e177a1bp-55,
502 0x1.8e17aa99cc05ep-1, -0x1.ec182ab042f61p-56,
503 0x1.921fb54442d18p-1, 0x1.1a62633145c07p-55,
504 };
505
508 inline constexpr std::array<double, 3> half_pi_parts {0x1.921fb54442d18p+0, 0x1.1a62633145c07p-54, -0x1.f1976b7ed8fbcp-110};
509
511 inline constexpr double sine_reduction_bound {0x1p20};
512
522 template <bool Cosine>
523 NUMERICAL_ALWAYS_INLINE double reduced_sine(double x) noexcept
524 {
525 constexpr double shifter {0x1.8p52};
526 const double shifted {std::fma(x, 0x1.45f306dc9c883p-1, shifter)}; // x·2/π rounded to an integer
527 const double k {shifted - shifter};
528 // the integer's low bits, k's two's complement: cos x = sin(x + π/2) is a quadrant on
529 const std::uint64_t quadrant {(std::bit_cast<std::uint64_t>(shifted) + static_cast<std::uint64_t>(Cosine)) & 3U};
530 double r {std::fma(-k, half_pi_parts[0], x)};
531 r = std::fma(-k, half_pi_parts[1], r);
532 r = std::fma(-k, half_pi_parts[2], r);
533 const double z {r * r};
534 const double odd {std::fma(z, std::fma(z, std::fma(z, std::fma(z, std::fma(z, 0x1.5d93a5acfd57cp-33, -0x1.ae5e68a2b9cebp-26), 0x1.71de357b1fe7dp-19), -0x1.a01a019c161d5p-13), 0x1.111111110f8a6p-7), -0x1.5555555555549p-3)};
535 const double even {std::fma(z, std::fma(z, std::fma(z, std::fma(z, std::fma(z, -0x1.8fae9be8838d4p-37, 0x1.1ee9ebdb4b1c4p-29), -0x1.27e4f809c52adp-22), 0x1.a01a019cb159p-16), -0x1.6c16c16c15177p-10), 0x1.555555555554cp-5)};
536 const double sine {blend(x == 0.0, x, std::fma(r * z, odd, r))}; // ±0 itself: −0 times π/2's negative part is +0
537 const double cosine {std::fma(z * z, even, std::fma(-0.5, z, 1.0))};
538 const double value {blend((quadrant & 1U) != 0U, cosine, sine)};
539 return blend((quadrant & 2U) != 0U, -value, value);
540 }
541
543 inline constexpr double sine_inverse_step {0x1.45f306dc9c883p+3};
544
546 inline constexpr std::array<double, 3> sine_step_parts {0x1.921fb54442d18p-4, 0x1.1a62633145c07p-58, -0x1.f1976b7ed8fbcp-114};
547
550 constexpr std::array<double, 256> sine_table {
551 0x0p+0, 0x0p+0, 0x1.0000000000000p+0, 0x0p+0,
552 0x1.917a6bc29b42cp-4, -0x1.e2718d26ed688p-60, 0x1.fd88da3d12526p-1, -0x1.87df6378811c7p-55,
553 0x1.8f8b83c69a60bp-3, -0x1.26d19b9ff8d82p-57, 0x1.f6297cff75cb0p-1, 0x1.562172a361fd3p-56,
554 0x1.294062ed59f06p-2, -0x1.5d28da2c4612dp-56, 0x1.e9f4156c62ddap-1, 0x1.760b1e2e3f81ep-55,
555 0x1.87de2a6aea963p-2, -0x1.72cedd3d5a610p-57, 0x1.d906bcf328d46p-1, 0x1.457e610231ac2p-56,
556 0x1.e2b5d3806f63bp-2, 0x1.e0d891d3c6841p-58, 0x1.c38b2f180bdb1p-1, -0x1.6e0b1757c8d07p-56,
557 0x1.1c73b39ae68c8p-1, 0x1.b25dd267f6600p-55, 0x1.a9b66290ea1a3p-1, 0x1.9f630e8b6dac8p-60,
558 0x1.44cf325091dd6p-1, 0x1.8076a2cfdc6b3p-57, 0x1.8bc806b151741p-1, -0x1.2c5e12ed1336dp-55,
559 0x1.6a09e667f3bcdp-1, -0x1.bdd3413b26456p-55, 0x1.6a09e667f3bcdp-1, -0x1.bdd3413b26456p-55,
560 0x1.8bc806b151741p-1, -0x1.2c5e12ed1336dp-55, 0x1.44cf325091dd6p-1, 0x1.8076a2cfdc6b3p-57,
561 0x1.a9b66290ea1a3p-1, 0x1.9f630e8b6dac8p-60, 0x1.1c73b39ae68c8p-1, 0x1.b25dd267f6600p-55,
562 0x1.c38b2f180bdb1p-1, -0x1.6e0b1757c8d07p-56, 0x1.e2b5d3806f63bp-2, 0x1.e0d891d3c6841p-58,
563 0x1.d906bcf328d46p-1, 0x1.457e610231ac2p-56, 0x1.87de2a6aea963p-2, -0x1.72cedd3d5a610p-57,
564 0x1.e9f4156c62ddap-1, 0x1.760b1e2e3f81ep-55, 0x1.294062ed59f06p-2, -0x1.5d28da2c4612dp-56,
565 0x1.f6297cff75cb0p-1, 0x1.562172a361fd3p-56, 0x1.8f8b83c69a60bp-3, -0x1.26d19b9ff8d82p-57,
566 0x1.fd88da3d12526p-1, -0x1.87df6378811c7p-55, 0x1.917a6bc29b42cp-4, -0x1.e2718d26ed688p-60,
567 0x1.0000000000000p+0, 0x0p+0, 0x0p+0, 0x0p+0,
568 0x1.fd88da3d12526p-1, -0x1.87df6378811c7p-55, -0x1.917a6bc29b42cp-4, 0x1.e2718d26ed688p-60,
569 0x1.f6297cff75cb0p-1, 0x1.562172a361fd3p-56, -0x1.8f8b83c69a60bp-3, 0x1.26d19b9ff8d82p-57,
570 0x1.e9f4156c62ddap-1, 0x1.760b1e2e3f81ep-55, -0x1.294062ed59f06p-2, 0x1.5d28da2c4612dp-56,
571 0x1.d906bcf328d46p-1, 0x1.457e610231ac2p-56, -0x1.87de2a6aea963p-2, 0x1.72cedd3d5a610p-57,
572 0x1.c38b2f180bdb1p-1, -0x1.6e0b1757c8d07p-56, -0x1.e2b5d3806f63bp-2, -0x1.e0d891d3c6841p-58,
573 0x1.a9b66290ea1a3p-1, 0x1.9f630e8b6dac8p-60, -0x1.1c73b39ae68c8p-1, -0x1.b25dd267f6600p-55,
574 0x1.8bc806b151741p-1, -0x1.2c5e12ed1336dp-55, -0x1.44cf325091dd6p-1, -0x1.8076a2cfdc6b3p-57,
575 0x1.6a09e667f3bcdp-1, -0x1.bdd3413b26456p-55, -0x1.6a09e667f3bcdp-1, 0x1.bdd3413b26456p-55,
576 0x1.44cf325091dd6p-1, 0x1.8076a2cfdc6b3p-57, -0x1.8bc806b151741p-1, 0x1.2c5e12ed1336dp-55,
577 0x1.1c73b39ae68c8p-1, 0x1.b25dd267f6600p-55, -0x1.a9b66290ea1a3p-1, -0x1.9f630e8b6dac8p-60,
578 0x1.e2b5d3806f63bp-2, 0x1.e0d891d3c6841p-58, -0x1.c38b2f180bdb1p-1, 0x1.6e0b1757c8d07p-56,
579 0x1.87de2a6aea963p-2, -0x1.72cedd3d5a610p-57, -0x1.d906bcf328d46p-1, -0x1.457e610231ac2p-56,
580 0x1.294062ed59f06p-2, -0x1.5d28da2c4612dp-56, -0x1.e9f4156c62ddap-1, -0x1.760b1e2e3f81ep-55,
581 0x1.8f8b83c69a60bp-3, -0x1.26d19b9ff8d82p-57, -0x1.f6297cff75cb0p-1, -0x1.562172a361fd3p-56,
582 0x1.917a6bc29b42cp-4, -0x1.e2718d26ed688p-60, -0x1.fd88da3d12526p-1, 0x1.87df6378811c7p-55,
583 0x0p+0, 0x0p+0, -0x1.0000000000000p+0, 0x0p+0,
584 -0x1.917a6bc29b42cp-4, 0x1.e2718d26ed688p-60, -0x1.fd88da3d12526p-1, 0x1.87df6378811c7p-55,
585 -0x1.8f8b83c69a60bp-3, 0x1.26d19b9ff8d82p-57, -0x1.f6297cff75cb0p-1, -0x1.562172a361fd3p-56,
586 -0x1.294062ed59f06p-2, 0x1.5d28da2c4612dp-56, -0x1.e9f4156c62ddap-1, -0x1.760b1e2e3f81ep-55,
587 -0x1.87de2a6aea963p-2, 0x1.72cedd3d5a610p-57, -0x1.d906bcf328d46p-1, -0x1.457e610231ac2p-56,
588 -0x1.e2b5d3806f63bp-2, -0x1.e0d891d3c6841p-58, -0x1.c38b2f180bdb1p-1, 0x1.6e0b1757c8d07p-56,
589 -0x1.1c73b39ae68c8p-1, -0x1.b25dd267f6600p-55, -0x1.a9b66290ea1a3p-1, -0x1.9f630e8b6dac8p-60,
590 -0x1.44cf325091dd6p-1, -0x1.8076a2cfdc6b3p-57, -0x1.8bc806b151741p-1, 0x1.2c5e12ed1336dp-55,
591 -0x1.6a09e667f3bcdp-1, 0x1.bdd3413b26456p-55, -0x1.6a09e667f3bcdp-1, 0x1.bdd3413b26456p-55,
592 -0x1.8bc806b151741p-1, 0x1.2c5e12ed1336dp-55, -0x1.44cf325091dd6p-1, -0x1.8076a2cfdc6b3p-57,
593 -0x1.a9b66290ea1a3p-1, -0x1.9f630e8b6dac8p-60, -0x1.1c73b39ae68c8p-1, -0x1.b25dd267f6600p-55,
594 -0x1.c38b2f180bdb1p-1, 0x1.6e0b1757c8d07p-56, -0x1.e2b5d3806f63bp-2, -0x1.e0d891d3c6841p-58,
595 -0x1.d906bcf328d46p-1, -0x1.457e610231ac2p-56, -0x1.87de2a6aea963p-2, 0x1.72cedd3d5a610p-57,
596 -0x1.e9f4156c62ddap-1, -0x1.760b1e2e3f81ep-55, -0x1.294062ed59f06p-2, 0x1.5d28da2c4612dp-56,
597 -0x1.f6297cff75cb0p-1, -0x1.562172a361fd3p-56, -0x1.8f8b83c69a60bp-3, 0x1.26d19b9ff8d82p-57,
598 -0x1.fd88da3d12526p-1, 0x1.87df6378811c7p-55, -0x1.917a6bc29b42cp-4, 0x1.e2718d26ed688p-60,
599 -0x1.0000000000000p+0, 0x0p+0, 0x0p+0, 0x0p+0,
600 -0x1.fd88da3d12526p-1, 0x1.87df6378811c7p-55, 0x1.917a6bc29b42cp-4, -0x1.e2718d26ed688p-60,
601 -0x1.f6297cff75cb0p-1, -0x1.562172a361fd3p-56, 0x1.8f8b83c69a60bp-3, -0x1.26d19b9ff8d82p-57,
602 -0x1.e9f4156c62ddap-1, -0x1.760b1e2e3f81ep-55, 0x1.294062ed59f06p-2, -0x1.5d28da2c4612dp-56,
603 -0x1.d906bcf328d46p-1, -0x1.457e610231ac2p-56, 0x1.87de2a6aea963p-2, -0x1.72cedd3d5a610p-57,
604 -0x1.c38b2f180bdb1p-1, 0x1.6e0b1757c8d07p-56, 0x1.e2b5d3806f63bp-2, 0x1.e0d891d3c6841p-58,
605 -0x1.a9b66290ea1a3p-1, -0x1.9f630e8b6dac8p-60, 0x1.1c73b39ae68c8p-1, 0x1.b25dd267f6600p-55,
606 -0x1.8bc806b151741p-1, 0x1.2c5e12ed1336dp-55, 0x1.44cf325091dd6p-1, 0x1.8076a2cfdc6b3p-57,
607 -0x1.6a09e667f3bcdp-1, 0x1.bdd3413b26456p-55, 0x1.6a09e667f3bcdp-1, -0x1.bdd3413b26456p-55,
608 -0x1.44cf325091dd6p-1, -0x1.8076a2cfdc6b3p-57, 0x1.8bc806b151741p-1, -0x1.2c5e12ed1336dp-55,
609 -0x1.1c73b39ae68c8p-1, -0x1.b25dd267f6600p-55, 0x1.a9b66290ea1a3p-1, 0x1.9f630e8b6dac8p-60,
610 -0x1.e2b5d3806f63bp-2, -0x1.e0d891d3c6841p-58, 0x1.c38b2f180bdb1p-1, -0x1.6e0b1757c8d07p-56,
611 -0x1.87de2a6aea963p-2, 0x1.72cedd3d5a610p-57, 0x1.d906bcf328d46p-1, 0x1.457e610231ac2p-56,
612 -0x1.294062ed59f06p-2, 0x1.5d28da2c4612dp-56, 0x1.e9f4156c62ddap-1, 0x1.760b1e2e3f81ep-55,
613 -0x1.8f8b83c69a60bp-3, 0x1.26d19b9ff8d82p-57, 0x1.f6297cff75cb0p-1, 0x1.562172a361fd3p-56,
614 -0x1.917a6bc29b42cp-4, 0x1.e2718d26ed688p-60, 0x1.fd88da3d12526p-1, -0x1.87df6378811c7p-55,
615 };
616
623 {
624 double a;
625 double a_tail;
626 double b;
627 double b_tail;
628 double tiny;
629 };
630
635 template <bool Cosine>
637 double x) noexcept;
638
640 template <>
642 double x) noexcept
643 {
644 return {sine_table[entry], sine_table[entry + 1], sine_table[entry + 2], sine_table[entry + 3], x};
645 }
646
648 template <>
650 double /*x*/) noexcept
651 {
652 return {sine_table[entry + 2], sine_table[entry + 3], -sine_table[entry], -sine_table[entry + 1], 1.0};
653 }
654
666 template <bool Cosine>
667 NUMERICAL_ALWAYS_INLINE double table_sine(double x) noexcept
668 {
669 constexpr double shifter {0x1.8p52};
670 const double shifted {std::fma(x, sine_inverse_step, shifter)}; // x·32/π rounded to an integer
671 const double j {shifted - shifter};
672 const std::uint64_t entry {4U * (std::bit_cast<std::uint64_t>(shifted) & 63U)}; // j's two's complement, mod 64
673 const double r {std::fma(-j, sine_step_parts[0], x)};
674 const double t {-j * sine_step_parts[1]};
675 const double t_error {std::fma(-j, sine_step_parts[1], -t)};
676 double r_high {0.0};
677 double r_error {0.0};
678 two_sum(r, t, r_high, r_error);
679 const double r_low {std::fma(-j, sine_step_parts[2], r_error + t_error)};
680 const double z {r_high * r_high};
681 const double odd {std::fma(z, std::fma(z, std::fma(z, std::fma(z, -0x1.ae64567f544e4p-26, 0x1.71de3a556c734p-19), -0x1.a01a01a01a01ap-13), 0x1.1111111111111p-7), -0x1.5555555555555p-3)};
682 const double even {std::fma(z, std::fma(z, std::fma(z, std::fma(z, -0x1.27e4fb7789f5cp-22, 0x1.a01a01a01a01ap-16), -0x1.6c16c16c16c17p-10), 0x1.5555555555555p-5), -0.5)};
683 const double sin_rest {std::fma(r_high * z, odd, r_low)}; // sin r − r_high
684 const double cos_rest {std::fma(z, even, -(r_high * r_low))}; // cos r − 1
685 const sine_terms terms {sine_terms_of<Cosine>(entry, x)};
686 const double a {terms.a};
687 const double b {terms.b};
688 const double p {b * r_high};
689 const double p_error {std::fma(b, r_high, -p)};
690 double head {0.0};
691 double error {0.0};
692 two_sum(a, p, head, error);
693 double small {error + p_error};
694 small = small + terms.a_tail;
695 small = std::fma(terms.b_tail, r_high, small);
696 small = std::fma(b, sin_rest, small);
697 small = std::fma(a, cos_rest, small);
698 const double value {head + small};
699 return blend(std::fabs(x) < 0x1p-27, terms.tiny, value);
700 }
701
709 NUMERICAL_ALWAYS_INLINE double table_atan(double x) noexcept
710 {
711 constexpr double shifter {0x1.8p52};
712 const double magnitude {std::fabs(x)};
713 const bool far {magnitude > 1.0};
714 const double inverse {1.0 / magnitude};
715 const double inverse_lo {std::fma(-inverse, magnitude, 1.0) * inverse}; // 1/|x| − inverse, to first order
716 const double y {blend(far, inverse, magnitude <= 1.0 ? magnitude : 0.0)}; // NaN as 0: k in the table
717 const double y_low {blend(far, inverse_lo, 0.0)};
718 const double z {std::fma(y, 64.0, shifter)};
719 const auto k {static_cast<std::size_t>(std::bit_cast<std::int64_t>(z) - std::bit_cast<std::int64_t>(shifter))};
720 const double c {(z - shifter) * 0x1p-6}; // k, exact, without a conversion AVX2 has no vector form of
721 const double difference {y - c}; // exact: y within 2^−7 of c, or c 0
722 const double num {difference + y_low};
723 const double num_low {(difference - num) + y_low};
724 const double den {std::fma(y, c, 1.0)};
725 const double den_low {std::fma(y_low, c, std::fma(y, c, 1.0 - den))};
726 const double t {num / den};
727 const double t_low {(std::fma(-t, den_low, std::fma(-t, den, num)) + num_low) / den};
728 const double square {t * t};
729 double q {std::fma(square, -1.0 / 11.0, 1.0 / 9.0)};
730 q = std::fma(square, q, -1.0 / 7.0);
731 q = std::fma(square, q, 1.0 / 5.0);
732 q = std::fma(square, q, -1.0 / 3.0);
733 const double head {atan_table[2 * k]};
734 const double tail {atan_table[(2 * k) + 1]};
735 double sum {};
736 double error {};
737 two_sum(head, t, sum, error);
738 const double small {std::fma(t * square, q, (error + tail) + t_low)}; // with atan(t) − t
739 double rest {};
740 double rest_e {};
742 const double near {sum + small};
743 const double beyond {rest + ((rest_e + atan_half_pi_low) - small)};
744 const double result {blend(magnitude > 0x1p54, atan_half_pi_high, blend(far, beyond, near))};
745 return blend(magnitude >= 0x1p-27, std::copysign(result, x), x); // NaN fails the comparison
746 }
747
750 inline constexpr double log_ln2_high {0x1.62e42fefa3800p-1};
751
753 inline constexpr double log_ln2_low {0x1.ef35793c76730p-45};
754
756 constexpr std::array<double, 91> log_inverse {
757 0x1.6c16c16c16c17p+0, 0x1.6816816816817p+0, 0x1.642c8590b2164p+0,
758 0x1.6058160581606p+0, 0x1.5c9882b931057p+0, 0x1.58ed2308158edp+0,
759 0x1.5555555555555p+0, 0x1.51d07eae2f815p+0, 0x1.4e5e0a72f0539p+0,
760 0x1.4afd6a052bf5bp+0, 0x1.47ae147ae147bp+0, 0x1.446f86562d9fbp+0,
761 0x1.4141414141414p+0, 0x1.3e22cbce4a902p+0, 0x1.3b13b13b13b14p+0,
762 0x1.3813813813814p+0, 0x1.3521cfb2b78c1p+0, 0x1.323e34a2b10bfp+0,
763 0x1.2f684bda12f68p+0, 0x1.2c9fb4d812ca0p+0, 0x1.29e4129e4129ep+0,
764 0x1.27350b8812735p+0, 0x1.2492492492492p+0, 0x1.21fb78121fb78p+0,
765 0x1.1f7047dc11f70p+0, 0x1.1cf06ada2811dp+0, 0x1.1a7b9611a7b96p+0,
766 0x1.1811811811812p+0, 0x1.15b1e5f75270dp+0, 0x1.135c81135c811p+0,
767 0x1.1111111111111p+0, 0x1.0ecf56be69c90p+0, 0x1.0c9714fbcda3bp+0,
768 0x1.0a6810a6810a7p+0, 0x1.0842108421084p+0, 0x1.0624dd2f1a9fcp+0,
769 0x1.0410410410410p+0, 0x1.0204081020408p+0, 0x1.0000000000000p+0,
770 0x1.fc07f01fc07f0p-1, 0x1.f81f81f81f820p-1, 0x1.f44659e4a4271p-1,
771 0x1.f07c1f07c1f08p-1, 0x1.ecc07b301ecc0p-1, 0x1.e9131abf0b767p-1,
772 0x1.e573ac901e574p-1, 0x1.e1e1e1e1e1e1ep-1, 0x1.de5d6e3f8868ap-1,
773 0x1.dae6076b981dbp-1, 0x1.d77b654b82c34p-1, 0x1.d41d41d41d41dp-1,
774 0x1.d0cb58f6ec074p-1, 0x1.cd85689039b0bp-1, 0x1.ca4b3055ee191p-1,
775 0x1.c71c71c71c71cp-1, 0x1.c3f8f01c3f8f0p-1, 0x1.c0e070381c0e0p-1,
776 0x1.bdd2b899406f7p-1, 0x1.bacf914c1bad0p-1, 0x1.b7d6c3dda338bp-1,
777 0x1.b4e81b4e81b4fp-1, 0x1.b2036406c80d9p-1, 0x1.af286bca1af28p-1,
778 0x1.ac5701ac5701bp-1, 0x1.a98ef606a63bep-1, 0x1.a6d01a6d01a6dp-1,
779 0x1.a41a41a41a41ap-1, 0x1.a16d3f97a4b02p-1, 0x1.9ec8e951033d9p-1,
780 0x1.9c2d14ee4a102p-1, 0x1.999999999999ap-1, 0x1.970e4f80cb872p-1,
781 0x1.948b0fcd6e9e0p-1, 0x1.920fb49d0e229p-1, 0x1.8f9c18f9c18fap-1,
782 0x1.8d3018d3018d3p-1, 0x1.8acb90f6bf3aap-1, 0x1.886e5f0abb04ap-1,
783 0x1.8618618618618p-1, 0x1.83c977ab2beddp-1, 0x1.8181818181818p-1,
784 0x1.7f405fd017f40p-1, 0x1.7d05f417d05f4p-1, 0x1.7ad2208e0ecc3p-1,
785 0x1.78a4c8178a4c8p-1, 0x1.767dce434a9b1p-1, 0x1.745d1745d1746p-1,
786 0x1.724287f46debcp-1, 0x1.702e05c0b8170p-1, 0x1.6e1f76b4337c7p-1,
787 0x1.6c16c16c16c17p-1,
788 };
789
793 constexpr std::array<double, 182> log_table {
794 -0x1.68ac83e9c7000p-2, 0x1.7acd66c548a30p-44,
795 -0x1.5d5bddf596000p-2, 0x1.9de2a08a465dcp-47,
796 -0x1.522ae0738a000p-2, -0x1.eba708164c759p-45,
797 -0x1.4718dc271c000p-2, -0x1.071d8fb4c14c5p-44,
798 -0x1.3c25277333000p-2, -0x1.83454b606bd5cp-46,
799 -0x1.314f1e1d36000p-2, 0x1.8e5bad3213cb8p-45,
800 -0x1.269621134e000p-2, 0x1.1ba1f10522625p-44,
801 -0x1.1bf99635a7000p-2, 0x1.1ade9575c2125p-44,
802 -0x1.1178e8227e000p-2, -0x1.1e9b8ce2d07f2p-44,
803 -0x1.07138604d6000p-2, 0x1.e70124e912b17p-44,
804 -0x1.f991c6cb3c000p-3, 0x1.90b84cd7cc834p-44,
805 -0x1.e530effe72000p-3, 0x1.fdafbb13f7c18p-44,
806 -0x1.d1037f2656000p-3, 0x1.8527e75b6f6e4p-47,
807 -0x1.bd087383be000p-3, 0x1.d5844595412b6p-45,
808 -0x1.a93ed3c8ae000p-3, 0x1.86a4350562169p-45,
809 -0x1.95a5adcf70000p-3, -0x1.8262858a0ff6fp-47,
810 -0x1.823c16551a000p-3, -0x1.e02db9a631e83p-46,
811 -0x1.6f0128b756000p-3, -0x1.571d90d31ef0fp-44,
812 -0x1.5bf406b544000p-3, 0x1.28023eb68981cp-46,
813 -0x1.4913d8333c000p-3, 0x1.53a43558124c4p-44,
814 -0x1.365fcb015a000p-3, 0x1.fd720afb9691bp-44,
815 -0x1.23d712a49c000p-3, -0x1.00aa38fd3df5cp-46,
816 -0x1.1178e8227e000p-3, -0x1.1e778ce2d07f2p-45,
817 -0x1.fe89139dbc000p-4, -0x1.56494d82f7a82p-44,
818 -0x1.da72763844000p-4, -0x1.a79401fa71733p-46,
819 -0x1.b6ac88dad4000p-4, -0x1.b1cbff50225c7p-44,
820 -0x1.9335e5d594000p-4, -0x1.30f5c3abd47dap-45,
821 -0x1.700d30aeac000p-4, -0x1.d068da99ded32p-49,
822 -0x1.4d3115d208000p-4, 0x1.53e2582f4e1efp-48,
823 -0x1.2aa04a4470000p-4, -0x1.7a16ba8b1cb41p-44,
824 -0x1.08598b59e4000p-4, 0x1.7e9dd7009902cp-46,
825 -0x1.ccb73cddd8000p-5, -0x1.967c36e09f5fep-44,
826 -0x1.894aa149f8000p-5, -0x1.9a55a8be97661p-44,
827 -0x1.466aed42e0000p-5, 0x1.c073375bdfd28p-45,
828 -0x1.0415d89e78000p-5, 0x1.ddfc7f461c516p-44,
829 -0x1.8492528c90000p-6, 0x1.a9dba325a0c34p-45,
830 -0x1.0205658930000p-6, -0x1.60dd27c8e8417p-44,
831 -0x1.0101575880000p-7, -0x1.bcd251998b506p-44,
832 0x0.0p+0, 0x0.0p+0,
833 0x1.fe02a6b100000p-8, 0x1.9e63f0dda40e4p-46,
834 0x1.fc0a8b0fc0000p-7, 0x1.e1e7cf6d3a69cp-50,
835 0x1.7b91b07d60000p-6, -0x1.3b685b602ace4p-44,
836 0x1.f829b0e780000p-6, 0x1.97c267c7e09e4p-45,
837 0x1.39e87b9fe8000p-5, 0x1.eb3d480ad9015p-44,
838 0x1.77458f6330000p-5, -0x1.1807ce586af09p-44,
839 0x1.b42dd71198000p-5, -0x1.c8d7ae5d6704cp-46,
840 0x1.f0a30c0118000p-5, -0x1.d579e83368e91p-45,
841 0x1.16536eea38000p-4, -0x1.472de768fa309p-46,
842 0x1.341d7961bc000p-4, 0x1.1cfb299837610p-44,
843 0x1.51b073f060000p-4, 0x1.83ba9278e686ap-44,
844 0x1.6f0d28ae58000p-4, -0x1.4b2241b664613p-44,
845 0x1.8c345d6318000p-4, 0x1.b22b5acb42a66p-44,
846 0x1.a926d3a4ac000p-4, 0x1.561c50bd22a9cp-44,
847 0x1.c5e548f5bc000p-4, 0x1.d0c97585fbe06p-46,
848 0x1.e27076e2b0000p-4, -0x1.a2c2c2af0003cp-45,
849 0x1.fec9131dc0000p-4, -0x1.54455d1ae6607p-44,
850 0x1.0d77e7cd08000p-3, 0x1.cb6cd2ee2f482p-44,
851 0x1.1b72ad52f6000p-3, 0x1.e86041811a396p-45,
852 0x1.29552f8200000p-3, -0x1.5bd67f4471dfcp-44,
853 0x1.371fc201e8000p-3, 0x1.eea079b2d8abcp-44,
854 0x1.44d2b6ccb8000p-3, -0x1.71f416135783cp-46,
855 0x1.526e5e3a1c000p-3, -0x1.790aa37fc5238p-44,
856 0x1.5ff3070a7a000p-3, -0x1.8546f183bebf2p-44,
857 0x1.6d60fe719e000p-3, -0x1.bc91557134767p-44,
858 0x1.7ab890210e000p-3, -0x1.be51072534a58p-45,
859 0x1.87fa06520c000p-3, 0x1.22130401202fcp-44,
860 0x1.9525a9cf46000p-3, -0x1.294937d9f158fp-44,
861 0x1.a23bc1fe2c000p-3, -0x1.53d6d91dc9f0bp-44,
862 0x1.af3c94e80c000p-3, -0x1.92e633fcd9066p-52,
863 0x1.bc286742d8000p-3, 0x1.9a873f39d121cp-44,
864 0x1.c8ff7c79aa000p-3, -0x1.7814f689f8434p-45,
865 0x1.d5c216b4fc000p-3, -0x1.1b0d1bbca681bp-45,
866 0x1.e27076e2b0000p-3, -0x1.a302c2af0003cp-44,
867 0x1.ef0adcbdc6000p-3, -0x1.b2a179c86af24p-45,
868 0x1.fb9186d5e4000p-3, -0x1.d6b2aab993c87p-47,
869 0x1.0402594b4d000p-2, 0x1.037b89ef42d7fp-48,
870 0x1.0a324e2739000p-2, 0x1.c4dee7ef4030ep-47,
871 0x1.1058bf9ae5000p-2, -0x1.4affd817d52cdp-44,
872 0x1.1675cababa000p-2, 0x1.83c0e731f55c4p-44,
873 0x1.1c898c169a000p-2, -0x1.81260e5c62affp-44,
874 0x1.22941fbcf8000p-2, -0x1.a6876f5eb0963p-44,
875 0x1.2895a13de8000p-2, 0x1.a917ad24c13f0p-44,
876 0x1.2e8e2bae12000p-2, -0x1.6791e99b72bd8p-45,
877 0x1.347dd9a988000p-2, -0x1.5522dd4c58092p-45,
878 0x1.3a64c55694000p-2, 0x1.7a81cbcd735d0p-44,
879 0x1.404308686a000p-2, 0x1.f8f043049f7d3p-44,
880 0x1.4618bc21c6000p-2, -0x1.3e02f484c84ccp-46,
881 0x1.4be5f95778000p-2, -0x1.d7c52cd9ad824p-44,
882 0x1.51aad872e0000p-2, -0x1.f49d8db0a7cc1p-44,
883 0x1.5767717456000p-2, -0x1.650fd9524d7cap-44,
884 0x1.5d1bdbf581000p-2, -0x1.8d97dc9c7c238p-44,
885 };
886
896 double u_low,
897 double& sum,
898 double& small) noexcept
899 {
900 constexpr double shifter {0x1.8p52};
901 const auto bits {std::bit_cast<std::uint64_t>(u)};
902 const double mantissa {std::bit_cast<double>((bits & 0x000fffffffffffffU) | 0x3ff0000000000000U)};
903 const bool upper {mantissa >= 1.40625};
904 const double m {blend(upper, mantissa * 0.5, mantissa)};
905 // the biased exponent, exact, without a conversion AVX2 has no vector form of
906 const double biased {std::bit_cast<double>((bits >> 52U) | 0x4330000000000000U) - 0x1p52};
907 const double e_d {biased - blend(upper, 1022.0, 1023.0)};
908 const std::uint64_t field {(bits >> 52U) + static_cast<std::uint64_t>(upper)}; // e + 1023
909 const double z {std::fma(m, 128.0, shifter)};
910 const auto k {static_cast<std::size_t>(std::bit_cast<std::int64_t>(z) - std::bit_cast<std::int64_t>(shifter)) - 90U};
911 const double inverse {log_inverse[k]};
912 const double head {log_table[2 * k]};
913 const double tail {log_table[(2 * k) + 1]};
914 const double product {m * inverse};
915 const double r_err {std::fma(m, inverse, -product)};
916 // 2^−e, a normal double: past 2^1022 u_low is far under an ulp of the result
917 const double unscale {std::bit_cast<double>((2046U - std::min<std::uint64_t>(field, 2045U)) << 52U)};
918 double r {};
919 double r_low {};
920 two_sum(product - 1.0, std::fma(u_low * unscale, inverse, r_err), r, r_low); // product − 1 and u_low's scaling exact
921 const double square {r * r};
922 const double sq_err {std::fma(r, r, -square)};
923 double q {std::fma(r, -1.0 / 10.0, 1.0 / 9.0)};
924 q = std::fma(r, q, -1.0 / 8.0);
925 q = std::fma(r, q, 1.0 / 7.0);
926 q = std::fma(r, q, -1.0 / 6.0);
927 q = std::fma(r, q, 1.0 / 5.0);
928 q = std::fma(r, q, -1.0 / 4.0);
929 q = std::fma(r, q, 1.0 / 3.0);
930 const double first {std::fma(e_d, log_ln2_high, head)}; // exact: multiples of 2^−42 under 2^11
931 double second {};
932 double second_e {};
933 two_sum(first, r, second, second_e);
934 double third_e {};
935 two_sum(second, -0.5 * square, sum, third_e); // the product exact
936 const double low_part {std::fma(e_d, log_ln2_low, tail) + std::fma(r_low, square - r, r_low)};
937 small = std::fma(square * r, q, std::fma(-0.5, sq_err, (second_e + third_e) + low_part));
938 }
939
945 NUMERICAL_ALWAYS_INLINE double table_log1p(double x) noexcept
946 {
947 // +∞ as −1, so one comparison: && would be a branch GCC does not vectorize
948 const bool finite {blend(x < HUGE_VAL, x, -1.0) > -1.0};
949 double u {};
950 double u_low {};
951 two_sum(1.0, blend(finite, x, 0.0), u, u_low);
952 double sum {};
953 double small {};
954 log_parts(u, u_low, sum, small);
955 const double edge {blend(x == -1.0, -HUGE_VAL, blend(x < -1.0, std::numeric_limits<double>::quiet_NaN(), x))};
956 return blend(std::fabs(x) >= 0x1p-54, blend(finite, sum + small, edge), x); // NaN fails the comparison
957 }
958
959#ifdef NUMERICAL_EXP_NEON
967 {
968 const float64x2_t shifter {vdupq_n_f64(0x1.8p52)};
969 const float64x2_t clamped {vminq_f64(x, vdupq_n_f64(710.0))}; // NaN stays NaN
972 const float64x2_t k {vsubq_f64(z, shifter)};
974 const float64x2_t r2 {vmulq_f64(r, r)};
975 float64x2_t p {vfmaq_f64(vdupq_n_f64(1.0 / 120.0), r, vdupq_n_f64(1.0 / 720.0))};
976 p = vfmaq_f64(vdupq_n_f64(1.0 / 24.0), r, p);
977 p = vfmaq_f64(vdupq_n_f64(1.0 / 6.0), r, p);
978 p = vfmaq_f64(vdupq_n_f64(0.5), r, p);
979 p = vfmaq_f64(r, r2, p);
980 const int64x2_t j {vandq_s64(n, vdupq_n_s64(127))};
981 const float64x2_t low {vld1q_f64(exp_table.data() + (2 * static_cast<std::size_t>(vgetq_lane_s64(j, 0))))};
982 const float64x2_t high {vld1q_f64(exp_table.data() + (2 * static_cast<std::size_t>(vgetq_lane_s64(j, 1))))};
983 const float64x2_t head {vzip1q_f64(low, high)};
984 const float64x2_t tail {vzip2q_f64(low, high)};
986 const int64x2_t e {vshrq_n_s64(n, 7)};
987 // e / 2 toward zero, as the scalar division: an odd negative e's half rounded up
989 const int64x2_t bias {vdupq_n_s64(1023)};
992 const float64x2_t result {vmulq_f64(vmulq_f64(scaled, first), second)};
993 return vbslq_f64(vceqq_f64(x, x), result, x);
994 }
995
998 float64x2_t b,
1000 float64x2_t& error) noexcept
1001 {
1002 sum = vaddq_f64(a, b);
1003 const float64x2_t b_part {vsubq_f64(sum, a)};
1005 }
1006
1010 float64x2_t& small) noexcept
1011 {
1012 const float64x2_t shifter {vdupq_n_f64(0x1.8p52)};
1015 const float64x2_t k {vsubq_f64(z, shifter)};
1018 float64x2_t q {vfmaq_f64(vdupq_n_f64(1.0 / 720.0), r, vdupq_n_f64(1.0 / 5040.0))};
1019 q = vfmaq_f64(vdupq_n_f64(1.0 / 120.0), r, q);
1020 q = vfmaq_f64(vdupq_n_f64(1.0 / 24.0), r, q);
1021 q = vfmaq_f64(vdupq_n_f64(1.0 / 6.0), r, q);
1022 q = vfmaq_f64(vdupq_n_f64(0.5), r, q);
1023 const float64x2_t square {vmulq_f64(r, r)};
1024 const float64x2_t rest {vmulq_f64(square, q)};
1025 const float64x2_t p {vfmaq_f64(r, square, q)};
1027 const int64x2_t j {vandq_s64(n, vdupq_n_s64(127))};
1028 const float64x2_t low {vld1q_f64(exp_table.data() + (2 * static_cast<std::size_t>(vgetq_lane_s64(j, 0))))};
1029 const float64x2_t high {vld1q_f64(exp_table.data() + (2 * static_cast<std::size_t>(vgetq_lane_s64(j, 1))))};
1030 const float64x2_t head {vzip1q_f64(low, high)};
1031 const float64x2_t tail {vzip2q_f64(low, high)};
1032 const int64x2_t e {vshrq_n_s64(n, 7)};
1033 const int64x2_t cap {vdupq_n_s64(1022)};
1034 const int64x2_t capped {vbslq_s64(vcgtq_s64(e, cap), cap, e)}; // NEON has no 64-bit minimum
1036 float64x2_t lead {};
1044 return e;
1045 }
1046
1049 {
1050 // NaN stays NaN through both: vminq_f64 and vmaxq_f64 return it
1051 const float64x2_t clamped {vmaxq_f64(vminq_f64(x, vdupq_n_f64(710.0)), vdupq_n_f64(-40.0))};
1052 float64x2_t sum {};
1053 float64x2_t small {};
1054 const int64x2_t e {expm1_parts(clamped, sum, small)};
1056 const int64x2_t bias {vdupq_n_s64(1023)};
1057 const float64x2_t first {vreinterpretq_f64_s64(vshlq_n_s64(vaddq_s64(half, bias), 52))};
1058 const float64x2_t second {vreinterpretq_f64_s64(vshlq_n_s64(vaddq_s64(vsubq_s64(e, half), bias), 52))};
1059 const float64x2_t result {vmulq_f64(vmulq_f64(vaddq_f64(sum, small), first), second)};
1060 return vbslq_f64(vcgeq_f64(vabsq_f64(x), vdupq_n_f64(0x1p-54)), result, x);
1061 }
1062
1065 {
1066 const float64x2_t magnitude {vabsq_f64(x)};
1067 float64x2_t sum {};
1068 float64x2_t small {};
1069 const int64x2_t e {expm1_parts(vmulq_f64(vdupq_n_f64(2.0), vminq_f64(magnitude, vdupq_n_f64(22.0))), sum, small)};
1070 float64x2_t high {};
1071 float64x2_t low {};
1072 two_sum(sum, small, high, low);
1074 const float64x2_t t {vmulq_f64(high, scale)};
1076 float64x2_t d_err {};
1077 two_sum(vdupq_n_f64(2.0), t, d_high, d_err);
1078 const float64x2_t d_low {vfmaq_f64(d_err, low, scale)};
1083 // the sign of x on the result: its sign bit and the result's other bits
1084 const uint64x2_t sign {vdupq_n_u64(0x8000000000000000U)};
1086 return vbslq_f64(vcgeq_f64(magnitude, vdupq_n_f64(0x1p-27)), signed_r, x);
1087 }
1088
1094 float64x2_t& u_low) noexcept
1095 {
1096 float64x2_t sum {};
1097 float64x2_t small {};
1098 const int64x2_t e {expm1_parts(vminq_f64(magnitude, vdupq_n_f64(711.0)), sum, small)};
1099 float64x2_t high {};
1100 float64x2_t low {};
1101 two_sum(sum, small, high, low);
1102 const int64x2_t cap {vdupq_n_s64(64)};
1103 const int64x2_t kept {vbslq_s64(vcgtq_s64(e, cap), cap, e)};
1105 float64x2_t d_err {};
1106 two_sum(vdupq_n_f64(1.0), vmulq_f64(high, scale), d_high, d_err);
1107 d_low = vfmaq_f64(d_err, low, scale);
1111 }
1112
1115 {
1116 const float64x2_t magnitude {vabsq_f64(x)};
1118 float64x2_t d_low {};
1120 float64x2_t u_low {};
1121 const float64x2_t power {hyperbolic_parts(magnitude, d_high, d_low, u_high, u_low)};
1122 float64x2_t lead {};
1126 const float64x2_t signed_r {vbslq_f64(vdupq_n_u64(0x8000000000000000U), x, result)};
1127 return vbslq_f64(vcgeq_f64(magnitude, vdupq_n_f64(0x1p-27)), signed_r, x);
1128 }
1129
1132 {
1134 float64x2_t d_low {};
1136 float64x2_t u_low {};
1138 float64x2_t lead {};
1142 return vbslq_f64(vceqq_f64(x, x), result, x);
1143 }
1144
1147 {
1148 const float64x2_t shifter {vdupq_n_f64(0x1.8p52)};
1149 const float64x2_t clamped {vminq_f64(x, vdupq_n_f64(1025.0))};
1150 const float64x2_t z {vfmaq_f64(shifter, clamped, vdupq_n_f64(128.0))};
1152 const float64x2_t k {vsubq_f64(z, shifter)};
1154 const float64x2_t r2 {vmulq_f64(r, r)};
1155 float64x2_t p {vfmaq_f64(vdupq_n_f64(1.0 / 120.0), r, vdupq_n_f64(1.0 / 720.0))};
1156 p = vfmaq_f64(vdupq_n_f64(1.0 / 24.0), r, p);
1157 p = vfmaq_f64(vdupq_n_f64(1.0 / 6.0), r, p);
1158 p = vfmaq_f64(vdupq_n_f64(0.5), r, p);
1159 p = vfmaq_f64(r, r2, p);
1160 const int64x2_t j {vandq_s64(n, vdupq_n_s64(127))};
1161 const float64x2_t low {vld1q_f64(exp_table.data() + (2 * static_cast<std::size_t>(vgetq_lane_s64(j, 0))))};
1162 const float64x2_t high {vld1q_f64(exp_table.data() + (2 * static_cast<std::size_t>(vgetq_lane_s64(j, 1))))};
1163 const float64x2_t head {vzip1q_f64(low, high)};
1164 const float64x2_t tail {vzip2q_f64(low, high)};
1166 const int64x2_t e {vshrq_n_s64(n, 7)};
1168 const int64x2_t bias {vdupq_n_s64(1023)};
1169 const float64x2_t first {vreinterpretq_f64_s64(vshlq_n_s64(vaddq_s64(half, bias), 52))};
1170 const float64x2_t second {vreinterpretq_f64_s64(vshlq_n_s64(vaddq_s64(vsubq_s64(e, half), bias), 52))};
1171 const float64x2_t result {vmulq_f64(vmulq_f64(scaled, first), second)};
1172 return vbslq_f64(vceqq_f64(x, x), result, x);
1173 }
1174
1177 {
1178 const float64x2_t one {vdupq_n_f64(1.0)};
1179 const float64x2_t shifter {vdupq_n_f64(0x1.8p52)};
1180 const float64x2_t magnitude {vabsq_f64(x)};
1181 const uint64x2_t far {vcgtq_f64(magnitude, one)};
1182 const float64x2_t inverse {vdivq_f64(one, magnitude)};
1184 const float64x2_t zero {vdupq_n_f64(0.0)};
1185 const float64x2_t y {vbslq_f64(far, inverse, vbslq_f64(vcleq_f64(magnitude, one), magnitude, zero))};
1187 const float64x2_t z {vfmaq_f64(shifter, y, vdupq_n_f64(64.0))};
1189 const float64x2_t c {vmulq_f64(vsubq_f64(z, shifter), vdupq_n_f64(0x1p-6))};
1190 const float64x2_t difference {vsubq_f64(y, c)};
1193 const float64x2_t den {vfmaq_f64(one, y, c)};
1195 const float64x2_t t {vdivq_f64(num, den)};
1197 const float64x2_t square {vmulq_f64(t, t)};
1198 float64x2_t q {vfmaq_f64(vdupq_n_f64(1.0 / 9.0), square, vdupq_n_f64(-1.0 / 11.0))};
1199 q = vfmaq_f64(vdupq_n_f64(-1.0 / 7.0), square, q);
1200 q = vfmaq_f64(vdupq_n_f64(1.0 / 5.0), square, q);
1201 q = vfmaq_f64(vdupq_n_f64(-1.0 / 3.0), square, q);
1202 const float64x2_t low {vld1q_f64(atan_table.data() + (2 * static_cast<std::size_t>(vgetq_lane_s64(k, 0))))};
1203 const float64x2_t high {vld1q_f64(atan_table.data() + (2 * static_cast<std::size_t>(vgetq_lane_s64(k, 1))))};
1204 const float64x2_t head {vzip1q_f64(low, high)};
1205 const float64x2_t tail {vzip2q_f64(low, high)};
1206 float64x2_t sum {};
1207 float64x2_t error {};
1208 two_sum(head, t, sum, error);
1209 const float64x2_t small {vfmaq_f64(vaddq_f64(vaddq_f64(error, tail), t_low), vmulq_f64(t, square), q)};
1210 float64x2_t rest {};
1213 const float64x2_t near {vaddq_f64(sum, small)};
1216 vbslq_f64(far, beyond, near))};
1217 const float64x2_t signed_r {vbslq_f64(vdupq_n_u64(0x8000000000000000U), x, result)};
1218 return vbslq_f64(vcgeq_f64(magnitude, vdupq_n_f64(0x1p-27)), signed_r, x);
1219 }
1220
1225 float64x2_t& small) noexcept
1226 {
1227 const float64x2_t shifter {vdupq_n_f64(0x1.8p52)};
1229 const int64x2_t bias {vdupq_n_s64(1023)};
1231 vdupq_n_u64(0x3ff0000000000000U)))};
1232 const uint64x2_t upper {vcgeq_f64(mantissa, vdupq_n_f64(1.40625))};
1234 // the biased exponent less the bias, plus 1 where upper (its mask −1 subtracted)
1236 const float64x2_t z {vfmaq_f64(shifter, m, vdupq_n_f64(128.0))};
1238 const auto k0 {static_cast<std::size_t>(vgetq_lane_s64(k, 0))};
1239 const auto k1 {static_cast<std::size_t>(vgetq_lane_s64(k, 1))};
1241 const float64x2_t low {vld1q_f64(log_table.data() + (2 * k0))};
1242 const float64x2_t high {vld1q_f64(log_table.data() + (2 * k1))};
1243 const float64x2_t head {vzip1q_f64(low, high)};
1244 const float64x2_t tail {vzip2q_f64(low, high)};
1248 const int64x2_t bounded {vbslq_s64(vcgtq_s64(e, vdupq_n_s64(1022)), vdupq_n_s64(1022), e)};
1250 float64x2_t r {};
1251 float64x2_t r_low {};
1253 const float64x2_t square {vmulq_f64(r, r)};
1255 float64x2_t q {vfmaq_f64(vdupq_n_f64(1.0 / 9.0), r, vdupq_n_f64(-1.0 / 10.0))};
1256 q = vfmaq_f64(vdupq_n_f64(-1.0 / 8.0), r, q);
1257 q = vfmaq_f64(vdupq_n_f64(1.0 / 7.0), r, q);
1258 q = vfmaq_f64(vdupq_n_f64(-1.0 / 6.0), r, q);
1259 q = vfmaq_f64(vdupq_n_f64(1.0 / 5.0), r, q);
1260 q = vfmaq_f64(vdupq_n_f64(-1.0 / 4.0), r, q);
1261 q = vfmaq_f64(vdupq_n_f64(1.0 / 3.0), r, q);
1262 const float64x2_t e_d {vcvtq_f64_s64(e)};
1264 float64x2_t second {};
1266 two_sum(first, r, second, second_e);
1268 two_sum(second, vmulq_f64(vdupq_n_f64(-0.5), square), sum, third_e);
1272 small = vfmaq_f64(vfmaq_f64(rest, vdupq_n_f64(-0.5), sq_err), vmulq_f64(square, r), q);
1273 }
1274
1277 {
1278 const float64x2_t one {vdupq_n_f64(1.0)};
1281 float64x2_t u {};
1282 float64x2_t u_low {};
1283 two_sum(one, vbslq_f64(finite, x, vdupq_n_f64(0.0)), u, u_low);
1284 float64x2_t sum {};
1285 float64x2_t small {};
1286 log_parts(u, u_low, sum, small);
1287 const float64x2_t result {vaddq_f64(sum, small)};
1289 vbslq_f64(vcltq_f64(x, vnegq_f64(one)), vdupq_n_f64(std::numeric_limits<double>::quiet_NaN()), x))};
1290 const float64x2_t value {vbslq_f64(finite, result, edge)};
1291 return vbslq_f64(vcgeq_f64(vabsq_f64(x), vdupq_n_f64(0x1p-54)), value, x);
1292 }
1293
1294#endif
1295} // namespace numerical::detail
1296
1297#endif // NUMERICAL_ELEMENTARY_HPP
What every exception NumericAL throws is, beside its standard base (all but std::bad_alloc and std::s...
Definition vector.hpp:40
A floating-point field: float, double, long double, or std::complex of one. The factorisations and th...
Definition concepts.hpp:28
bool finite(const T &x)
Whether the real x is finite (an overload per kind of scalar, as conj_of: each one expression).
Definition blas.hpp:79
constexpr double exp_inv_ln2_128
128/ln 2 (tools/exp_table.py derives each constant here).
constexpr std::array< double, 256 > sine_table
For j = 0…63, at 4j sin(jπ/32) and at 4j + 1 what that double leaves out, at 4j + 2 cos(jπ/32) and at...
constexpr double atan_half_pi_low
What atan_half_pi_high leaves out of π/2, to the nearest double.
constexpr std::array< double, 3 > half_pi_parts
π/2 in three parts: its double, then the double of the rest, then the double of what is left (their s...
constexpr double sine_inverse_step
32/π, the double nearest it (tools/sine_table.py derives each constant here).
constexpr std::array< double, 130 > atan_table
atan(k/64) for k = 0…64, at 2k the double nearest it and at 2k + 1 the double nearest what that one l...
constexpr double sine_reduction_bound
The magnitude to which reduced_sine reduces its argument: k below 2²⁰, k·π/2's parts exact enough.
double table_expm1(double x) noexcept
expm1(x) by table_exp's reduction, x = (128 e + j) ln 2/128 + r: in units of 2^e, 2^(j/128) − 2^−e
double table_exp2(double x) noexcept
2^x for an x whose result is normal or infinite (−1022 or above): x = (128 e + j)/128 + f,...
std::complex< R > scaled(R c, const std::complex< R > &v) noexcept
c · v of a real c and a complex v: each part's product, rounded.
Definition fft.hpp:1290
T square(const T &x)
x · x, by the type's own product (a complex one's C Annex G, as the library's *).
constexpr double log_ln2_high
ln 2 to 42 significant bits, toward zero: its product by an exponent of 11 bits or fewer is exact (to...
constexpr double exp2_ln2
ln 2 to the nearest double.
constexpr double atan_half_pi_high
π/2, the double nearest it (tools/atan_table.py derives each constant here).
sine_terms sine_terms_of< true >(std::uint64_t entry, double) noexcept
The terms of the cosine (sine_terms): a = C, b = −S, 1 where tiny.
double table_atan(double x) noexcept
atan(x): y = |x|, or 1/|x| past 1 (its rounding error carried), c = k/64 the nearest point of atan_ta...
constexpr std::array< double, 256 > exp_table
2^(j/128) for j = 0…127, at 2j the double nearest it and at 2j + 1 the double nearest what that one l...
sine_terms sine_terms_of< false >(std::uint64_t entry, double x) noexcept
The terms of the sine (sine_terms): a = S, b = C, x itself where tiny.
double table_sinh(double x) noexcept
sinh(x) = (d − u)/2 (hyperbolic_parts), the sign of x: within 0.503 ulp of the exact value up to the ...
double table_exp(double x) noexcept
exp(x) for an x whose result is normal or infinite (exp_normal_bound or above): x = (128 e + j) ln 2/...
double table_tanh(double x) noexcept
tanh(x) = t / (t + 2), t = expm1(2|x|) as a pair (expm1_parts, its two parts summed exactly),...
constexpr double exp_normal_bound
ln 2^-1022 toward zero: below it, exp is subnormal, where table_exp's two scalings would round twice.
double reduced_sine(double x) noexcept
sin(x), or cos(x) where Cosine, for |x| ≤ sine_reduction_bound (another x gives a value of no use): k...
constexpr double exp_ln2_128_low
What exp_ln2_128_high leaves out of ln 2/128, to the nearest double.
std::int64_t expm1_parts(double clamped, double &sum, double &small) noexcept
expm1(clamped) in units of 2^e, as sum + small unrounded, and e: table_exp's reduction,...
void log_parts(double u, double u_low, double &sum, double &small) noexcept
ln(u + u_low) as sum + small unrounded, for u at least 2^−54 and finite and |u_low| at most half an u...
double table_log1p(double x) noexcept
ln(1 + x) by log_parts of 1 + x as a pair (its two-sum): near 0, where c = 1, r is x exactly....
double table_cosh(double x) noexcept
cosh(x) = (d + u)/2 (hyperbolic_parts): within 0.502 ulp of the exact value up to the overflow; NaN f...
@ zero
C = A·B: C's elements before the product are not read.
constexpr std::array< double, 182 > log_table
For k = 90…180, at 2(k − 90) −ln of log_inverse's entry to the nearest multiple of 2^−42 (its sum wit...
void two_sum(double a, double b, double &sum, double &error) noexcept
The sum of a and b as sum rounded and error exact (Knuth's two-sum: no order of magnitudes needed).
double blend(bool condition, double if_true, double if_false) noexcept
if_true where condition holds, else if_false, by a mask of their bits: GCC turns a ?...
double table_sine(double x) noexcept
sin(x), or cos(x) where Cosine, of a double within half an ulp and a few thousandths,...
constexpr double exp_ln2_128_high
ln 2/128 to 32 significant bits, toward zero: its product by an integer of 18 bits or fewer is exact,...
constexpr std::array< double, 3 > sine_step_parts
π/32 in three parts: its double, then the double of the rest, then the double of what is left.
constexpr double log_ln2_low
What log_ln2_high leaves out of ln 2, to the nearest double.
constexpr std::array< double, 91 > log_inverse
For k = 90…180, at k − 90 the double nearest 128/k. Not inline, as exp_table.
double hyperbolic_parts(double magnitude, double &d_high, double &d_low, double &u_high, double &u_low) noexcept
The parts of sinh and cosh of magnitude (0 to 711, past which both overflow): t = expm1(magnitude) (e...
sine_terms sine_terms_of(std::uint64_t entry, double x) noexcept
sine_terms of the table's row at entry (sin head, sin tail, cos head, cos tail) for x: the sine's whe...
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
matrix< T > inverse(matrix< T > a)
The inverse of a square matrix.
The native vector of float and double the kernels hold their lanes in (GNU vector extensions,...
#define NUMERICAL_ALWAYS_INLINE
Definition simd.hpp:25
The terms table_sine adds for sin x = a + b·r + …: a = S and b = C for the sine, a = C and b = −S for...
double a
The leading term's head.
double tiny
The value where |x| < 2⁻²⁷.
double b_tail
r's factor, its tail.
double b
r's factor, its head.
double a_tail
The leading term's tail.