NumericAL
A header-only C++20 lexer built on REAL
Loading...
Searching...
No Matches
x86.hpp
Go to the documentation of this file.
1
11#ifndef NUMERICAL_X86_HPP
12#define NUMERICAL_X86_HPP
13
14#include <algorithm>
15#include <array>
16#include <bit>
17#include <cmath>
18#include <complex>
19#include <cstddef>
20#include <cstdint>
21#include <type_traits>
22#include <vector>
23
24#include "simd.hpp"
25
26#if defined(__x86_64__) || defined(_M_X64)
27# define NUMERICAL_X86_COMPILED // the x86-64 tiles are compiled in (tested with #ifdef only)
28# include <immintrin.h>
29# if defined(_MSC_VER) && !defined(__clang__)
30# include <intrin.h>
31# define NUMERICAL_TARGET(isa)
32# define NUMERICAL_UNROLL_TILE
33# define NUMERICAL_UNROLL_DEPTH
34# define NUMERICAL_UNROLL_DEPTH_AVX2
35# else
36# include <cpuid.h>
37# define NUMERICAL_TARGET(isa) __attribute__((target(isa))) // NOLINT(cppcoreguidelines-macro-usage): an attribute, which no function can stand for
38// A tile loop unrolled whole, so each accumulator stays in its register (rolled, GCC keeps them in memory).
39# define NUMERICAL_UNROLL_TILE _Pragma("GCC unroll 24")
40// The depth loop unrolled by two: half the loop's own instructions, which compete with the multiply-adds for
41// issue slots (an AVX-512 tile in the L1 cache: 103 GFLOP/s rolled, 115 unrolled, on one x86-64 core).
42# define NUMERICAL_UNROLL_DEPTH _Pragma("GCC unroll 2")
43// The AVX2 tile's depth loop unrolled by four, as OpenBLAS's Haswell kernel unrolls it (by eight): a product of order
44// 1000 on one AVX2 core, 50.5 → 52.9 GFLOP/s, by two; by eight no better.
45# define NUMERICAL_UNROLL_DEPTH_AVX2 _Pragma("GCC unroll 4")
46# endif
47#endif
48
49namespace numerical::detail {
50
51#ifdef NUMERICAL_X86_COMPILED
53 struct x86_features
54 {
55 bool avx2_fma {false};
56 bool avx512f {false};
57 };
58
60 [[nodiscard]] inline std::array<std::uint32_t, 4> cpuid(std::uint32_t leaf,
61 std::uint32_t sub) noexcept
62 {
63 std::array<std::uint32_t, 4> r {};
64# if defined(_MSC_VER) && !defined(__clang__)
65 std::array<int, 4> regs {};
66 __cpuidex(regs.data(), static_cast<int>(leaf), static_cast<int>(sub));
67 for (std::size_t i {0}; i < 4; ++i) {
68 r[i] = static_cast<std::uint32_t>(regs[i]);
69 }
70# else
71 __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);
72# endif
73 return r;
74 }
75
78 {
79 if ((leaf1_ecx & (1U << 27U)) == 0U) { // OSXSAVE
80 return 0;
81 }
82 return static_cast<std::uint64_t>(_xgetbv(0));
83 }
84
86 [[nodiscard]] constexpr x86_features decode_features(std::uint32_t leaf1_ecx,
87 std::uint32_t leaf7_ebx,
88 std::uint64_t state) noexcept
89 {
90 const bool avx {(leaf1_ecx & (1U << 28U)) != 0U};
91 const bool ymm {avx && (state & 0x6U) == 0x6U}; // the AVX instructions, and SSE and AVX state
92 const bool zmm {avx && (state & 0xe6U) == 0xe6U}; // and the opmask and both halves of the 512-bit registers
93 const bool fma {(leaf1_ecx & (1U << 12U)) != 0U};
94 const bool avx2 {(leaf7_ebx & (1U << 5U)) != 0U};
95 const bool avx512f {(leaf7_ebx & (1U << 16U)) != 0U};
96 return {.avx2_fma = ymm && avx2 && fma, .avx512f = zmm && avx512f};
97 }
98
101 {
102 static const x86_features features {[] {
103 const std::array<std::uint32_t, 4> leaf1 {cpuid(1, 0)};
104 // Leaf 7 only where the processor has it: past its last leaf, cpuid repeats
105 // another leaf's words.
106 const std::array<std::uint32_t, 4> leaf7 {cpuid(0, 0)[0] >= 7U ? cpuid(7, 0)
107 : std::array<std::uint32_t, 4> {}};
108 return decode_features(leaf1[2], leaf7[1], saved_state(leaf1[2]));
109 }()};
110 return features;
111 }
112
121 template <typename Body>
123 std::size_t index)
124 {
125 body(index);
126 }
127
130 struct zmm_register
131 {
132 __m512d v;
133 };
134
136 struct ymm_register
137 {
138 __m256d v;
139 };
140
142 struct xmm_register
143 {
144 __m128d v;
145 };
146
148 inline constexpr std::size_t avx512_mr {12};
149
151 inline constexpr std::size_t avx512_nr {16};
152
154 inline constexpr std::size_t avx2_mr {6};
155
157 inline constexpr std::size_t avx2_nr {8};
158
165 template <std::size_t Rows = avx512_mr>
166 NUMERICAL_TARGET("avx512f") inline void avx512_full_tile_f64(std::size_t kc,
167 const double* ap,
168 const double* bp,
169 std::size_t ldb,
170 double* tile,
171 std::size_t ld,
172 bool fresh = false) noexcept
173 {
174 std::array<zmm_register, Rows * 2> acc {};
176 for (std::size_t i {0}; i < Rows; ++i) {
177 acc[2 * i].v = fresh ? _mm512_setzero_pd() : _mm512_loadu_pd(tile + (i * ld));
178 acc[(2 * i) + 1].v = fresh ? _mm512_setzero_pd() : _mm512_loadu_pd(tile + (i * ld) + 8);
179 }
181 for (std::size_t p {0}; p < kc; ++p) {
182 const __m512d b0 {_mm512_loadu_pd(bp + (p * ldb))};
183 const __m512d b1 {_mm512_loadu_pd(bp + (p * ldb) + 8)};
185 for (std::size_t i {0}; i < Rows; ++i) {
186 const __m512d a {_mm512_set1_pd(ap[(p * avx512_mr) + i])};
187 acc[2 * i].v = _mm512_fmadd_pd(a, b0, acc[2 * i].v);
188 acc[(2 * i) + 1].v = _mm512_fmadd_pd(a, b1, acc[(2 * i) + 1].v);
189 }
190 }
192 for (std::size_t i {0}; i < Rows; ++i) {
193 _mm512_storeu_pd(tile + (i * ld), acc[2 * i].v);
194 _mm512_storeu_pd(tile + (i * ld) + 8, acc[(2 * i) + 1].v);
195 }
196 _mm256_zeroupper(); // the upper halves cleared: code past the tile may be SSE alone
197 }
198
201 inline void avx512_tile_f64(std::size_t kc,
202 const double* ap,
203 const double* bp,
204 std::size_t ldb,
205 double* c,
206 std::size_t ldc,
207 std::size_t rows,
208 std::size_t cols,
209 bool fresh = false) noexcept
210 {
211 through_rows<avx512_mr, avx512_nr>(c, ldc, rows, cols, [&]<std::size_t Rows>(double* tile, std::size_t ld) noexcept { avx512_full_tile_f64<Rows>(kc, ap, bp, ldb, tile, ld, fresh); });
212 }
213
215 inline constexpr std::size_t avx2_prefetch_ahead {64};
216
221 template <std::size_t Rows = avx2_mr>
222 NUMERICAL_TARGET("avx2,fma") inline void avx2_full_tile_f64(std::size_t kc,
223 const double* ap,
224 const double* bp,
225 std::size_t ldb,
226 double* tile,
227 std::size_t ld,
228 bool fresh = false) noexcept
229 {
230 std::array<ymm_register, Rows * 2> acc {};
232 for (std::size_t i {0}; i < Rows; ++i) {
233 acc[2 * i].v = fresh ? _mm256_setzero_pd() : _mm256_loadu_pd(tile + (i * ld));
234 acc[(2 * i) + 1].v = fresh ? _mm256_setzero_pd() : _mm256_loadu_pd(tile + (i * ld) + 4);
235 }
237 for (std::size_t p {0}; p < kc; ++p) {
238 // A and B a few steps ahead, into the L1 cache: the hardware's streams start a step late at each sliver (as
239 // OpenBLAS's Haswell kernel fetches both 512 bytes ahead; with the unrolling, order 1000 52.9 → 53.9 GFLOP/s).
240 _mm_prefetch(reinterpret_cast<const char*>(bp + (p * ldb) + avx2_prefetch_ahead), _MM_HINT_T0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the intrinsic takes a char pointer
241 _mm_prefetch(reinterpret_cast<const char*>(ap + (p * avx2_mr) + avx2_prefetch_ahead), _MM_HINT_T0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
242 const __m256d b0 {_mm256_loadu_pd(bp + (p * ldb))};
243 const __m256d b1 {_mm256_loadu_pd(bp + (p * ldb) + 4)};
245 for (std::size_t i {0}; i < Rows; ++i) {
246 const __m256d a {_mm256_broadcast_sd(ap + (p * avx2_mr) + i)};
247 acc[2 * i].v = _mm256_fmadd_pd(a, b0, acc[2 * i].v);
248 acc[(2 * i) + 1].v = _mm256_fmadd_pd(a, b1, acc[(2 * i) + 1].v);
249 }
250 }
252 for (std::size_t i {0}; i < Rows; ++i) {
253 _mm256_storeu_pd(tile + (i * ld), acc[2 * i].v);
254 _mm256_storeu_pd(tile + (i * ld) + 4, acc[(2 * i) + 1].v);
255 }
256 _mm256_zeroupper(); // the upper halves cleared: code past the tile may be SSE alone
257 }
258
261 inline void avx2_tile_f64(std::size_t kc,
262 const double* ap,
263 const double* bp,
264 std::size_t ldb,
265 double* c,
266 std::size_t ldc,
267 std::size_t rows,
268 std::size_t cols,
269 bool fresh = false) noexcept
270 {
271 through_rows<avx2_mr, avx2_nr>(c, ldc, rows, cols, [&]<std::size_t Rows>(double* tile, std::size_t ld) noexcept { avx2_full_tile_f64<Rows>(kc, ap, bp, ldb, tile, ld, fresh); });
272 }
273
275 struct zmm_register_f32
276 {
277 __m512 v;
278 };
279
281 struct ymm_register_f32
282 {
283 __m256 v;
284 };
285
287 inline constexpr std::size_t avx512_nr_f32 {32};
288
290 inline constexpr std::size_t avx2_nr_f32 {16};
291
293 template <std::size_t Rows = avx512_mr>
294 NUMERICAL_TARGET("avx512f") inline void avx512_full_tile_f32(std::size_t kc,
295 const float* ap,
296 const float* bp,
297 std::size_t ldb,
298 float* tile,
299 std::size_t ld,
300 bool fresh = false) noexcept
301 {
302 std::array<zmm_register_f32, Rows * 2> acc {};
304 for (std::size_t i {0}; i < Rows; ++i) {
305 acc[2 * i].v = fresh ? _mm512_setzero_ps() : _mm512_loadu_ps(tile + (i * ld));
306 acc[(2 * i) + 1].v = fresh ? _mm512_setzero_ps() : _mm512_loadu_ps(tile + (i * ld) + 16);
307 }
309 for (std::size_t p {0}; p < kc; ++p) {
310 const __m512 b0 {_mm512_loadu_ps(bp + (p * ldb))};
311 const __m512 b1 {_mm512_loadu_ps(bp + (p * ldb) + 16)};
313 for (std::size_t i {0}; i < Rows; ++i) {
314 const __m512 a {_mm512_set1_ps(ap[(p * avx512_mr) + i])};
315 acc[2 * i].v = _mm512_fmadd_ps(a, b0, acc[2 * i].v);
316 acc[(2 * i) + 1].v = _mm512_fmadd_ps(a, b1, acc[(2 * i) + 1].v);
317 }
318 }
320 for (std::size_t i {0}; i < Rows; ++i) {
321 _mm512_storeu_ps(tile + (i * ld), acc[2 * i].v);
322 _mm512_storeu_ps(tile + (i * ld) + 16, acc[(2 * i) + 1].v);
323 }
324 _mm256_zeroupper(); // the upper halves cleared: code past the tile may be SSE alone
325 }
326
328 inline void avx512_tile_f32(std::size_t kc,
329 const float* ap,
330 const float* bp,
331 std::size_t ldb,
332 float* c,
333 std::size_t ldc,
334 std::size_t rows,
335 std::size_t cols,
336 bool fresh = false) noexcept
337 {
338 through_rows<avx512_mr, avx512_nr_f32>(c, ldc, rows, cols, [&]<std::size_t Rows>(float* tile, std::size_t ld) noexcept { avx512_full_tile_f32<Rows>(kc, ap, bp, ldb, tile, ld, fresh); });
339 }
340
342 template <std::size_t Rows = avx2_mr>
343 NUMERICAL_TARGET("avx2,fma") inline void avx2_full_tile_f32(std::size_t kc,
344 const float* ap,
345 const float* bp,
346 std::size_t ldb,
347 float* tile,
348 std::size_t ld,
349 bool fresh = false) noexcept
350 {
351 std::array<ymm_register_f32, Rows * 2> acc {};
353 for (std::size_t i {0}; i < Rows; ++i) {
354 acc[2 * i].v = fresh ? _mm256_setzero_ps() : _mm256_loadu_ps(tile + (i * ld));
355 acc[(2 * i) + 1].v = fresh ? _mm256_setzero_ps() : _mm256_loadu_ps(tile + (i * ld) + 8);
356 }
358 for (std::size_t p {0}; p < kc; ++p) {
359 _mm_prefetch(reinterpret_cast<const char*>(bp + (p * ldb) + (2 * avx2_prefetch_ahead)), _MM_HINT_T0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the intrinsic takes a char pointer
360 _mm_prefetch(reinterpret_cast<const char*>(ap + (p * avx2_mr) + (2 * avx2_prefetch_ahead)), _MM_HINT_T0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
361 const __m256 b0 {_mm256_loadu_ps(bp + (p * ldb))};
362 const __m256 b1 {_mm256_loadu_ps(bp + (p * ldb) + 8)};
364 for (std::size_t i {0}; i < Rows; ++i) {
365 const __m256 a {_mm256_broadcast_ss(ap + (p * avx2_mr) + i)};
366 acc[2 * i].v = _mm256_fmadd_ps(a, b0, acc[2 * i].v);
367 acc[(2 * i) + 1].v = _mm256_fmadd_ps(a, b1, acc[(2 * i) + 1].v);
368 }
369 }
371 for (std::size_t i {0}; i < Rows; ++i) {
372 _mm256_storeu_ps(tile + (i * ld), acc[2 * i].v);
373 _mm256_storeu_ps(tile + (i * ld) + 8, acc[(2 * i) + 1].v);
374 }
375 _mm256_zeroupper(); // the upper halves cleared: code past the tile may be SSE alone
376 }
377
379 inline void avx2_tile_f32(std::size_t kc,
380 const float* ap,
381 const float* bp,
382 std::size_t ldb,
383 float* c,
384 std::size_t ldc,
385 std::size_t rows,
386 std::size_t cols,
387 bool fresh = false) noexcept
388 {
389 through_rows<avx2_mr, avx2_nr_f32>(c, ldc, rows, cols, [&]<std::size_t Rows>(float* tile, std::size_t ld) noexcept { avx2_full_tile_f32<Rows>(kc, ap, bp, ldb, tile, ld, fresh); });
390 }
391
393 inline constexpr std::size_t avx2_complex_mr {3};
394
396 inline constexpr std::size_t avx2_complex_nr {4};
397
405 template <std::size_t Rows = avx2_complex_mr,
406 bool AInPlace = false>
407 NUMERICAL_TARGET("avx2,fma") inline void avx2_full_tile_c64(std::size_t kc,
408 const std::complex<double>* ap,
409 const std::complex<double>* bp,
410 std::size_t ldb,
411 std::complex<double>* tile,
412 std::size_t ld,
413 std::size_t a_step = avx2_complex_mr,
414 std::size_t a_row = 1,
415 bool fresh = false) noexcept
416 {
417 const std::size_t step {AInPlace ? a_step : avx2_complex_mr}; // A's elements apart from one step to the next
418 const std::size_t across {AInPlace ? a_row : 1}; // and from one row to the next
419 const auto* const a {reinterpret_cast<const double*>(ap)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
420 const auto* const b {reinterpret_cast<const double*>(bp)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
421 auto* const c {reinterpret_cast<double*>(tile)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
422 std::array<ymm_register, Rows * 4> acc {};
424 for (std::size_t p {0}; p < kc; ++p) {
425 const __m256d b0 {_mm256_loadu_pd(b + (2 * p * ldb))};
426 const __m256d b1 {_mm256_loadu_pd(b + (2 * p * ldb) + 4)};
428 for (std::size_t i {0}; i < Rows; ++i) {
429 const __m256d re {_mm256_broadcast_sd(a + (2 * ((p * step) + (i * across))))};
430 const __m256d im {_mm256_broadcast_sd(a + (2 * ((p * step) + (i * across))) + 1)};
431 acc[4 * i].v = _mm256_fmadd_pd(re, b0, acc[4 * i].v);
432 acc[(4 * i) + 1].v = _mm256_fmadd_pd(re, b1, acc[(4 * i) + 1].v);
433 acc[(4 * i) + 2].v = _mm256_fmadd_pd(im, b0, acc[(4 * i) + 2].v);
434 acc[(4 * i) + 3].v = _mm256_fmadd_pd(im, b1, acc[(4 * i) + 3].v);
435 }
436 }
438 for (std::size_t i {0}; i < Rows; ++i) {
439 double* const row {c + (2 * i * ld)};
440 const __m256d lo {_mm256_addsub_pd(acc[4 * i].v, _mm256_permute_pd(acc[(4 * i) + 2].v, 0b0101))};
441 const __m256d hi {_mm256_addsub_pd(acc[(4 * i) + 1].v, _mm256_permute_pd(acc[(4 * i) + 3].v, 0b0101))};
444 }
445 _mm256_zeroupper(); // the upper halves cleared: code past the tile may be SSE alone
446 }
447
450 inline void avx2_tile_c64(std::size_t kc,
451 const std::complex<double>* ap,
452 const std::complex<double>* bp,
453 std::size_t ldb,
454 std::complex<double>* c,
455 std::size_t ldc,
456 std::size_t rows,
457 std::size_t cols,
458 bool fresh = false) noexcept
459 {
460 through_rows<avx2_complex_mr, avx2_complex_nr>(c, ldc, rows, cols, [&]<std::size_t Rows>(std::complex<double>* tile, std::size_t ld) noexcept {
462 });
463 }
464
467 inline void avx2_tile_c64_a_in_place(std::size_t kc,
468 const std::complex<double>* a,
469 std::size_t a_step,
470 std::size_t a_row,
471 const std::complex<double>* bp,
472 std::size_t ldb,
473 std::complex<double> * c,
474 std::size_t ldc,
475 std::size_t rows,
476 std::size_t cols,
477 bool fresh = false) noexcept
478 {
479 through_rows<avx2_complex_mr, avx2_complex_nr>(c, ldc, rows, cols, [&]<std::size_t Rows>(std::complex<double>* tile, std::size_t ld) noexcept {
481 });
482 }
483
485 inline constexpr std::size_t avx512_complex_mr {6};
486
488 inline constexpr std::size_t avx512_complex_nr {8};
489
495 template <std::size_t Rows = avx512_complex_mr,
496 bool AInPlace = false>
497 NUMERICAL_TARGET("avx512f") inline void avx512_full_tile_c64(std::size_t kc,
498 const std::complex<double>* ap,
499 const std::complex<double>* bp,
500 std::size_t ldb,
501 std::complex<double>* tile,
502 std::size_t ld,
503 std::size_t a_step = avx512_complex_mr,
504 std::size_t a_row = 1,
505 bool fresh = false) noexcept
506 {
507 const std::size_t step {AInPlace ? a_step : avx512_complex_mr}; // A's elements apart from one step to the next
508 const std::size_t across {AInPlace ? a_row : 1}; // and from one row to the next
509 const auto* const a {reinterpret_cast<const double*>(ap)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
510 const auto* const b {reinterpret_cast<const double*>(bp)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
511 auto* const c {reinterpret_cast<double*>(tile)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
512 std::array<zmm_register, Rows * 4> acc {};
514 for (std::size_t p {0}; p < kc; ++p) {
515 const __m512d b0 {_mm512_loadu_pd(b + (2 * p * ldb))};
516 const __m512d b1 {_mm512_loadu_pd(b + (2 * p * ldb) + 8)};
518 for (std::size_t i {0}; i < Rows; ++i) {
519 const __m512d re {_mm512_set1_pd(a[2 * ((p * step) + (i * across))])};
520 const __m512d im {_mm512_set1_pd(a[(2 * ((p * step) + (i * across))) + 1])};
521 acc[4 * i].v = _mm512_fmadd_pd(re, b0, acc[4 * i].v);
522 acc[(4 * i) + 1].v = _mm512_fmadd_pd(re, b1, acc[(4 * i) + 1].v);
523 acc[(4 * i) + 2].v = _mm512_fmadd_pd(im, b0, acc[(4 * i) + 2].v);
524 acc[(4 * i) + 3].v = _mm512_fmadd_pd(im, b1, acc[(4 * i) + 3].v);
525 }
526 }
527 const __m512d signs {_mm512_set_pd(1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0)}; // −1 on the real parts
529 for (std::size_t i {0}; i < Rows; ++i) {
530 double* const row {c + (2 * i * ld)};
531 const __m512d lo {_mm512_fmadd_pd(_mm512_permute_pd(acc[(4 * i) + 2].v, 0x55), signs, acc[4 * i].v)};
532 const __m512d hi {_mm512_fmadd_pd(_mm512_permute_pd(acc[(4 * i) + 3].v, 0x55), signs, acc[(4 * i) + 1].v)};
535 }
536 }
537
539 inline void avx512_tile_c64(std::size_t kc,
540 const std::complex<double>* ap,
541 const std::complex<double>* bp,
542 std::size_t ldb,
543 std::complex<double>* c,
544 std::size_t ldc,
545 std::size_t rows,
546 std::size_t cols,
547 bool fresh = false) noexcept
548 {
549 through_rows<avx512_complex_mr, avx512_complex_nr>(c, ldc, rows, cols, [&]<std::size_t Rows>(std::complex<double>* tile, std::size_t ld) noexcept {
551 });
552 }
553
556 inline void avx512_tile_c64_a_in_place(std::size_t kc,
557 const std::complex<double>* a,
558 std::size_t a_step,
559 std::size_t a_row,
560 const std::complex<double>* bp,
561 std::size_t ldb,
562 std::complex<double> * c,
563 std::size_t ldc,
564 std::size_t rows,
565 std::size_t cols,
566 bool fresh = false) noexcept
567 {
568 through_rows<avx512_complex_mr, avx512_complex_nr>(c, ldc, rows, cols, [&]<std::size_t Rows>(std::complex<double>* tile, std::size_t ld) noexcept {
570 });
571 }
572
574 inline constexpr std::size_t avx2_complex_nr_f32 {8};
575
580 template <std::size_t Rows = avx2_complex_mr,
581 bool AInPlace = false>
582 NUMERICAL_TARGET("avx2,fma") inline void avx2_full_tile_c32(std::size_t kc,
583 const std::complex<float>* ap,
584 const std::complex<float>* bp,
585 std::size_t ldb,
586 std::complex<float>* tile,
587 std::size_t ld,
588 std::size_t a_step = avx2_complex_mr,
589 std::size_t a_row = 1,
590 bool fresh = false) noexcept
591 {
592 const std::size_t step {AInPlace ? a_step : avx2_complex_mr}; // A's elements apart from one step to the next
593 const std::size_t across {AInPlace ? a_row : 1}; // and from one row to the next
594 const auto* const a {reinterpret_cast<const float*>(ap)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee of [complex.numbers]
595 const auto* const b {reinterpret_cast<const float*>(bp)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
596 auto* const c {reinterpret_cast<float*>(tile)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
597 std::array<ymm_register_f32, Rows * 4> acc {};
599 for (std::size_t p {0}; p < kc; ++p) {
600 const __m256 b0 {_mm256_loadu_ps(b + (2 * p * ldb))};
601 const __m256 b1 {_mm256_loadu_ps(b + (2 * p * ldb) + 8)};
603 for (std::size_t i {0}; i < Rows; ++i) {
604 const __m256 re {_mm256_broadcast_ss(a + (2 * ((p * step) + (i * across))))};
605 const __m256 im {_mm256_broadcast_ss(a + (2 * ((p * step) + (i * across))) + 1)};
606 acc[4 * i].v = _mm256_fmadd_ps(re, b0, acc[4 * i].v);
607 acc[(4 * i) + 1].v = _mm256_fmadd_ps(re, b1, acc[(4 * i) + 1].v);
608 acc[(4 * i) + 2].v = _mm256_fmadd_ps(im, b0, acc[(4 * i) + 2].v);
609 acc[(4 * i) + 3].v = _mm256_fmadd_ps(im, b1, acc[(4 * i) + 3].v);
610 }
611 }
613 for (std::size_t i {0}; i < Rows; ++i) {
614 float* const row {c + (2 * i * ld)};
615 const __m256 lo {_mm256_addsub_ps(acc[4 * i].v, _mm256_permute_ps(acc[(4 * i) + 2].v, 0xB1))};
616 const __m256 hi {_mm256_addsub_ps(acc[(4 * i) + 1].v, _mm256_permute_ps(acc[(4 * i) + 3].v, 0xB1))};
619 }
620 _mm256_zeroupper(); // the upper halves cleared: code past the tile may be SSE alone
621 }
622
624 inline void avx2_tile_c32(std::size_t kc,
625 const std::complex<float>* ap,
626 const std::complex<float>* bp,
627 std::size_t ldb,
628 std::complex<float>* c,
629 std::size_t ldc,
630 std::size_t rows,
631 std::size_t cols,
632 bool fresh = false) noexcept
633 {
634 through_rows<avx2_complex_mr, avx2_complex_nr_f32>(c, ldc, rows, cols, [&]<std::size_t Rows>(std::complex<float>* tile, std::size_t ld) noexcept {
636 });
637 }
638
641 inline void avx2_tile_c32_a_in_place(std::size_t kc,
642 const std::complex<float>* a,
643 std::size_t a_step,
644 std::size_t a_row,
645 const std::complex<float>* bp,
646 std::size_t ldb,
647 std::complex<float> * c,
648 std::size_t ldc,
649 std::size_t rows,
650 std::size_t cols,
651 bool fresh = false) noexcept
652 {
653 through_rows<avx2_complex_mr, avx2_complex_nr_f32>(c, ldc, rows, cols, [&]<std::size_t Rows>(std::complex<float>* tile, std::size_t ld) noexcept {
655 });
656 }
657
659 inline constexpr std::size_t avx512_complex_nr_f32 {16};
660
663 template <std::size_t Rows = avx512_complex_mr,
664 bool AInPlace = false>
665 NUMERICAL_TARGET("avx512f") inline void avx512_full_tile_c32(std::size_t kc,
666 const std::complex<float>* ap,
667 const std::complex<float>* bp,
668 std::size_t ldb,
669 std::complex<float>* tile,
670 std::size_t ld,
671 std::size_t a_step = avx512_complex_mr,
672 std::size_t a_row = 1,
673 bool fresh = false) noexcept
674 {
675 const std::size_t step {AInPlace ? a_step : avx512_complex_mr}; // A's elements apart from one step to the next
676 const std::size_t across {AInPlace ? a_row : 1}; // and from one row to the next
677 const auto* const a {reinterpret_cast<const float*>(ap)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee of [complex.numbers]
678 const auto* const b {reinterpret_cast<const float*>(bp)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
679 auto* const c {reinterpret_cast<float*>(tile)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
680 std::array<zmm_register_f32, Rows * 4> acc {};
682 for (std::size_t p {0}; p < kc; ++p) {
683 const __m512 b0 {_mm512_loadu_ps(b + (2 * p * ldb))};
684 const __m512 b1 {_mm512_loadu_ps(b + (2 * p * ldb) + 16)};
686 for (std::size_t i {0}; i < Rows; ++i) {
687 const __m512 re {_mm512_set1_ps(a[2 * ((p * step) + (i * across))])};
688 const __m512 im {_mm512_set1_ps(a[(2 * ((p * step) + (i * across))) + 1])};
689 acc[4 * i].v = _mm512_fmadd_ps(re, b0, acc[4 * i].v);
690 acc[(4 * i) + 1].v = _mm512_fmadd_ps(re, b1, acc[(4 * i) + 1].v);
691 acc[(4 * i) + 2].v = _mm512_fmadd_ps(im, b0, acc[(4 * i) + 2].v);
692 acc[(4 * i) + 3].v = _mm512_fmadd_ps(im, b1, acc[(4 * i) + 3].v);
693 }
694 }
695 const __m512 signs {_mm512_set_ps(1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F)}; // −1 on the real parts
697 for (std::size_t i {0}; i < Rows; ++i) {
698 float* const row {c + (2 * i * ld)};
699 const __m512 lo {_mm512_fmadd_ps(_mm512_permute_ps(acc[(4 * i) + 2].v, 0xB1), signs, acc[4 * i].v)};
700 const __m512 hi {_mm512_fmadd_ps(_mm512_permute_ps(acc[(4 * i) + 3].v, 0xB1), signs, acc[(4 * i) + 1].v)};
703 }
704 }
705
707 inline void avx512_tile_c32(std::size_t kc,
708 const std::complex<float>* ap,
709 const std::complex<float>* bp,
710 std::size_t ldb,
711 std::complex<float>* c,
712 std::size_t ldc,
713 std::size_t rows,
714 std::size_t cols,
715 bool fresh = false) noexcept
716 {
717 through_rows<avx512_complex_mr, avx512_complex_nr_f32>(c, ldc, rows, cols, [&]<std::size_t Rows>(std::complex<float>* tile, std::size_t ld) noexcept {
719 });
720 }
721
724 inline void avx512_tile_c32_a_in_place(std::size_t kc,
725 const std::complex<float>* a,
726 std::size_t a_step,
727 std::size_t a_row,
728 const std::complex<float>* bp,
729 std::size_t ldb,
730 std::complex<float> * c,
731 std::size_t ldc,
732 std::size_t rows,
733 std::size_t cols,
734 bool fresh = false) noexcept
735 {
736 through_rows<avx512_complex_mr, avx512_complex_nr_f32>(c, ldc, rows, cols, [&]<std::size_t Rows>(std::complex<float>* tile, std::size_t ld) noexcept {
738 });
739 }
740
747 NUMERICAL_TARGET("avx2") inline double lanes_sum(__m256d lo,
748 __m256d hi) noexcept
749 {
750 const __m256d pairs {_mm256_add_pd(lo, hi)}; // v_k + v_(k+4)
753 }
754
757 {
759 }
760
765 NUMERICAL_TARGET("avx2") inline float lanes_sum(__m256 lo,
766 __m256 hi) noexcept
767 {
768 const __m256 pairs {_mm256_add_ps(lo, hi)}; // p_k
770 const __m128 twos {_mm_add_ps(quads, _mm_movehl_ps(quads, quads))}; // + p_(k+2)…
772 }
773
776 {
778 }
779
781 inline double lanes_sum(const double* lane,
782 std::size_t apart) noexcept
783 {
784 const auto v {[&](std::size_t k) { return lane[k * apart]; }};
785 return ((v(0) + v(4)) + (v(2) + v(6))) + ((v(1) + v(5)) + (v(3) + v(7)));
786 }
787
789 inline float lanes_sum(const float* lane,
790 std::size_t apart) noexcept
791 {
792 const auto p {[&](std::size_t k) { return lane[k * apart] + lane[(k + 8) * apart]; }};
793 return ((p(0) + p(4)) + (p(2) + p(6))) + ((p(1) + p(5)) + (p(3) + p(7)));
794 }
795
801 NUMERICAL_TARGET("avx512f") inline double avx512_symmetric_row(const double* row,
802 const double* v,
803 double f,
804 double* out,
805 std::size_t count) noexcept
806 {
807 constexpr std::size_t width {8};
808 constexpr std::size_t lanes {4};
809 const __m512d splat {_mm512_set1_pd(f)};
810 std::array<zmm_register, lanes> sum {};
811 const std::size_t done {count - (count % (width * lanes))};
812 for (std::size_t i {0}; i < done; i += width * lanes) {
814 for (std::size_t l {0}; l < lanes; ++l) {
815 const __m512d r {_mm512_loadu_pd(row + i + (l * width))};
816 sum[l].v = _mm512_fmadd_pd(r, _mm512_loadu_pd(v + i + (l * width)), sum[l].v);
817 _mm512_storeu_pd(out + i + (l * width), _mm512_fmadd_pd(r, splat, _mm512_loadu_pd(out + i + (l * width))));
818 }
819 }
820 double total {0.0};
821 for (std::size_t l {0}; l < lanes; ++l) {
822 total += lanes_sum(sum[l].v);
823 }
824 for (std::size_t i {done}; i < count; ++i) {
825 total += row[i] * v[i];
826 out[i] += row[i] * f;
827 }
828 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
829 return total;
830 }
831
834 NUMERICAL_TARGET("avx2,fma") inline double avx2_symmetric_row(const double* row,
835 const double* v,
836 double f,
837 double* out,
838 std::size_t count) noexcept
839 {
840 constexpr std::size_t width {4};
841 constexpr std::size_t lanes {8};
842 const __m256d splat {_mm256_set1_pd(f)};
843 std::array<ymm_register, lanes> sum {};
844 const std::size_t done {count - (count % (width * lanes))};
845 for (std::size_t i {0}; i < done; i += width * lanes) {
847 for (std::size_t l {0}; l < lanes; ++l) {
848 const __m256d r {_mm256_loadu_pd(row + i + (l * width))};
849 sum[l].v = _mm256_fmadd_pd(r, _mm256_loadu_pd(v + i + (l * width)), sum[l].v);
850 _mm256_storeu_pd(out + i + (l * width), _mm256_fmadd_pd(r, splat, _mm256_loadu_pd(out + i + (l * width))));
851 }
852 }
853 double total {0.0};
854 for (std::size_t l {0}; l < lanes; l += 2) {
855 total += lanes_sum(sum[l].v, sum[l + 1].v);
856 }
857 for (std::size_t i {done}; i < count; ++i) {
858 total += row[i] * v[i];
859 out[i] += row[i] * f;
860 }
861 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
862 return total;
863 }
864
869 NUMERICAL_TARGET("avx512f") inline std::array<double, symmetric_row_group> avx512_symmetric_rows(const double* rows,
870 std::size_t n,
871 const double* v,
872 double* out,
873 std::size_t i) noexcept
874 {
875 constexpr std::size_t width {8};
876 std::array<zmm_register, symmetric_row_group> sum {};
877 std::array<zmm_register, symmetric_row_group> splat {};
878 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
879 splat[r].v = _mm512_set1_pd(v[i + r]);
880 }
881 const std::size_t done {i - (i % width)};
882 for (std::size_t j {0}; j < done; j += width) {
883 const __m512d vv {_mm512_loadu_pd(v + j)};
884 __m512d ov {_mm512_loadu_pd(out + j)};
886 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
887 const __m512d rv {_mm512_loadu_pd(rows + (r * n) + j)};
888 sum[r].v = _mm512_fmadd_pd(rv, vv, sum[r].v);
889 ov = _mm512_fmadd_pd(rv, splat[r].v, ov);
890 }
891 _mm512_storeu_pd(out + j, ov);
892 }
893 std::array<double, symmetric_row_group> sums {};
894 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
895 sums[r] = lanes_sum(sum[r].v);
896 }
897 _mm256_zeroupper(); // the upper halves cleared: the tail may be SSE alone
898 return symmetric_rows_tail(rows, n, v, out, i, done, sums);
899 }
900
903 NUMERICAL_TARGET("avx2,fma") inline std::array<double, symmetric_row_group> avx2_symmetric_rows(const double* rows,
904 std::size_t n,
905 const double* v,
906 double* out,
907 std::size_t i) noexcept
908 {
909 constexpr std::size_t width {8};
910 constexpr std::size_t half {4};
911 std::array<ymm_register, symmetric_row_group> low {};
912 std::array<ymm_register, symmetric_row_group> high {};
913 std::array<ymm_register, symmetric_row_group> splat {};
914 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
915 splat[r].v = _mm256_set1_pd(v[i + r]);
916 }
917 const std::size_t done {i - (i % width)};
918 for (std::size_t j {0}; j < done; j += width) {
919 const __m256d vl {_mm256_loadu_pd(v + j)};
920 const __m256d vh {_mm256_loadu_pd(v + j + half)};
921 __m256d ol {_mm256_loadu_pd(out + j)};
922 __m256d oh {_mm256_loadu_pd(out + j + half)};
924 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
925 const __m256d rl {_mm256_loadu_pd(rows + (r * n) + j)};
926 const __m256d rh {_mm256_loadu_pd(rows + (r * n) + j + half)};
927 low[r].v = _mm256_fmadd_pd(rl, vl, low[r].v);
928 high[r].v = _mm256_fmadd_pd(rh, vh, high[r].v);
929 ol = _mm256_fmadd_pd(rl, splat[r].v, ol);
930 oh = _mm256_fmadd_pd(rh, splat[r].v, oh);
931 }
932 _mm256_storeu_pd(out + j, ol);
933 _mm256_storeu_pd(out + j + half, oh);
934 }
935 std::array<double, symmetric_row_group> sums {};
936 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
937 sums[r] = lanes_sum(low[r].v, high[r].v);
938 }
939 _mm256_zeroupper(); // the upper halves cleared: the tail may be SSE alone
940 return symmetric_rows_tail(rows, n, v, out, i, done, sums);
941 }
942
945 NUMERICAL_TARGET("avx512f") inline void avx512_scaled_add(double* y,
946 const double* x,
947 double f,
948 std::size_t count) noexcept
949 {
950 const __m512d splat {_mm512_set1_pd(f)};
951 const std::size_t done {count - (count % 8)};
952 for (std::size_t i {0}; i < done; i += 8) {
954 }
955 for (std::size_t i {done}; i < count; ++i) {
956 y[i] = std::fma(f, x[i], y[i]);
957 }
958 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
959 }
960
966 NUMERICAL_TARGET("avx512f") inline void avx512_subtract_products(double* y,
967 const double* f,
968 const double* x,
969 std::size_t ldx,
970 std::size_t terms,
971 std::size_t count) noexcept
972 {
973 std::size_t i {0};
974 for (; i + 32 <= count; i += 32) {
976 __m512d y1 {_mm512_loadu_pd(y + i + 8)};
977 __m512d y2 {_mm512_loadu_pd(y + i + 16)};
978 __m512d y3 {_mm512_loadu_pd(y + i + 24)};
979 for (std::size_t t {0}; t < terms; ++t) {
980 const __m512d minus {_mm512_set1_pd(-f[t])};
981 const double* const row {x + (t * ldx) + i};
983 y1 = _mm512_fmadd_pd(minus, _mm512_loadu_pd(row + 8), y1);
984 y2 = _mm512_fmadd_pd(minus, _mm512_loadu_pd(row + 16), y2);
985 y3 = _mm512_fmadd_pd(minus, _mm512_loadu_pd(row + 24), y3);
986 }
987 _mm512_storeu_pd(y + i, y0);
988 _mm512_storeu_pd(y + i + 8, y1);
989 _mm512_storeu_pd(y + i + 16, y2);
990 _mm512_storeu_pd(y + i + 24, y3);
991 }
992 for (; i + 8 <= count; i += 8) {
994 for (std::size_t t {0}; t < terms; ++t) {
995 sum = _mm512_fmadd_pd(_mm512_set1_pd(-f[t]), _mm512_loadu_pd(x + (t * ldx) + i), sum);
996 }
997 _mm512_storeu_pd(y + i, sum);
998 }
999 for (; i < count; ++i) {
1000 for (std::size_t t {0}; t < terms; ++t) {
1001 y[i] = std::fma(-f[t], x[(t * ldx) + i], y[i]);
1002 }
1003 }
1004 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1005 }
1006
1012 NUMERICAL_TARGET("avx512f") inline void avx512_subtract_rows(double* y,
1013 std::size_t ldy,
1014 const double* f,
1015 std::size_t ldf,
1016 const double* x,
1017 std::size_t ldx,
1018 std::size_t terms,
1019 std::size_t count) noexcept
1020 {
1021 struct lane // a register in a struct: an array of the register type itself loses its alignment attribute
1022 {
1023 __m512d value;
1024 };
1025 for (std::size_t i {0}; i < count; i += 8) {
1026 const auto mask {static_cast<__mmask8>(count - i >= 8 ? 0xFFU : (1U << (count - i)) - 1U)};
1027 std::array<lane, 8> held {};
1028 for (std::size_t r {0}; r < 8; ++r) {
1029 held[r].value = _mm512_maskz_loadu_pd(mask, y + (r * ldy) + i);
1030 }
1031 for (std::size_t t {0}; t < terms; ++t) {
1032 const __m512d row {_mm512_maskz_loadu_pd(mask, x + (t * ldx) + i)};
1033 for (std::size_t r {0}; r < 8; ++r) {
1034 held[r].value = _mm512_fmadd_pd(_mm512_set1_pd(-f[(r * ldf) + t]), row, held[r].value);
1035 }
1036 }
1037 for (std::size_t r {0}; r < 8; ++r) {
1038 _mm512_mask_storeu_pd(y + (r * ldy) + i, mask, held[r].value);
1039 }
1040 }
1041 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1042 }
1043
1045 NUMERICAL_TARGET("avx2,fma") inline void avx2_subtract_rows(double* y,
1046 std::size_t ldy,
1047 const double* f,
1048 std::size_t ldf,
1049 const double* x,
1050 std::size_t ldx,
1051 std::size_t terms,
1052 std::size_t count) noexcept
1053 {
1054 struct lane // as avx512_subtract_rows's
1055 {
1056 __m256d value;
1057 };
1058 std::size_t i {0};
1059 for (; i + 4 <= count; i += 4) {
1060 std::array<lane, 8> held {};
1061 for (std::size_t r {0}; r < 8; ++r) {
1062 held[r].value = _mm256_loadu_pd(y + (r * ldy) + i);
1063 }
1064 for (std::size_t t {0}; t < terms; ++t) {
1065 const __m256d row {_mm256_loadu_pd(x + (t * ldx) + i)};
1066 for (std::size_t r {0}; r < 8; ++r) {
1067 held[r].value = _mm256_fmadd_pd(_mm256_set1_pd(-f[(r * ldf) + t]), row, held[r].value);
1068 }
1069 }
1070 for (std::size_t r {0}; r < 8; ++r) {
1071 _mm256_storeu_pd(y + (r * ldy) + i, held[r].value);
1072 }
1073 }
1074 for (; i < count; ++i) {
1075 std::array<double, 8> held {};
1076 for (std::size_t r {0}; r < 8; ++r) {
1077 held[r] = y[(r * ldy) + i];
1078 }
1079 for (std::size_t t {0}; t < terms; ++t) {
1080 const double value {x[(t * ldx) + i]};
1081 for (std::size_t r {0}; r < 8; ++r) {
1082 held[r] = std::fma(-f[(r * ldf) + t], value, held[r]);
1083 }
1084 }
1085 for (std::size_t r {0}; r < 8; ++r) {
1086 y[(r * ldy) + i] = held[r];
1087 }
1088 }
1089 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1090 }
1091
1093 NUMERICAL_TARGET("avx2,fma") inline void avx2_subtract_products(double* y,
1094 const double* f,
1095 const double* x,
1096 std::size_t ldx,
1097 std::size_t terms,
1098 std::size_t count) noexcept
1099 {
1100 std::size_t i {0};
1101 for (; i + 16 <= count; i += 16) {
1102 __m256d y0 {_mm256_loadu_pd(y + i)};
1103 __m256d y1 {_mm256_loadu_pd(y + i + 4)};
1104 __m256d y2 {_mm256_loadu_pd(y + i + 8)};
1105 __m256d y3 {_mm256_loadu_pd(y + i + 12)};
1106 for (std::size_t t {0}; t < terms; ++t) {
1107 const __m256d minus {_mm256_set1_pd(-f[t])};
1108 const double* const row {x + (t * ldx) + i};
1110 y1 = _mm256_fmadd_pd(minus, _mm256_loadu_pd(row + 4), y1);
1111 y2 = _mm256_fmadd_pd(minus, _mm256_loadu_pd(row + 8), y2);
1112 y3 = _mm256_fmadd_pd(minus, _mm256_loadu_pd(row + 12), y3);
1113 }
1114 _mm256_storeu_pd(y + i, y0);
1115 _mm256_storeu_pd(y + i + 4, y1);
1116 _mm256_storeu_pd(y + i + 8, y2);
1117 _mm256_storeu_pd(y + i + 12, y3);
1118 }
1119 for (; i + 4 <= count; i += 4) {
1121 for (std::size_t t {0}; t < terms; ++t) {
1122 sum = _mm256_fmadd_pd(_mm256_set1_pd(-f[t]), _mm256_loadu_pd(x + (t * ldx) + i), sum);
1123 }
1124 _mm256_storeu_pd(y + i, sum);
1125 }
1126 for (; i < count; ++i) {
1127 for (std::size_t t {0}; t < terms; ++t) {
1128 y[i] = std::fma(-f[t], x[(t * ldx) + i], y[i]);
1129 }
1130 }
1131 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1132 }
1133
1141 const double* f,
1142 const double* x,
1143 std::size_t ldx,
1144 std::size_t terms,
1145 std::size_t count) noexcept
1146 {
1147 const __m512d sign {_mm512_set_pd(1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0)}; // (−, +) for each number
1148 std::size_t i {0};
1149 for (; i + 16 <= count; i += 16) {
1150 std::array<zmm_register, 4> acc {};
1152 for (std::size_t r {0}; r < 4; ++r) {
1153 acc[r].v = _mm512_loadu_pd(y + (2 * i) + (8 * r));
1154 }
1155 for (std::size_t t {0}; t < terms; ++t) {
1156 const __m512d real {_mm512_set1_pd(f[2 * t])};
1157 const __m512d imag {_mm512_mul_pd(_mm512_set1_pd(f[(2 * t) + 1]), sign)};
1158 const double* const row {x + (2 * ((t * ldx) + i))};
1160 for (std::size_t r {0}; r < 4; ++r) {
1161 const __m512d value {_mm512_loadu_pd(row + (8 * r))};
1162 acc[r].v = _mm512_fnmadd_pd(real, value, acc[r].v);
1163 acc[r].v = _mm512_fnmadd_pd(imag, _mm512_permute_pd(value, 0x55), acc[r].v);
1164 }
1165 }
1167 for (std::size_t r {0}; r < 4; ++r) {
1168 _mm512_storeu_pd(y + (2 * i) + (8 * r), acc[r].v);
1169 }
1170 }
1171 for (; i + 4 <= count; i += 4) {
1172 __m512d sum {_mm512_loadu_pd(y + (2 * i))};
1173 for (std::size_t t {0}; t < terms; ++t) {
1174 const __m512d value {_mm512_loadu_pd(x + (2 * ((t * ldx) + i)))};
1175 sum = _mm512_fnmadd_pd(_mm512_set1_pd(f[2 * t]), value, sum);
1176 sum = _mm512_fnmadd_pd(_mm512_mul_pd(_mm512_set1_pd(f[(2 * t) + 1]), sign), _mm512_permute_pd(value, 0x55), sum);
1177 }
1178 _mm512_storeu_pd(y + (2 * i), sum);
1179 }
1180 for (; i < count; ++i) {
1181 double real {y[2 * i]};
1182 double imag {y[(2 * i) + 1]};
1183 for (std::size_t t {0}; t < terms; ++t) {
1184 const double xr {x[2 * ((t * ldx) + i)]};
1185 const double xi {x[(2 * ((t * ldx) + i)) + 1]};
1186 real = std::fma(-f[2 * t], xr, real);
1187 imag = std::fma(-f[2 * t], xi, imag);
1188 real = std::fma(f[(2 * t) + 1], xi, real);
1189 imag = std::fma(-f[(2 * t) + 1], xr, imag);
1190 }
1191 y[2 * i] = real;
1192 y[(2 * i) + 1] = imag;
1193 }
1194 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1195 }
1196
1199 const double* f,
1200 const double* x,
1201 std::size_t ldx,
1202 std::size_t terms,
1203 std::size_t count) noexcept
1204 {
1205 const __m256d sign {_mm256_set_pd(1.0, -1.0, 1.0, -1.0)};
1206 std::size_t i {0};
1207 for (; i + 8 <= count; i += 8) {
1208 std::array<ymm_register, 4> acc {};
1210 for (std::size_t r {0}; r < 4; ++r) {
1211 acc[r].v = _mm256_loadu_pd(y + (2 * i) + (4 * r));
1212 }
1213 for (std::size_t t {0}; t < terms; ++t) {
1214 const __m256d real {_mm256_set1_pd(f[2 * t])};
1215 const __m256d imag {_mm256_mul_pd(_mm256_set1_pd(f[(2 * t) + 1]), sign)};
1216 const double* const row {x + (2 * ((t * ldx) + i))};
1218 for (std::size_t r {0}; r < 4; ++r) {
1219 const __m256d value {_mm256_loadu_pd(row + (4 * r))};
1220 acc[r].v = _mm256_fnmadd_pd(real, value, acc[r].v);
1221 acc[r].v = _mm256_fnmadd_pd(imag, _mm256_permute_pd(value, 0x5), acc[r].v);
1222 }
1223 }
1225 for (std::size_t r {0}; r < 4; ++r) {
1226 _mm256_storeu_pd(y + (2 * i) + (4 * r), acc[r].v);
1227 }
1228 }
1229 for (; i + 2 <= count; i += 2) {
1230 __m256d sum {_mm256_loadu_pd(y + (2 * i))};
1231 for (std::size_t t {0}; t < terms; ++t) {
1232 const __m256d value {_mm256_loadu_pd(x + (2 * ((t * ldx) + i)))};
1233 sum = _mm256_fnmadd_pd(_mm256_set1_pd(f[2 * t]), value, sum);
1234 sum = _mm256_fnmadd_pd(_mm256_mul_pd(_mm256_set1_pd(f[(2 * t) + 1]), sign), _mm256_permute_pd(value, 0x5), sum);
1235 }
1236 _mm256_storeu_pd(y + (2 * i), sum);
1237 }
1238 for (; i < count; ++i) {
1239 double real {y[2 * i]};
1240 double imag {y[(2 * i) + 1]};
1241 for (std::size_t t {0}; t < terms; ++t) {
1242 const double xr {x[2 * ((t * ldx) + i)]};
1243 const double xi {x[(2 * ((t * ldx) + i)) + 1]};
1244 real = std::fma(-f[2 * t], xr, real);
1245 imag = std::fma(-f[2 * t], xi, imag);
1246 real = std::fma(f[(2 * t) + 1], xi, real);
1247 imag = std::fma(-f[(2 * t) + 1], xr, imag);
1248 }
1249 y[2 * i] = real;
1250 y[(2 * i) + 1] = imag;
1251 }
1252 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1253 }
1254
1261 NUMERICAL_TARGET("avx512f") inline std::complex<double> avx512_hermitian_row(const double* row,
1262 const double* v,
1263 double vr,
1264 double vi,
1265 double* out,
1266 std::size_t count) noexcept
1267 {
1268 const __m512d real {_mm512_set_pd(-vr, vr, -vr, vr, -vr, vr, -vr, vr)}; // (vr, −vr) for each number
1269 const __m512d imag {_mm512_set1_pd(vi)};
1270 __m512d same {_mm512_setzero_pd()}; // rr·vr, ri·vi
1271 __m512d across {_mm512_setzero_pd()}; // rr·vi, ri·vr
1272 std::size_t j {0};
1273 for (; j + 4 <= count; j += 4) {
1274 const __m512d r {_mm512_loadu_pd(row + (2 * j))};
1275 const __m512d x {_mm512_loadu_pd(v + (2 * j))};
1276 const __m512d s {_mm512_permute_pd(r, 0x55)};
1277 __m512d o {_mm512_loadu_pd(out + (2 * j))};
1278 o = _mm512_fmadd_pd(real, r, o);
1279 o = _mm512_fmadd_pd(imag, s, o);
1280 same = _mm512_fmadd_pd(r, x, same);
1281 across = _mm512_fmadd_pd(s, x, across);
1282 _mm512_storeu_pd(out + (2 * j), o);
1283 }
1284 // Registers stored whole: GCC 14 stores the second aligned (in C++20, under the address sanitizer), so both are.
1285 alignas(64) std::array<double, 8> lanes_same {};
1286 alignas(64) std::array<double, 8> lanes_across {};
1289 double re {0.0};
1290 double im {0.0};
1291 for (std::size_t l {0}; l < 8; l += 2) {
1292 re += lanes_same[l] - lanes_same[l + 1];
1293 im += lanes_across[l] + lanes_across[l + 1];
1294 }
1295 for (; j < count; ++j) {
1296 const double rr {row[2 * j]};
1297 const double ri {row[(2 * j) + 1]};
1298 out[2 * j] = std::fma(vi, ri, std::fma(vr, rr, out[2 * j]));
1299 out[(2 * j) + 1] = std::fma(vi, rr, std::fma(-vr, ri, out[(2 * j) + 1]));
1300 re += (rr * v[2 * j]) - (ri * v[(2 * j) + 1]);
1301 im += (ri * v[2 * j]) + (rr * v[(2 * j) + 1]);
1302 }
1303 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1304 return std::complex<double> {re, im};
1305 }
1306
1309 NUMERICAL_TARGET("avx2,fma") inline std::complex<double> avx2_hermitian_row(const double* row,
1310 const double* v,
1311 double vr,
1312 double vi,
1313 double* out,
1314 std::size_t count) noexcept
1315 {
1316 const __m256d real {_mm256_set_pd(-vr, vr, -vr, vr)};
1317 const __m256d imag {_mm256_set1_pd(vi)};
1318 std::array<ymm_register, 2> same {};
1319 std::array<ymm_register, 2> across {};
1320 std::size_t j {0};
1321 for (; j + 4 <= count; j += 4) {
1322 for (std::size_t h {0}; h < 2; ++h) {
1323 const std::size_t at {2 * (j + (2 * h))};
1324 const __m256d r {_mm256_loadu_pd(row + at)};
1325 const __m256d x {_mm256_loadu_pd(v + at)};
1326 const __m256d s {_mm256_permute_pd(r, 0x5)};
1327 __m256d o {_mm256_loadu_pd(out + at)};
1328 o = _mm256_fmadd_pd(real, r, o);
1329 o = _mm256_fmadd_pd(imag, s, o);
1330 same[h].v = _mm256_fmadd_pd(r, x, same[h].v);
1331 across[h].v = _mm256_fmadd_pd(s, x, across[h].v);
1332 _mm256_storeu_pd(out + at, o);
1333 }
1334 }
1335 alignas(64) std::array<double, 8> lanes_same {}; // registers stored whole, aligned (as above)
1336 alignas(64) std::array<double, 8> lanes_across {};
1337 for (std::size_t h {0}; h < 2; ++h) {
1338 _mm256_storeu_pd(lanes_same.data() + (4 * h), same[h].v);
1339 _mm256_storeu_pd(lanes_across.data() + (4 * h), across[h].v);
1340 }
1341 double re {0.0};
1342 double im {0.0};
1343 for (std::size_t l {0}; l < 8; l += 2) {
1344 re += lanes_same[l] - lanes_same[l + 1];
1345 im += lanes_across[l] + lanes_across[l + 1];
1346 }
1347 for (; j < count; ++j) {
1348 const double rr {row[2 * j]};
1349 const double ri {row[(2 * j) + 1]};
1350 out[2 * j] = std::fma(vi, ri, std::fma(vr, rr, out[2 * j]));
1351 out[(2 * j) + 1] = std::fma(vi, rr, std::fma(-vr, ri, out[(2 * j) + 1]));
1352 re += (rr * v[2 * j]) - (ri * v[(2 * j) + 1]);
1353 im += (ri * v[2 * j]) + (rr * v[(2 * j) + 1]);
1354 }
1355 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1356 return std::complex<double> {re, im};
1357 }
1358
1366 template <bool Maximum>
1367 NUMERICAL_TARGET("avx2") inline std::size_t avx2_extreme(const double* first,
1368 std::size_t count,
1369 double& result,
1370 bool& nan) noexcept
1371 {
1372 constexpr std::size_t block {16};
1373 const std::size_t done {count - (count % block)};
1374 if (done == 0) {
1375 return 0;
1376 }
1377 std::array<ymm_register, 4> best {};
1378 std::array<ymm_register, 4> nans {};
1379 for (std::size_t l {0}; l < 4; ++l) {
1380 best[l].v = _mm256_loadu_pd(first + (l * 4));
1381 nans[l].v = _mm256_setzero_pd();
1382 }
1383 for (std::size_t i {0}; i < done; i += block) {
1384 for (std::size_t l {0}; l < 4; ++l) {
1385 const __m256d x {_mm256_loadu_pd(first + i + (l * 4))};
1386 best[l].v = Maximum ? _mm256_max_pd(x, best[l].v) : _mm256_min_pd(x, best[l].v);
1387 nans[l].v = _mm256_or_pd(nans[l].v, _mm256_cmp_pd(x, x, _CMP_UNORD_Q));
1388 }
1389 }
1390 std::array<double, block> values {};
1391 std::array<double, block> flags {};
1392 for (std::size_t l {0}; l < 4; ++l) {
1393 _mm256_storeu_pd(values.data() + (l * 4), best[l].v);
1394 _mm256_storeu_pd(flags.data() + (l * 4), nans[l].v);
1395 }
1396 _mm256_zeroupper(); // the upper halves cleared: the tail may be SSE alone
1397 for (std::size_t k {0}; k < block; ++k) {
1398 nan = nan || std::bit_cast<std::uint64_t>(flags[k]) != 0U;
1399 result = (Maximum ? values[k] > result : values[k] < result) ? values[k] : result;
1400 }
1401 return done;
1402 }
1403
1405 template <bool Maximum>
1406 NUMERICAL_TARGET("avx2") inline std::size_t avx2_extreme(const float* first,
1407 std::size_t count,
1408 float& result,
1409 bool& nan) noexcept
1410 {
1411 constexpr std::size_t block {32};
1412 const std::size_t done {count - (count % block)};
1413 if (done == 0) {
1414 return 0;
1415 }
1416 std::array<ymm_register_f32, 4> best {};
1417 std::array<ymm_register_f32, 4> nans {};
1418 for (std::size_t l {0}; l < 4; ++l) {
1419 best[l].v = _mm256_loadu_ps(first + (l * 8));
1420 nans[l].v = _mm256_setzero_ps();
1421 }
1422 for (std::size_t i {0}; i < done; i += block) {
1423 for (std::size_t l {0}; l < 4; ++l) {
1424 const __m256 x {_mm256_loadu_ps(first + i + (l * 8))};
1425 best[l].v = Maximum ? _mm256_max_ps(x, best[l].v) : _mm256_min_ps(x, best[l].v);
1426 nans[l].v = _mm256_or_ps(nans[l].v, _mm256_cmp_ps(x, x, _CMP_UNORD_Q));
1427 }
1428 }
1429 std::array<float, block> values {};
1430 std::array<float, block> flags {};
1431 for (std::size_t l {0}; l < 4; ++l) {
1432 _mm256_storeu_ps(values.data() + (l * 8), best[l].v);
1433 _mm256_storeu_ps(flags.data() + (l * 8), nans[l].v);
1434 }
1435 _mm256_zeroupper(); // the upper halves cleared: the tail may be SSE alone
1436 for (std::size_t k {0}; k < block; ++k) {
1437 nan = nan || std::bit_cast<std::uint32_t>(flags[k]) != 0U;
1438 result = (Maximum ? values[k] > result : values[k] < result) ? values[k] : result;
1439 }
1440 return done;
1441 }
1442
1444 NUMERICAL_TARGET("avx2,fma") inline void avx2_scaled_add(double* y,
1445 const double* x,
1446 double f,
1447 std::size_t count) noexcept
1448 {
1449 const __m256d splat {_mm256_set1_pd(f)};
1450 const std::size_t done {count - (count % 4)};
1451 for (std::size_t i {0}; i < done; i += 4) {
1453 }
1454 for (std::size_t i {done}; i < count; ++i) {
1455 y[i] = std::fma(f, x[i], y[i]);
1456 }
1457 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1458 }
1459
1461 NUMERICAL_TARGET("avx2,fma") inline void avx2_scaled_add(float* y,
1462 const float* x,
1463 float f,
1464 std::size_t count) noexcept
1465 {
1466 const __m256 splat {_mm256_set1_ps(f)};
1467 const std::size_t done {count - (count % 8)};
1468 for (std::size_t i {0}; i < done; i += 8) {
1470 }
1471 for (std::size_t i {done}; i < count; ++i) {
1472 y[i] = std::fma(f, x[i], y[i]);
1473 }
1474 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1475 }
1476
1483 NUMERICAL_TARGET("avx2") inline std::size_t avx2_scaled_add_c32(std::complex<float>* y,
1484 const std::complex<float>* x,
1485 std::complex<float> f,
1486 std::size_t count) noexcept
1487 {
1488 auto* const out {reinterpret_cast<float*>(y)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee of [complex.numbers]
1489 const auto* const in {reinterpret_cast<const float*>(x)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
1490 const __m256 re {_mm256_set1_ps(f.real())};
1491 const __m256 im {_mm256_set1_ps(f.imag())};
1492 const std::size_t done {count - (count % 4)};
1493 for (std::size_t i {0}; i < 2 * done; i += 8) {
1494 const __m256 xv {_mm256_loadu_ps(in + i)};
1497 }
1498 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1499 return done;
1500 }
1501
1503 NUMERICAL_TARGET("avx2") inline std::size_t avx2_scaled_add_c64(std::complex<double>* y,
1504 const std::complex<double>* x,
1505 std::complex<double> f,
1506 std::size_t count) noexcept
1507 {
1508 auto* const out {reinterpret_cast<double*>(y)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee of [complex.numbers]
1509 const auto* const in {reinterpret_cast<const double*>(x)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
1510 const __m256d re {_mm256_set1_pd(f.real())};
1511 const __m256d im {_mm256_set1_pd(f.imag())};
1512 const std::size_t done {count - (count % 2)};
1513 for (std::size_t i {0}; i < 2 * done; i += 4) {
1514 const __m256d xv {_mm256_loadu_pd(in + i)};
1517 }
1518 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1519 return done;
1520 }
1521
1523 template <std::size_t Width,
1524 typename T>
1526 const T* b,
1527 std::size_t row_step,
1528 std::size_t column_step,
1529 std::size_t inner,
1530 std::size_t j,
1531 T* y) noexcept
1532 {
1533 std::array<T, Width> sums {};
1534 for (std::size_t k {0}; k < inner; ++k) {
1535 const T f {a[k]};
1536 const T* const row {b + (k * row_step) + (j * column_step)};
1537 for (std::size_t c {0}; c < Width; ++c) {
1538 sums[c] = std::fma(f, row[c * column_step], sums[c]);
1539 }
1540 }
1541 std::copy(sums.begin(), sums.end(), y + j);
1542 }
1543
1551 template <typename T>
1553 const T* b,
1554 std::size_t row_step,
1555 std::size_t column_step,
1556 std::size_t inner,
1557 std::size_t first,
1558 std::size_t last,
1559 T* y) noexcept
1560 {
1561 constexpr std::size_t group {8}; // the switch below names each narrower last group
1562 std::size_t j {first};
1563 for (; j + group <= last; j += group) {
1565 }
1566 switch (last - j) {
1567 case 1: avx2_strided_column_group<1>(a, b, row_step, column_step, inner, j, y); break;
1568 case 2: avx2_strided_column_group<2>(a, b, row_step, column_step, inner, j, y); break;
1569 case 3: avx2_strided_column_group<3>(a, b, row_step, column_step, inner, j, y); break;
1570 case 4: avx2_strided_column_group<4>(a, b, row_step, column_step, inner, j, y); break;
1571 case 5: avx2_strided_column_group<5>(a, b, row_step, column_step, inner, j, y); break;
1572 case 6: avx2_strided_column_group<6>(a, b, row_step, column_step, inner, j, y); break;
1573 case 7: avx2_strided_column_group<7>(a, b, row_step, column_step, inner, j, y); break;
1574 default: break; // a whole number of groups
1575 }
1576 }
1577
1580 NUMERICAL_TARGET("avx512f") inline double avx512_dot_blocks(const double* x,
1581 const double* y,
1582 std::size_t done) noexcept
1583 {
1584 constexpr std::size_t width {8};
1585 constexpr std::size_t lanes {4};
1586 std::array<zmm_register, lanes> sum {};
1587 for (std::size_t i {0}; i < done; i += width * lanes) {
1589 for (std::size_t l {0}; l < lanes; ++l) {
1590 sum[l].v = _mm512_fmadd_pd(_mm512_loadu_pd(x + i + (l * width)), _mm512_loadu_pd(y + i + (l * width)), sum[l].v);
1591 }
1592 }
1593 double total {0.0};
1594 for (std::size_t l {0}; done != 0 && l < lanes; ++l) { // none summed: +0 left as it is
1595 total += lanes_sum(sum[l].v);
1596 }
1597 return total;
1598 }
1599
1601 NUMERICAL_TARGET("avx512f") inline double avx512_dot(const double* x,
1602 const double* y,
1603 std::size_t count) noexcept
1604 {
1605 const std::size_t done {count - (count % 32)};
1606 double total {avx512_dot_blocks(x, y, done)};
1607 for (std::size_t i {done}; i < count; ++i) {
1608 total += x[i] * y[i];
1609 }
1610 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1611 return total;
1612 }
1613
1616 NUMERICAL_TARGET("avx2,fma") inline double avx2_dot_blocks(const double* x,
1617 const double* y,
1618 std::size_t done) noexcept
1619 {
1620 constexpr std::size_t width {4};
1621 constexpr std::size_t lanes {8};
1622 std::array<ymm_register, lanes> sum {};
1623 for (std::size_t i {0}; i < done; i += width * lanes) {
1625 for (std::size_t l {0}; l < lanes; ++l) {
1626 sum[l].v = _mm256_fmadd_pd(_mm256_loadu_pd(x + i + (l * width)), _mm256_loadu_pd(y + i + (l * width)), sum[l].v);
1627 }
1628 }
1629 double total {0.0};
1630 for (std::size_t l {0}; done != 0 && l < lanes; l += 2) { // none summed: +0 left as it is
1631 total += lanes_sum(sum[l].v, sum[l + 1].v);
1632 }
1633 return total;
1634 }
1635
1637 NUMERICAL_TARGET("avx2,fma") inline double avx2_dot(const double* x,
1638 const double* y,
1639 std::size_t count) noexcept
1640 {
1641 const std::size_t done {count - (count % 32)};
1642 double total {avx2_dot_blocks(x, y, done)};
1643 for (std::size_t i {done}; i < count; ++i) {
1644 total += x[i] * y[i];
1645 }
1646 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1647 return total;
1648 }
1649
1652 NUMERICAL_TARGET("avx512f") inline float avx512_dot_blocks(const float* x,
1653 const float* y,
1654 std::size_t done) noexcept
1655 {
1656 constexpr std::size_t width {16};
1657 constexpr std::size_t lanes {4};
1658 std::array<zmm_register_f32, lanes> sum {};
1659 for (std::size_t i {0}; i < done; i += width * lanes) {
1661 for (std::size_t l {0}; l < lanes; ++l) {
1662 sum[l].v = _mm512_fmadd_ps(_mm512_loadu_ps(x + i + (l * width)), _mm512_loadu_ps(y + i + (l * width)), sum[l].v);
1663 }
1664 }
1665 float total {0.0F};
1666 for (std::size_t l {0}; done != 0 && l < lanes; ++l) { // none summed: +0 left as it is
1667 total += lanes_sum(sum[l].v);
1668 }
1669 return total;
1670 }
1671
1673 NUMERICAL_TARGET("avx512f") inline float avx512_dot(const float* x,
1674 const float* y,
1675 std::size_t count) noexcept
1676 {
1677 const std::size_t done {count - (count % 64)};
1678 float total {avx512_dot_blocks(x, y, done)};
1679 for (std::size_t i {done}; i < count; ++i) {
1680 total += x[i] * y[i];
1681 }
1682 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1683 return total;
1684 }
1685
1688 NUMERICAL_TARGET("avx2,fma") inline float avx2_dot_blocks(const float* x,
1689 const float* y,
1690 std::size_t done) noexcept
1691 {
1692 constexpr std::size_t width {8};
1693 constexpr std::size_t lanes {8};
1694 std::array<ymm_register_f32, lanes> sum {};
1695 for (std::size_t i {0}; i < done; i += width * lanes) {
1697 for (std::size_t l {0}; l < lanes; ++l) {
1698 sum[l].v = _mm256_fmadd_ps(_mm256_loadu_ps(x + i + (l * width)), _mm256_loadu_ps(y + i + (l * width)), sum[l].v);
1699 }
1700 }
1701 float total {0.0F};
1702 for (std::size_t l {0}; done != 0 && l < lanes; l += 2) { // none summed: +0 left as it is
1703 total += lanes_sum(sum[l].v, sum[l + 1].v);
1704 }
1705 return total;
1706 }
1707
1709 NUMERICAL_TARGET("avx2,fma") inline float avx2_dot(const float* x,
1710 const float* y,
1711 std::size_t count) noexcept
1712 {
1713 const std::size_t done {count - (count % 64)};
1714 float total {avx2_dot_blocks(x, y, done)};
1715 for (std::size_t i {done}; i < count; ++i) {
1716 total += x[i] * y[i];
1717 }
1718 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1719 return total;
1720 }
1721
1729 template <std::size_t Partials>
1730 NUMERICAL_TARGET("avx2,fma") inline void avx2_column_partials(const double* a,
1731 std::size_t lda,
1732 std::size_t rows,
1733 std::size_t done,
1734 const double* x,
1735 double* sums) noexcept
1736 {
1737 for (std::size_t j {0}; j < done; ++j) {
1738 double* const s {sums + ((j % Partials) * rows)};
1739 const double* const col {a + (j * lda)};
1740 const __m256d xv {_mm256_set1_pd(x[j])};
1741 std::size_t r {0};
1742 for (; r + 4 <= rows; r += 4) {
1744 }
1745 for (; r < rows; ++r) {
1746 s[r] = std::fma(col[r], x[j], s[r]);
1747 }
1748 }
1749 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1750 }
1751
1760 NUMERICAL_TARGET("avx2,fma") inline void columns_finish(const double* a,
1761 std::size_t lda,
1762 std::size_t rows,
1763 std::size_t done,
1764 std::size_t cols,
1765 const double* x,
1766 const double* sums,
1767 double* y) noexcept
1768 {
1769 constexpr std::size_t width {8};
1770 constexpr std::size_t lanes {4};
1771 for (std::size_t r {0}; r < rows; ++r) {
1772 double total {0.0};
1773 for (std::size_t l {0}; done != 0 && l < lanes; ++l) {
1774 total += lanes_sum(sums + (l * width * rows) + r, rows); // partial l·width + k of row r at k · rows from there
1775 }
1776 y[r] = total;
1777 }
1778 for (std::size_t j {done}; j < cols; ++j) {
1779 const double* const col {a + (j * lda)};
1780 for (std::size_t r {0}; r < rows; ++r) {
1781 y[r] += col[r] * x[j];
1782 }
1783 }
1784 }
1785
1791 template <std::size_t Partials>
1792 NUMERICAL_TARGET("avx2,fma") inline void avx2_column_partials(const float* a,
1793 std::size_t lda,
1794 std::size_t rows,
1795 std::size_t done,
1796 const float* x,
1797 float* sums) noexcept
1798 {
1799 for (std::size_t j {0}; j < done; ++j) {
1800 float* const s {sums + ((j % Partials) * rows)};
1801 const float* const col {a + (j * lda)};
1802 const __m256 xv {_mm256_set1_ps(x[j])};
1803 std::size_t r {0};
1804 for (; r + 8 <= rows; r += 8) {
1806 }
1807 for (; r < rows; ++r) {
1808 s[r] = std::fma(col[r], x[j], s[r]);
1809 }
1810 }
1811 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1812 }
1813
1816 NUMERICAL_TARGET("avx2,fma") inline void columns_finish(const float* a,
1817 std::size_t lda,
1818 std::size_t rows,
1819 std::size_t done,
1820 std::size_t cols,
1821 const float* x,
1822 const float* sums,
1823 float* y) noexcept
1824 {
1825 constexpr std::size_t width {16};
1826 constexpr std::size_t lanes {4};
1827 for (std::size_t r {0}; r < rows; ++r) {
1828 float total {0.0F};
1829 for (std::size_t l {0}; done != 0 && l < lanes; ++l) {
1830 total += lanes_sum(sums + (l * width * rows) + r, rows);
1831 }
1832 y[r] = total;
1833 }
1834 for (std::size_t j {done}; j < cols; ++j) {
1835 const float* const col {a + (j * lda)};
1836 for (std::size_t r {0}; r < rows; ++r) {
1837 y[r] += col[r] * x[j];
1838 }
1839 }
1840 }
1841
1848 template <typename T, std::size_t Width, T (*Blocks)(const T*, const T*, std::size_t) noexcept>
1850 std::size_t ld,
1851 std::size_t rows,
1852 const T* x,
1853 std::size_t count,
1854 T* out) noexcept
1855 {
1856 const std::size_t done {count - (count % Width)};
1857 std::size_t r {0};
1858 for (; r + 4 <= rows; r += 4) {
1859 const T* const m0 {m + (r * ld)};
1860 T t0 {done == 0 ? T {} : Blocks(m0, x, done)};
1861 T t1 {done == 0 ? T {} : Blocks(m0 + ld, x, done)};
1862 T t2 {done == 0 ? T {} : Blocks(m0 + (2 * ld), x, done)};
1863 T t3 {done == 0 ? T {} : Blocks(m0 + (3 * ld), x, done)};
1864 for (std::size_t i {done}; i < count; ++i) {
1865 t0 += m0[i] * x[i];
1866 t1 += m0[ld + i] * x[i];
1867 t2 += m0[(2 * ld) + i] * x[i];
1868 t3 += m0[(3 * ld) + i] * x[i];
1869 }
1870 out[r] = t0;
1871 out[r + 1] = t1;
1872 out[r + 2] = t2;
1873 out[r + 3] = t3;
1874 }
1875 for (; r < rows; ++r) {
1876 T total {Blocks(m + (r * ld), x, done)};
1877 for (std::size_t i {done}; i < count; ++i) {
1878 total += m[(r * ld) + i] * x[i];
1879 }
1880 out[r] = total;
1881 }
1882 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1883 }
1884
1892 NUMERICAL_TARGET("avx2,fma") inline std::complex<double> avx2_complex_dot(const std::complex<double>* x,
1893 const std::complex<double>* y,
1894 std::size_t count) noexcept
1895 {
1896 constexpr std::size_t width {4}; // parts in a register: two complex numbers
1897 constexpr std::size_t lanes {4};
1898 const auto* const a {reinterpret_cast<const double*>(x)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is an array of its two parts
1899 const auto* const b {reinterpret_cast<const double*>(y)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): as above
1900 const std::size_t parts {2 * count};
1901 std::array<ymm_register, lanes> same {};
1902 std::array<ymm_register, lanes> cross {};
1903 const std::size_t done {parts - (parts % (width * lanes))};
1904 for (std::size_t i {0}; i < done; i += width * lanes) {
1906 for (std::size_t l {0}; l < lanes; ++l) {
1907 const __m256d av {_mm256_loadu_pd(a + i + (l * width))};
1908 const __m256d bv {_mm256_loadu_pd(b + i + (l * width))};
1909 same[l].v = _mm256_fmadd_pd(av, bv, same[l].v);
1910 cross[l].v = _mm256_fmadd_pd(av, _mm256_permute_pd(bv, 0x5), cross[l].v); // each pair's parts swapped
1911 }
1912 }
1913 double rr {0.0};
1914 double ii {0.0};
1915 double ri {0.0};
1916 double ir {0.0};
1917 for (std::size_t l {0}; l < lanes; ++l) {
1918 alignas(32) std::array<double, width> s {}; // a register stored whole: some compilers store it aligned
1919 alignas(32) std::array<double, width> c {};
1920 _mm256_storeu_pd(s.data(), same[l].v);
1921 _mm256_storeu_pd(c.data(), cross[l].v);
1922 rr += s[0] + s[2];
1923 ii += s[1] + s[3];
1924 ri += c[0] + c[2];
1925 ir += c[1] + c[3];
1926 }
1927 for (std::size_t i {done}; i < parts; i += 2) {
1928 rr += a[i] * b[i];
1929 ii += a[i + 1] * b[i + 1];
1930 ri += a[i] * b[i + 1];
1931 ir += a[i + 1] * b[i];
1932 }
1933 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1934 return {rr - ii, ri + ir};
1935 }
1936
1938 NUMERICAL_TARGET("avx2,fma") inline std::complex<float> avx2_complex_dot(const std::complex<float>* x,
1939 const std::complex<float>* y,
1940 std::size_t count) noexcept
1941 {
1942 constexpr std::size_t width {8};
1943 constexpr std::size_t lanes {4};
1944 const auto* const a {reinterpret_cast<const float*>(x)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is an array of its two parts
1945 const auto* const b {reinterpret_cast<const float*>(y)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): as above
1946 const std::size_t parts {2 * count};
1947 std::array<ymm_register_f32, lanes> same {};
1948 std::array<ymm_register_f32, lanes> cross {};
1949 const std::size_t done {parts - (parts % (width * lanes))};
1950 for (std::size_t i {0}; i < done; i += width * lanes) {
1952 for (std::size_t l {0}; l < lanes; ++l) {
1953 const __m256 av {_mm256_loadu_ps(a + i + (l * width))};
1954 const __m256 bv {_mm256_loadu_ps(b + i + (l * width))};
1955 same[l].v = _mm256_fmadd_ps(av, bv, same[l].v);
1956 cross[l].v = _mm256_fmadd_ps(av, _mm256_permute_ps(bv, 0xB1), cross[l].v); // each pair's parts swapped
1957 }
1958 }
1959 float rr {0.0F};
1960 float ii {0.0F};
1961 float ri {0.0F};
1962 float ir {0.0F};
1963 for (std::size_t l {0}; l < lanes; ++l) {
1964 alignas(32) std::array<float, width> s {};
1965 alignas(32) std::array<float, width> c {};
1966 _mm256_storeu_ps(s.data(), same[l].v);
1967 _mm256_storeu_ps(c.data(), cross[l].v);
1968 rr += (s[0] + s[2]) + (s[4] + s[6]);
1969 ii += (s[1] + s[3]) + (s[5] + s[7]);
1970 ri += (c[0] + c[2]) + (c[4] + c[6]);
1971 ir += (c[1] + c[3]) + (c[5] + c[7]);
1972 }
1973 for (std::size_t i {done}; i < parts; i += 2) {
1974 rr += a[i] * b[i];
1975 ii += a[i + 1] * b[i + 1];
1976 ri += a[i] * b[i + 1];
1977 ir += a[i + 1] * b[i];
1978 }
1979 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
1980 return {rr - ii, ri + ir};
1981 }
1982
1986 template <typename C>
1988 std::size_t ld,
1989 std::size_t rows,
1990 const C* x,
1991 std::size_t count,
1992 C* out) noexcept
1993 {
1994 for (std::size_t r {0}; r < rows; ++r) {
1995 out[r] = avx2_complex_dot(m + (r * ld), x, count);
1996 }
1997 }
1998
2000 NUMERICAL_TARGET("avx2,fma") inline __m256d avx2_load(const double* p) noexcept
2001 {
2002 return _mm256_loadu_pd(p);
2003 }
2004
2007 {
2008 return _mm256_loadu_ps(p);
2009 }
2010
2012 NUMERICAL_TARGET("avx2,fma") inline void avx2_store(double* p,
2013 __m256d v) noexcept
2014 {
2015 _mm256_storeu_pd(p, v);
2016 }
2017
2019 NUMERICAL_TARGET("avx2,fma") inline void avx2_store(float* p,
2020 __m256 v) noexcept
2021 {
2022 _mm256_storeu_ps(p, v);
2023 }
2024
2027 __m256d v,
2028 __m256d s) noexcept
2029 {
2030 return _mm256_fmadd_pd(m, v, s);
2031 }
2032
2035 __m256 v,
2036 __m256 s) noexcept
2037 {
2038 return _mm256_fmadd_ps(m, v, s);
2039 }
2040
2042 NUMERICAL_TARGET("avx2,fma") inline __m256d avx2_pairs(double first,
2043 double second) noexcept
2044 {
2045 return _mm256_setr_pd(first, second, first, second);
2046 }
2047
2049 NUMERICAL_TARGET("avx2,fma") inline __m256 avx2_pairs(float first,
2050 float second) noexcept
2051 {
2052 return _mm256_setr_ps(first, second, first, second, first, second, first, second);
2053 }
2054
2064 template <std::size_t Slots,
2065 typename R>
2067 std::size_t lda,
2068 std::size_t rows,
2069 std::size_t done,
2070 const R* x,
2071 R* sums) noexcept
2072 {
2073 // Each slot its own sum over its columns in ascending order: a slot at a time over four registers of rows, its sums
2074 // in registers, each column read once and each slot written once.
2075 using vec = decltype(avx2_load(a)); // __m256d or __m256
2076 constexpr std::size_t width {32 / sizeof(R)}; // parts in a register
2077 constexpr std::size_t tile {4 * width}; // parts of the rows a slot takes at once
2078 const std::size_t parts {2 * rows};
2079 for (std::size_t q {0}; q < Slots; ++q) {
2080 R* const same {sums + (2 * q * parts)};
2081 R* const cross {same + parts};
2082 std::size_t p {0};
2083 for (; p + tile <= parts; p += tile) {
2084 using held = std::conditional_t<std::is_same_v<R, double>, ymm_register, ymm_register_f32>; // a register in an array
2085 std::array<held, 4> s {};
2086 std::array<held, 4> c {};
2087 for (std::size_t j {q}; j < done; j += Slots) {
2088 const R* const col {a + (2 * j * lda) + p};
2089 const vec xs {avx2_pairs(x[2 * j], x[(2 * j) + 1])};
2090 const vec xc {avx2_pairs(x[(2 * j) + 1], x[2 * j])};
2092 for (std::size_t t {0}; t < 4; ++t) {
2093 const vec mv {avx2_load(col + (t * width))};
2094 s[t].v = avx2_fused(mv, xs, s[t].v);
2095 c[t].v = avx2_fused(mv, xc, c[t].v);
2096 }
2097 }
2098 for (std::size_t t {0}; t < 4; ++t) {
2099 avx2_store(same + p + (t * width), s[t].v);
2100 avx2_store(cross + p + (t * width), c[t].v);
2101 }
2102 }
2103 for (; p < parts; p += 2) { // the rows past whole tiles: the same fused multiply-adds, one at a time
2104 for (std::size_t j {q}; j < done; j += Slots) {
2105 const R* const col {a + (2 * j * lda) + p};
2106 const R xr {x[2 * j]};
2107 const R xi {x[(2 * j) + 1]};
2108 same[p] = std::fma(col[0], xr, same[p]);
2109 same[p + 1] = std::fma(col[1], xi, same[p + 1]);
2110 cross[p] = std::fma(col[0], xi, cross[p]);
2111 cross[p + 1] = std::fma(col[1], xr, cross[p + 1]);
2112 }
2113 }
2114 }
2115 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
2116 }
2117
2124 template <std::size_t Slots,
2125 typename R>
2127 std::size_t lda,
2128 std::size_t rows,
2129 std::size_t done,
2130 std::size_t cols,
2131 const R* x,
2132 const R* sums,
2133 std::complex<R>* y) noexcept
2134 {
2135 constexpr std::size_t lanes {4};
2136 constexpr std::size_t per {Slots / lanes}; // the slots of one register: two complex numbers, or four
2137 const auto slot {[&](std::size_t q, std::size_t kind, std::size_t r, std::size_t part) {
2138 return sums[(2 * q * 2 * rows) + (kind * 2 * rows) + (2 * r) + part];
2139 }};
2140 std::vector<R> acc (4 * rows);
2141 for (std::size_t r {0}; r < rows; ++r) {
2142 R rr {0};
2143 R ii {0};
2144 R ri {0};
2145 R ir {0};
2146 for (std::size_t l {0}; done != 0 && l < lanes; ++l) {
2147 const std::size_t q {l * per};
2148 if constexpr (per == 2) {
2149 rr += slot(q, 0, r, 0) + slot(q + 1, 0, r, 0);
2150 ii += slot(q, 0, r, 1) + slot(q + 1, 0, r, 1);
2151 ri += slot(q, 1, r, 0) + slot(q + 1, 1, r, 0);
2152 ir += slot(q, 1, r, 1) + slot(q + 1, 1, r, 1);
2153 }
2154 else {
2155 rr += (slot(q, 0, r, 0) + slot(q + 1, 0, r, 0)) + (slot(q + 2, 0, r, 0) + slot(q + 3, 0, r, 0));
2156 ii += (slot(q, 0, r, 1) + slot(q + 1, 0, r, 1)) + (slot(q + 2, 0, r, 1) + slot(q + 3, 0, r, 1));
2157 ri += (slot(q, 1, r, 0) + slot(q + 1, 1, r, 0)) + (slot(q + 2, 1, r, 0) + slot(q + 3, 1, r, 0));
2158 ir += (slot(q, 1, r, 1) + slot(q + 1, 1, r, 1)) + (slot(q + 2, 1, r, 1) + slot(q + 3, 1, r, 1));
2159 }
2160 }
2161 acc[4 * r] = rr;
2162 acc[(4 * r) + 1] = ii;
2163 acc[(4 * r) + 2] = ri;
2164 acc[(4 * r) + 3] = ir;
2165 }
2166 for (std::size_t j {done}; j < cols; ++j) {
2167 const R* const col {a + (2 * j * lda)};
2168 const R xr {x[2 * j]};
2169 const R xi {x[(2 * j) + 1]};
2170 for (std::size_t r {0}; r < rows; ++r) {
2171 acc[4 * r] += col[2 * r] * xr;
2172 acc[(4 * r) + 1] += col[(2 * r) + 1] * xi;
2173 acc[(4 * r) + 2] += col[2 * r] * xi;
2174 acc[(4 * r) + 3] += col[(2 * r) + 1] * xr;
2175 }
2176 }
2177 for (std::size_t r {0}; r < rows; ++r) {
2178 y[r] = {acc[4 * r] - acc[(4 * r) + 1], acc[(4 * r) + 2] + acc[(4 * r) + 3]};
2179 }
2180 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
2181 }
2182
2186 __m256d re,
2187 __m256d im) noexcept
2188 {
2189 return _mm256_add_pd(sum, _mm256_addsub_pd(re, _mm256_permute_pd(im, 0b0101)));
2190 }
2191
2194 __m256 re,
2195 __m256 im) noexcept
2196 {
2197 return _mm256_add_ps(sum, _mm256_addsub_ps(re, _mm256_permute_ps(im, 0xB1)));
2198 }
2199
2203 std::size_t apart) noexcept
2204 {
2206 }
2207
2210 std::size_t apart) noexcept
2211 {
2212 const auto pair {[](const float* lo, const float* hi) {
2213 return _mm_castpd_ps(_mm_loadh_pd(_mm_load_sd(reinterpret_cast<const double*>(lo)), reinterpret_cast<const double*>(hi))); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex float's two parts loaded as 64 bits, unaligned loads
2214 }};
2215 return _mm256_insertf128_ps(_mm256_castps128_ps256(pair(at, at + apart)), pair(at + (2 * apart), at + (3 * apart)), 1);
2216 }
2217
2227 template <typename R>
2229 const R* b,
2230 std::size_t row_step,
2231 std::size_t column_step,
2232 std::size_t inner,
2233 std::size_t depth,
2234 std::size_t count,
2235 R* y) noexcept
2236 {
2237 using vec = decltype(avx2_load(x)); // __m256d or __m256
2238 using held = std::conditional_t<std::is_same_v<R, double>, ymm_register, ymm_register_f32>; // a register in an array
2239 constexpr std::size_t width {32 / sizeof(R)}; // parts in a register
2240 constexpr std::size_t tile {4 * width}; // parts held through a panel
2241 const std::size_t parts {2 * count};
2242 const held zero {};
2243 for (std::size_t pc {0}; pc < inner; pc += depth) { // a panel of B's rows at a time, read once
2244 const std::size_t end {std::min(pc + depth, inner)};
2245 const bool first {pc == 0}; // y holds nothing before the first panel
2246 std::size_t p {0};
2247 for (; p + tile <= parts; p += tile) {
2248 std::array<held, 4> re {};
2249 std::array<held, 4> im {};
2250 for (std::size_t k {pc}; k < end; ++k) {
2251 const R* const row {b + (2 * k * row_step) + (p * column_step)};
2252 const vec xr {avx2_pairs(x[2 * k], x[2 * k])};
2253 const vec xi {avx2_pairs(x[(2 * k) + 1], x[(2 * k) + 1])};
2255 for (std::size_t t {0}; t < 4; ++t) {
2256 const vec bv {column_step == 1 ? avx2_load(row + (t * width)) : avx2_complex_columns(row + (t * width * column_step), 2 * column_step)};
2257 re[t].v = avx2_fused(xr, bv, re[t].v);
2258 im[t].v = avx2_fused(xi, bv, im[t].v);
2259 }
2260 }
2261 for (std::size_t t {0}; t < 4; ++t) {
2262 R* const at {y + p + (t * width)};
2263 avx2_store(at, avx2_crossed_sum(first ? zero.v : avx2_load(at), re[t].v, im[t].v));
2264 }
2265 }
2266 for (; p < parts; p += 2) { // the columns past whole registers: the same terms, one at a time
2267 const R* const col {b + (p * column_step)};
2268 R rr {0};
2269 R ri {0};
2270 R ir {0};
2271 R ii {0};
2272 for (std::size_t k {pc}; k < end; ++k) {
2273 const R br {col[2 * k * row_step]};
2274 const R bi {col[(2 * k * row_step) + 1]};
2275 rr = std::fma(x[2 * k], br, rr);
2276 ri = std::fma(x[2 * k], bi, ri);
2277 ir = std::fma(x[(2 * k) + 1], br, ir);
2278 ii = std::fma(x[(2 * k) + 1], bi, ii);
2279 }
2280 y[p] = (first ? R {0} : y[p]) + (rr - ii);
2281 y[p + 1] = (first ? R {0} : y[p + 1]) + (ri + ir);
2282 }
2283 }
2284 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
2285 }
2286
2289 __m256d b) noexcept
2290 {
2291 return _mm256_add_pd(a, b);
2292 }
2293
2296 __m256 b) noexcept
2297 {
2298 return _mm256_add_ps(a, b);
2299 }
2300
2303 __m256d b) noexcept
2304 {
2305 return _mm256_sub_pd(a, b);
2306 }
2307
2310 __m256 b) noexcept
2311 {
2312 return _mm256_sub_ps(a, b);
2313 }
2314
2317 {
2318 return _mm256_xor_pd(_mm256_permute_pd(v, 0b0101), _mm256_setr_pd(-0.0, 0.0, -0.0, 0.0));
2319 }
2320
2323 {
2324 return _mm256_xor_ps(_mm256_permute_ps(v, 0xB1), _mm256_setr_ps(-0.0F, 0.0F, -0.0F, 0.0F, -0.0F, 0.0F, -0.0F, 0.0F));
2325 }
2326
2331 __m256d im,
2332 __m256d v) noexcept
2333 {
2334 return _mm256_fmaddsub_pd(re, v, _mm256_mul_pd(im, _mm256_permute_pd(v, 0b0101)));
2335 }
2336
2339 __m256 im,
2340 __m256 v) noexcept
2341 {
2342 return _mm256_fmaddsub_ps(re, v, _mm256_mul_ps(im, _mm256_permute_ps(v, 0xB1)));
2343 }
2344
2347 {
2348 return _mm256_movedup_pd(v);
2349 }
2350
2353 {
2354 return _mm256_moveldup_ps(v);
2355 }
2356
2359 {
2360 return _mm256_permute_pd(v, 0b1111);
2361 }
2362
2365 {
2366 return _mm256_movehdup_ps(v);
2367 }
2368
2371 NUMERICAL_TARGET("avx2,fma") inline void avx2_store_butterflies(double* to,
2372 __m256d y0,
2373 __m256d y1,
2374 __m256d y2,
2376 {
2381 }
2382
2384 NUMERICAL_TARGET("avx2,fma") inline void avx2_store_butterflies(float* to,
2385 __m256 y0,
2386 __m256 y1,
2387 __m256 y2,
2389 {
2398 }
2399
2401 template <typename R>
2402 NUMERICAL_TARGET("avx2,fma") inline std::complex<R> avx2_fused_product(const std::complex<R>& w,
2403 const std::complex<R>& v) noexcept
2404 {
2405 const R ii {w.imag() * v.imag()};
2406 const R ir {w.imag() * v.real()};
2407 return {std::fma(w.real(), v.real(), -ii), std::fma(w.real(), v.imag(), ir)};
2408 }
2409
2412 {
2413 return _mm256_xor_pd(_mm256_permute_pd(v, 0b0101), _mm256_setr_pd(0.0, -0.0, 0.0, -0.0));
2414 }
2415
2418 {
2419 return _mm256_xor_ps(_mm256_permute_ps(v, 0xB1), _mm256_setr_ps(0.0F, -0.0F, 0.0F, -0.0F, 0.0F, -0.0F, 0.0F, -0.0F));
2420 }
2421
2423 template <typename V>
2424 NUMERICAL_TARGET("avx2,fma") inline void avx2_four_point(V v0,
2425 V v1,
2426 V v2,
2427 V v3,
2428 V& y0,
2429 V& y1,
2430 V& y2,
2431 V& y3) noexcept
2432 {
2433 const V a_plus_c {avx2_add(v0, v2)};
2434 const V a_minus_c {avx2_sub(v0, v2)};
2435 const V b_plus_d {avx2_add(v1, v3)};
2436 const V i_b_d {avx2_times_i(avx2_sub(v1, v3))};
2441 }
2442
2444 NUMERICAL_TARGET("avx2,fma") inline __m256d avx2_splat(double c) noexcept
2445 {
2446 return _mm256_set1_pd(c);
2447 }
2448
2450 NUMERICAL_TARGET("avx2,fma") inline __m256 avx2_splat(float c) noexcept
2451 {
2452 return _mm256_set1_ps(c);
2453 }
2454
2457 __m256d b) noexcept
2458 {
2459 return _mm256_mul_pd(a, b);
2460 }
2461
2464 __m256 b) noexcept
2465 {
2466 return _mm256_mul_ps(a, b);
2467 }
2468
2471 __m256d b,
2472 __m256d c) noexcept
2473 {
2474 return _mm256_fmadd_pd(a, b, c);
2475 }
2476
2479 __m256 b,
2480 __m256 c) noexcept
2481 {
2482 return _mm256_fmadd_ps(a, b, c);
2483 }
2484
2487 template <typename R>
2488 NUMERICAL_TARGET("avx2,fma") inline std::complex<R> avx2_scaled_add(const std::complex<R>& s,
2489 R c,
2490 const std::complex<R>& v) noexcept
2491 {
2492 return {std::fma(c, v.real(), s.real()), std::fma(c, v.imag(), s.imag())};
2493 }
2494
2497 template <typename R>
2499 decltype(avx2_load(static_cast<const R*>(nullptr))) value,
2500 const std::complex<R> * twiddles,
2501 std::size_t run,
2502 std::size_t p,
2503 std::size_t k) noexcept
2504 {
2505 if (p != 0) {
2506 const std::complex<R> w {twiddles[((k - 1) * run) + p - 1]};
2507 value = avx2_twiddled(avx2_pairs(w.real(), w.real()), avx2_pairs(w.imag(), w.imag()), value);
2508 }
2509 avx2_store(place, value);
2510 }
2511
2517 template <typename R>
2518 NUMERICAL_TARGET("avx2,fma") inline void avx2_radix_three_pass(std::size_t current,
2519 std::size_t stride,
2520 const std::complex<R>* in,
2521 std::complex<R>* out,
2522 const std::complex<R>* twiddles,
2523 R half_root_three) noexcept
2524 {
2525 using C = std::complex<R>;
2526 using vec = decltype(avx2_load(static_cast<const R*>(nullptr))); // __m256d or __m256
2527 constexpr std::size_t width {32 / sizeof(C)};
2528 const std::size_t part {current / 3};
2529 const std::size_t run {part - 1};
2530 const std::size_t apart {stride * part};
2531 const vec half {avx2_splat(R(0.5))};
2532 const vec root {avx2_splat(half_root_three)};
2533 for (std::size_t p {0}; p < part; ++p) {
2534 const C* const in_p {in + (stride * p)};
2535 C* const to {out + (stride * 3 * p)};
2536 std::size_t q {0};
2537 if (stride >= width) {
2538 const C w1 {p == 0 ? C {1} : twiddles[p - 1]};
2539 const C w2 {p == 0 ? C {1} : twiddles[run + p - 1]};
2540 const vec w1_re {avx2_pairs(w1.real(), w1.real())};
2541 const vec w1_im {avx2_pairs(w1.imag(), w1.imag())};
2542 const vec w2_re {avx2_pairs(w2.real(), w2.real())};
2543 const vec w2_im {avx2_pairs(w2.imag(), w2.imag())};
2544 for (; q + width <= stride; q += width) {
2545 const R* const at {reinterpret_cast<const R*>(in_p + q)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is its two parts ([complex.numbers])
2546 const vec x0 {avx2_load(at)};
2547 const vec x1 {avx2_load(at + (2 * apart))};
2548 const vec x2 {avx2_load(at + (4 * apart))};
2549 const vec t {avx2_add(x1, x2)};
2550 const vec v {avx2_sub(x0, avx2_mul(half, t))};
2552 vec y1 {avx2_sub(v, iu)};
2553 vec y2 {avx2_add(v, iu)};
2554 if (p != 0) {
2557 }
2558 R* const place {reinterpret_cast<R*>(to + q)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
2560 avx2_store(place + (2 * stride), y1);
2561 avx2_store(place + (4 * stride), y2);
2562 }
2563 }
2564 for (; q < stride; ++q) { // a stride narrower than a register
2565 const C x0 {in_p[q]};
2566 const C x1 {in_p[q + apart]};
2567 const C x2 {in_p[q + (2 * apart)]};
2568 const C t {x1 + x2};
2569 const C v {x0 - C {R(0.5) * t.real(), R(0.5) * t.imag()}};
2570 const C d {x1 - x2};
2571 const C u {half_root_three * d.real(), half_root_three * d.imag()};
2572 const C iu {-u.imag(), u.real()};
2573 to[q] = x0 + t;
2574 to[q + stride] = p == 0 ? v - iu : avx2_fused_product(twiddles[p - 1], v - iu);
2575 to[q + 2 * stride] = p == 0 ? v + iu : avx2_fused_product(twiddles[run + p - 1], v + iu);
2576 }
2577 }
2578 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
2579 }
2580
2585 template <typename R>
2586 NUMERICAL_TARGET("avx2,fma") inline void avx2_radix_five_pass(std::size_t current,
2587 std::size_t stride,
2588 const std::complex<R>* in,
2589 std::complex<R>* out,
2590 const std::complex<R>* twiddles,
2591 R c1,
2592 R c2,
2593 R s1,
2594 R s2) noexcept
2595 {
2596 using C = std::complex<R>;
2597 using vec = decltype(avx2_load(static_cast<const R*>(nullptr))); // __m256d or __m256
2598 constexpr std::size_t width {32 / sizeof(C)};
2599 const std::size_t part {current / 5};
2600 const std::size_t run {part - 1};
2601 const std::size_t apart {stride * part};
2602 const vec vc1 {avx2_splat(c1)};
2603 const vec vc2 {avx2_splat(c2)};
2604 const vec vs1 {avx2_splat(s1)};
2605 const vec vs2 {avx2_splat(s2)};
2606 const vec vms1 {avx2_splat(-s1)};
2607 for (std::size_t p {0}; p < part; ++p) {
2608 const C* const in_p {in + (stride * p)};
2609 C* const to {out + (stride * 5 * p)};
2610 std::size_t q {0};
2611 if (stride >= width) {
2612 for (; q + width <= stride; q += width) {
2613 const R* const at {reinterpret_cast<const R*>(in_p + q)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is its two parts ([complex.numbers])
2614 const vec x0 {avx2_load(at)};
2615 const vec x1 {avx2_load(at + (2 * apart))};
2616 const vec x2 {avx2_load(at + (4 * apart))};
2617 const vec x3 {avx2_load(at + (6 * apart))};
2618 const vec x4 {avx2_load(at + (8 * apart))};
2619 const vec t1 {avx2_add(x1, x4)};
2620 const vec t2 {avx2_add(x2, x3)};
2621 const vec t3 {avx2_sub(x1, x4)};
2622 const vec t4 {avx2_sub(x2, x3)};
2623 const vec a1 {avx2_fma(vc2, t2, avx2_fma(vc1, t1, x0))};
2624 const vec a2 {avx2_fma(vc1, t2, avx2_fma(vc2, t1, x0))};
2627 R* const place {reinterpret_cast<R*>(to + q)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
2629 avx2_store_twiddled(place + (2 * stride), avx2_sub(a1, ib1), twiddles, run, p, 1);
2630 avx2_store_twiddled(place + (4 * stride), avx2_sub(a2, ib2), twiddles, run, p, 2);
2631 avx2_store_twiddled(place + (6 * stride), avx2_add(a2, ib2), twiddles, run, p, 3);
2632 avx2_store_twiddled(place + (8 * stride), avx2_add(a1, ib1), twiddles, run, p, 4);
2633 }
2634 }
2635 for (; q < stride; ++q) { // a stride narrower than a register
2636 const C x0 {in_p[q]};
2637 const C x1 {in_p[q + apart]};
2638 const C x2 {in_p[q + (2 * apart)]};
2639 const C x3 {in_p[q + (3 * apart)]};
2640 const C x4 {in_p[q + (4 * apart)]};
2641 const C t1 {x1 + x4};
2642 const C t2 {x2 + x3};
2643 const C t3 {x1 - x4};
2644 const C t4 {x2 - x3};
2645 const C a1 {avx2_scaled_add(avx2_scaled_add(x0, c1, t1), c2, t2)};
2646 const C a2 {avx2_scaled_add(avx2_scaled_add(x0, c2, t1), c1, t2)};
2647 const C b1 {avx2_scaled_add(C {s1 * t3.real(), s1 * t3.imag()}, s2, t4)};
2648 const C b2 {avx2_scaled_add(C {s2 * t3.real(), s2 * t3.imag()}, -s1, t4)};
2649 const C ib1 {-b1.imag(), b1.real()};
2650 const C ib2 {-b2.imag(), b2.real()};
2651 const std::array<C, 4> y {a1 - ib1, a2 - ib2, a2 + ib2, a1 + ib1};
2652 to[q] = (x0 + t1) + t2;
2653 for (std::size_t j {1}; j < 5; ++j) {
2654 to[q + (j * stride)] = p == 0 ? y[j - 1] : avx2_fused_product(twiddles[((j - 1) * run) + p - 1], y[j - 1]);
2655 }
2656 }
2657 }
2658 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
2659 }
2660
2665 {
2666 return _mm256_blend_pd(even, odd, 0b1010);
2667 }
2668
2672 {
2673 return _mm256_blend_ps(even, odd, 0b10101010);
2674 }
2675
2678 {
2679 return _mm256_xor_pd(v, _mm256_setr_pd(0.0, -0.0, 0.0, -0.0));
2680 }
2681
2684 {
2685 return _mm256_xor_ps(v, _mm256_setr_ps(0.0F, -0.0F, 0.0F, -0.0F, 0.0F, -0.0F, 0.0F, -0.0F));
2686 }
2687
2690 {
2691 return _mm256_permute_pd(v, 0b0101);
2692 }
2693
2696 {
2697 return _mm256_permute_ps(v, 0xB1);
2698 }
2699
2705 template <typename R>
2706 NUMERICAL_TARGET("avx2,fma") inline void avx2_radix_eight_pass(std::size_t current,
2707 std::size_t stride,
2708 const std::complex<R>* in,
2709 std::complex<R>* out,
2710 const std::complex<R>* twiddles,
2711 R half_root_two) noexcept
2712 {
2713 using vec = decltype(avx2_load(static_cast<const R*>(nullptr))); // __m256d or __m256
2714 constexpr std::size_t width {32 / sizeof(std::complex<R>)};
2715 const std::size_t part {current / 8};
2716 const std::size_t run {part - 1};
2717 const std::size_t apart {2 * stride * part}; // parts between a butterfly's inputs
2718 const vec root {avx2_splat(half_root_two)};
2719 const R* const from {reinterpret_cast<const R*>(in)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is its two parts ([complex.numbers])
2720 R* const into {reinterpret_cast<R*>(out)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
2721 for (std::size_t p {0}; p < part; ++p) {
2722 const R* const in_p {from + (2 * stride * p)};
2723 R* const to {into + (2 * stride * 8 * p)};
2724 for (std::size_t q {0}; q + width <= stride; q += width) {
2725 const R* const at {in_p + (2 * q)};
2726 const vec x0 {avx2_load(at)};
2727 const vec x1 {avx2_load(at + apart)};
2728 const vec x2 {avx2_load(at + (2 * apart))};
2729 const vec x3 {avx2_load(at + (3 * apart))};
2730 const vec x4 {avx2_load(at + (4 * apart))};
2731 const vec x5 {avx2_load(at + (5 * apart))};
2732 const vec x6 {avx2_load(at + (6 * apart))};
2733 const vec x7 {avx2_load(at + (7 * apart))};
2734 const vec b1 {avx2_sub(x1, x5)};
2735 const vec b3 {avx2_sub(x3, x7)};
2736 const vec s1 {avx2_add(b1, avx2_exchanged(b1))}; // re + im in both lanes
2737 const vec d1 {avx2_sub(b1, avx2_exchanged(b1))}; // re − im, im − re
2738 const vec s3 {avx2_add(b3, avx2_exchanged(b3))};
2739 const vec d3 {avx2_sub(avx2_exchanged(b3), b3)}; // im − re, re − im
2740 const vec w1 {avx2_mul(root, avx2_parts_of(s1, d1))};
2742 vec e0 {};
2743 vec e1 {};
2744 vec e2 {};
2745 vec e3 {};
2746 vec o0 {};
2747 vec o1 {};
2748 vec o2 {};
2749 vec o3 {};
2752 R* const place {to + (2 * q)};
2754 avx2_store_twiddled(place + (2 * stride), o0, twiddles, run, p, 1);
2755 avx2_store_twiddled(place + (4 * stride), e1, twiddles, run, p, 2);
2756 avx2_store_twiddled(place + (6 * stride), o1, twiddles, run, p, 3);
2757 avx2_store_twiddled(place + (8 * stride), e2, twiddles, run, p, 4);
2758 avx2_store_twiddled(place + (10 * stride), o2, twiddles, run, p, 5);
2759 avx2_store_twiddled(place + (12 * stride), e3, twiddles, run, p, 6);
2760 avx2_store_twiddled(place + (14 * stride), o3, twiddles, run, p, 7);
2761 }
2762 }
2763 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
2764 }
2765
2768 {
2769 return _mm256_permute2f128_pd(v, v, 0x01);
2770 }
2771
2774 {
2776 }
2777
2780 {
2781 return _mm256_xor_pd(v, _mm256_set1_pd(-0.0));
2782 }
2783
2786 {
2787 return _mm256_xor_ps(v, _mm256_set1_ps(-0.0F));
2788 }
2789
2796 template <typename R>
2797 NUMERICAL_TARGET("avx2,fma") inline std::size_t avx2_real_forward_registers(std::size_t h,
2798 const std::complex<R>* z,
2799 const std::complex<R>* roots,
2800 std::complex<R>* X) noexcept
2801 {
2802 using vec = decltype(avx2_load(static_cast<const R*>(nullptr))); // __m256d or __m256
2803 constexpr std::size_t width {32 / sizeof(std::complex<R>)};
2804 const vec half {avx2_splat(R(0.5))};
2805 const R* const from {reinterpret_cast<const R*>(z)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is its two parts ([complex.numbers])
2806 const R* const root {reinterpret_cast<const R*>(roots)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
2807 R* const into {reinterpret_cast<R*>(X)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
2808 std::size_t k {1};
2809 for (; k + width <= h; k += width) {
2810 const vec a {avx2_load(from + (2 * k))};
2811 const vec b {avx2_conjugated(avx2_reversed(avx2_load(from + (2 * (h - k - (width - 1))))))};
2812 const vec even {avx2_mul(avx2_add(a, b), half)};
2813 const vec odd {avx2_mul(avx2_times_minus_i(avx2_sub(a, b)), half)};
2814 const vec w {avx2_load(root + (2 * k))};
2816 }
2817 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
2818 return k;
2819 }
2820
2827 template <typename R>
2828 NUMERICAL_TARGET("avx2,fma") inline std::size_t avx2_real_inverse_registers(std::size_t h,
2829 const std::complex<R>* X,
2830 const std::complex<R>* roots,
2831 std::complex<R>* z) noexcept
2832 {
2833 using vec = decltype(avx2_load(static_cast<const R*>(nullptr))); // __m256d or __m256
2834 constexpr std::size_t width {32 / sizeof(std::complex<R>)};
2835 const vec half {avx2_splat(R(0.5))};
2836 const R* const from {reinterpret_cast<const R*>(X)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is its two parts ([complex.numbers])
2837 const R* const root {reinterpret_cast<const R*>(roots)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
2838 R* const into {reinterpret_cast<R*>(z)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
2839 std::size_t k {1};
2840 for (; k + width <= h; k += width) {
2841 const vec a {avx2_load(from + (2 * k))};
2842 const vec b {avx2_conjugated(avx2_reversed(avx2_load(from + (2 * (h - k - (width - 1))))))};
2843 const vec even {avx2_mul(avx2_add(a, b), half)};
2844 const vec w {avx2_load(root + (2 * k))};
2847 }
2848 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
2849 return k;
2850 }
2851
2852 // The transform's kernels, written once for AVX2 (__m256d, __m256) and AVX-512 (__m512d, __m512): each operation below
2853 // is overloaded on the register and built for its unit; the kernels' bodies, built for none, are inlined into the
2854 // functions built for each (fft_avx2_pass, fft_avx512_pass), where the operations then inline. The same operations
2855 // lane by lane, whatever the width: the same bits.
2856
2858 struct fft_avx2
2859 {
2860 static constexpr bool whole_strides {false};
2861 };
2862
2864 struct fft_avx512
2865 {
2867 static constexpr bool whole_strides {true};
2868 };
2869
2875 template <typename Isa,
2876 typename R>
2877 struct fft_reg;
2878
2880 template <>
2881 struct fft_reg<fft_avx2, double>
2882 {
2883 __m256d v;
2884 };
2885
2887 template <>
2888 struct fft_reg<fft_avx2, float>
2889 {
2890 __m256 v;
2891 };
2892
2894 template <>
2895 struct fft_reg<fft_avx512, double>
2896 {
2897 __m512d v;
2898 };
2899
2901 template <>
2902 struct fft_reg<fft_avx512, float>
2903 {
2904 __m512 v;
2905 };
2906
2908 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, double> fft_load(fft_avx2 /*isa*/,
2909 const double* p) noexcept
2910 {
2911 return {_mm256_loadu_pd(p)};
2912 }
2913
2915 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, double> fft_splat(fft_avx2 /*isa*/,
2916 double c) noexcept
2917 {
2918 return {_mm256_set1_pd(c)};
2919 }
2920
2922 NUMERICAL_TARGET("avx2,fma") inline void fft_store(double * p,
2923 fft_reg<fft_avx2, double> v) noexcept
2924 {
2925 _mm256_storeu_pd(p, v.v);
2926 }
2927
2929 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, double> fft_add(fft_reg<fft_avx2, double> a,
2930 fft_reg<fft_avx2, double> b) noexcept
2931 {
2932 return {_mm256_add_pd(a.v, b.v)};
2933 }
2934
2936 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, double> fft_sub(fft_reg<fft_avx2, double> a,
2937 fft_reg<fft_avx2, double> b) noexcept
2938 {
2939 return {_mm256_sub_pd(a.v, b.v)};
2940 }
2941
2944 {
2945 return {avx2_times_i(v.v)};
2946 }
2947
2950 {
2951 return {avx2_times_minus_i(v.v)};
2952 }
2953
2955 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, double> fft_twiddled(fft_reg<fft_avx2, double> re,
2956 fft_reg<fft_avx2, double> im,
2957 fft_reg<fft_avx2, double> v) noexcept
2958 {
2959 return {avx2_twiddled(re.v, im.v, v.v)};
2960 }
2961
2964 {
2965 return {_mm256_movedup_pd(v.v)};
2966 }
2967
2970 {
2971 return {_mm256_permute_pd(v.v, 0b1111)};
2972 }
2973
2975 NUMERICAL_TARGET("avx2,fma") inline void fft_store_butterflies(double * to,
2976 fft_reg<fft_avx2, double> y0,
2977 fft_reg<fft_avx2, double> y1,
2978 fft_reg<fft_avx2, double> y2,
2979 fft_reg<fft_avx2, double> y3) noexcept
2980 {
2981 avx2_store_butterflies(to, y0.v, y1.v, y2.v, y3.v);
2982 }
2983
2985 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, float> fft_load(fft_avx2 /*isa*/,
2986 const float* p) noexcept
2987 {
2988 return {_mm256_loadu_ps(p)};
2989 }
2990
2992 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, float> fft_splat(fft_avx2 /*isa*/,
2993 float c) noexcept
2994 {
2995 return {_mm256_set1_ps(c)};
2996 }
2997
2999 NUMERICAL_TARGET("avx2,fma") inline void fft_store(float * p,
3000 fft_reg<fft_avx2, float> v) noexcept
3001 {
3002 _mm256_storeu_ps(p, v.v);
3003 }
3004
3006 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, float> fft_add(fft_reg<fft_avx2, float> a,
3007 fft_reg<fft_avx2, float> b) noexcept
3008 {
3009 return {_mm256_add_ps(a.v, b.v)};
3010 }
3011
3013 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, float> fft_sub(fft_reg<fft_avx2, float> a,
3014 fft_reg<fft_avx2, float> b) noexcept
3015 {
3016 return {_mm256_sub_ps(a.v, b.v)};
3017 }
3018
3021 {
3022 return {avx2_times_i(v.v)};
3023 }
3024
3027 {
3028 return {avx2_times_minus_i(v.v)};
3029 }
3030
3032 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, float> fft_twiddled(fft_reg<fft_avx2, float> re,
3033 fft_reg<fft_avx2, float> im,
3034 fft_reg<fft_avx2, float> v) noexcept
3035 {
3036 return {avx2_twiddled(re.v, im.v, v.v)};
3037 }
3038
3041 {
3042 return {_mm256_moveldup_ps(v.v)};
3043 }
3044
3047 {
3048 return {_mm256_movehdup_ps(v.v)};
3049 }
3050
3052 NUMERICAL_TARGET("avx2,fma") inline void fft_store_butterflies(float * to,
3053 fft_reg<fft_avx2, float> y0,
3054 fft_reg<fft_avx2, float> y1,
3055 fft_reg<fft_avx2, float> y2,
3056 fft_reg<fft_avx2, float> y3) noexcept
3057 {
3058 avx2_store_butterflies(to, y0.v, y1.v, y2.v, y3.v);
3059 }
3060
3062 NUMERICAL_TARGET("avx512f") inline fft_reg<fft_avx512, double> fft_load(fft_avx512 /*isa*/,
3063 const double* p) noexcept
3064 {
3065 return {_mm512_loadu_pd(p)};
3066 }
3067
3069 NUMERICAL_TARGET("avx512f") inline fft_reg<fft_avx512, double> fft_splat(fft_avx512 /*isa*/,
3070 double c) noexcept
3071 {
3072 return {_mm512_set1_pd(c)};
3073 }
3074
3076 NUMERICAL_TARGET("avx512f") inline void fft_store(double * p,
3077 fft_reg<fft_avx512, double> v) noexcept
3078 {
3079 _mm512_storeu_pd(p, v.v);
3080 }
3081
3083 NUMERICAL_TARGET("avx512f") inline fft_reg<fft_avx512, double> fft_add(fft_reg<fft_avx512, double> a,
3084 fft_reg<fft_avx512, double> b) noexcept
3085 {
3086 return {_mm512_add_pd(a.v, b.v)};
3087 }
3088
3090 NUMERICAL_TARGET("avx512f") inline fft_reg<fft_avx512, double> fft_sub(fft_reg<fft_avx512, double> a,
3091 fft_reg<fft_avx512, double> b) noexcept
3092 {
3093 return {_mm512_sub_pd(a.v, b.v)};
3094 }
3095
3098 {
3100 }
3101
3104 {
3106 }
3107
3110 fft_reg<fft_avx512, double> im,
3111 fft_reg<fft_avx512, double> v) noexcept
3112 {
3113 return {_mm512_fmaddsub_pd(re.v, v.v, _mm512_mul_pd(im.v, _mm512_permute_pd(v.v, 0x55)))};
3114 }
3115
3118 {
3119 return {_mm512_movedup_pd(v.v)};
3120 }
3121
3124 {
3125 return {_mm512_permute_pd(v.v, 0xFF)};
3126 }
3127
3129 NUMERICAL_TARGET("avx512f") inline void fft_store_butterflies(double * to,
3130 fft_reg<fft_avx512, double> y0,
3131 fft_reg<fft_avx512, double> y1,
3132 fft_reg<fft_avx512, double> y2,
3133 fft_reg<fft_avx512, double> y3) noexcept
3134 {
3135 const __m512d low01 {_mm512_shuffle_f64x2(y0.v, y1.v, 0x44)}; // y0's butterflies 0 1, y1's 0 1
3136 const __m512d high01 {_mm512_shuffle_f64x2(y0.v, y1.v, 0xEE)}; // y0's 2 3, y1's 2 3
3137 const __m512d low23 {_mm512_shuffle_f64x2(y2.v, y3.v, 0x44)};
3138 const __m512d high23 {_mm512_shuffle_f64x2(y2.v, y3.v, 0xEE)};
3139 _mm512_storeu_pd(to, _mm512_shuffle_f64x2(low01, low23, 0x88)); // butterfly 0: y0 y1 y2 y3
3140 _mm512_storeu_pd(to + 8, _mm512_shuffle_f64x2(low01, low23, 0xDD)); // butterfly 1
3141 _mm512_storeu_pd(to + 16, _mm512_shuffle_f64x2(high01, high23, 0x88)); // butterfly 2
3142 _mm512_storeu_pd(to + 24, _mm512_shuffle_f64x2(high01, high23, 0xDD)); // butterfly 3
3143 }
3144
3146 NUMERICAL_TARGET("avx512f") inline fft_reg<fft_avx512, float> fft_load(fft_avx512 /*isa*/,
3147 const float* p) noexcept
3148 {
3149 return {_mm512_loadu_ps(p)};
3150 }
3151
3153 NUMERICAL_TARGET("avx512f") inline fft_reg<fft_avx512, float> fft_splat(fft_avx512 /*isa*/,
3154 float c) noexcept
3155 {
3156 return {_mm512_set1_ps(c)};
3157 }
3158
3160 NUMERICAL_TARGET("avx512f") inline void fft_store(float * p,
3161 fft_reg<fft_avx512, float> v) noexcept
3162 {
3163 _mm512_storeu_ps(p, v.v);
3164 }
3165
3167 NUMERICAL_TARGET("avx512f") inline fft_reg<fft_avx512, float> fft_add(fft_reg<fft_avx512, float> a,
3168 fft_reg<fft_avx512, float> b) noexcept
3169 {
3170 return {_mm512_add_ps(a.v, b.v)};
3171 }
3172
3174 NUMERICAL_TARGET("avx512f") inline fft_reg<fft_avx512, float> fft_sub(fft_reg<fft_avx512, float> a,
3175 fft_reg<fft_avx512, float> b) noexcept
3176 {
3177 return {_mm512_sub_ps(a.v, b.v)};
3178 }
3179
3182 {
3183 return {_mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(_mm512_permute_ps(v.v, 0xB1)), _mm512_set1_epi64(static_cast<std::int64_t>(0x0000000080000000))))};
3184 }
3185
3188 {
3190 }
3191
3194 fft_reg<fft_avx512, float> im,
3195 fft_reg<fft_avx512, float> v) noexcept
3196 {
3197 return {_mm512_fmaddsub_ps(re.v, v.v, _mm512_mul_ps(im.v, _mm512_permute_ps(v.v, 0xB1)))};
3198 }
3199
3202 {
3203 return {_mm512_moveldup_ps(v.v)};
3204 }
3205
3208 {
3209 return {_mm512_movehdup_ps(v.v)};
3210 }
3211
3213 NUMERICAL_TARGET("avx512f") inline void fft_store_butterflies(float * to,
3214 fft_reg<fft_avx512, float> y0,
3215 fft_reg<fft_avx512, float> y1,
3216 fft_reg<fft_avx512, float> y2,
3217 fft_reg<fft_avx512, float> y3) noexcept
3218 {
3219 const __m512d even01 {_mm512_unpacklo_pd(_mm512_castps_pd(y0.v), _mm512_castps_pd(y1.v))}; // butterflies 0 2 4 6: y0 y1
3223 const __m512d even_low {_mm512_shuffle_f64x2(even01, even23, 0x44)}; // 0 2, each y0 y1 then y2 y3
3224 const __m512d odd_low {_mm512_shuffle_f64x2(odd01, odd23, 0x44)}; // 1 3
3225 const __m512d even_high {_mm512_shuffle_f64x2(even01, even23, 0xEE)}; // 4 6
3226 const __m512d odd_high {_mm512_shuffle_f64x2(odd01, odd23, 0xEE)}; // 5 7
3227 double* const into {reinterpret_cast<double*>(to)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex float is eight bytes, stored whole
3228 _mm512_storeu_pd(into, _mm512_shuffle_f64x2(even_low, odd_low, 0x88)); // butterflies 0 1
3229 _mm512_storeu_pd(into + 8, _mm512_shuffle_f64x2(even_low, odd_low, 0xDD)); // 2 3
3230 _mm512_storeu_pd(into + 16, _mm512_shuffle_f64x2(even_high, odd_high, 0x88)); // 4 5
3231 _mm512_storeu_pd(into + 24, _mm512_shuffle_f64x2(even_high, odd_high, 0xDD)); // 6 7
3232 }
3233
3235 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, double> fft_mul(fft_reg<fft_avx2, double> a,
3236 fft_reg<fft_avx2, double> b) noexcept
3237 {
3238 return {_mm256_mul_pd(a.v, b.v)};
3239 }
3240
3242 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, double> fft_fused(fft_reg<fft_avx2, double> a,
3243 fft_reg<fft_avx2, double> b,
3244 fft_reg<fft_avx2, double> c) noexcept
3245 {
3246 return {_mm256_fmadd_pd(a.v, b.v, c.v)};
3247 }
3248
3250 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, float> fft_mul(fft_reg<fft_avx2, float> a,
3251 fft_reg<fft_avx2, float> b) noexcept
3252 {
3253 return {_mm256_mul_ps(a.v, b.v)};
3254 }
3255
3257 NUMERICAL_TARGET("avx2,fma") inline fft_reg<fft_avx2, float> fft_fused(fft_reg<fft_avx2, float> a,
3258 fft_reg<fft_avx2, float> b,
3259 fft_reg<fft_avx2, float> c) noexcept
3260 {
3261 return {_mm256_fmadd_ps(a.v, b.v, c.v)};
3262 }
3263
3265 NUMERICAL_TARGET("avx512f") inline fft_reg<fft_avx512, double> fft_mul(fft_reg<fft_avx512, double> a,
3266 fft_reg<fft_avx512, double> b) noexcept
3267 {
3268 return {_mm512_mul_pd(a.v, b.v)};
3269 }
3270
3272 NUMERICAL_TARGET("avx512f") inline fft_reg<fft_avx512, double> fft_fused(fft_reg<fft_avx512, double> a,
3273 fft_reg<fft_avx512, double> b,
3274 fft_reg<fft_avx512, double> c) noexcept
3275 {
3276 return {_mm512_fmadd_pd(a.v, b.v, c.v)};
3277 }
3278
3281 fft_reg<fft_avx512, double> odd) noexcept
3282 {
3283 return {_mm512_mask_blend_pd(0xAA, even.v, odd.v)};
3284 }
3285
3288 {
3290 }
3291
3294 {
3295 return {_mm512_permute_pd(v.v, 0x55)};
3296 }
3297
3299 NUMERICAL_TARGET("avx512f") inline fft_reg<fft_avx512, float> fft_mul(fft_reg<fft_avx512, float> a,
3300 fft_reg<fft_avx512, float> b) noexcept
3301 {
3302 return {_mm512_mul_ps(a.v, b.v)};
3303 }
3304
3307 fft_reg<fft_avx512, float> b,
3308 fft_reg<fft_avx512, float> c) noexcept
3309 {
3310 return {_mm512_fmadd_ps(a.v, b.v, c.v)};
3311 }
3312
3316 {
3317 return {_mm512_mask_blend_ps(0xAAAA, even.v, odd.v)};
3318 }
3319
3322 {
3324 }
3325
3328 {
3329 return {_mm512_permute_ps(v.v, 0xB1)};
3330 }
3331
3332#endif
3333} // namespace numerical::detail
3334
3335#endif // NUMERICAL_X86_HPP
constexpr std::size_t symmetric_row_group
The rows of a lower triangle symmetric_rows takes at once.
Definition simd.hpp:158
std::array< T, symmetric_row_group > symmetric_rows_tail(const T *rows, std::size_t n, const T *v, T *out, std::size_t i, std::size_t done, std::array< T, symmetric_row_group > sums)
The columns done…i+r of symmetric_rows, one by one: for each row r in order, out[k] += row_r[k] · v[i...
Definition simd.hpp:165
T root(const T &x)
The square root of a scalar, as std::sqrt gives it.
@ zero
C = A·B: C's elements before the product are not read.
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
The native vector of float and double the kernels hold their lanes in (GNU vector extensions,...