NumericAL
A header-only C++20 lexer built on REAL
Loading...
Searching...
No Matches
simd.hpp
Go to the documentation of this file.
1
6#ifndef NUMERICAL_SIMD_HPP
7#define NUMERICAL_SIMD_HPP
8
9#include <algorithm>
10#include <array>
11#include <bit>
12#include <cmath>
13#include <complex>
14#include <cstddef>
15#include <cstdint>
16#include <cstring>
17#include <limits>
18#include <type_traits>
19#include <vector>
20
21// A function inlined wherever it is called, past the compiler's budget: a call there costs more than its body.
22#if defined(_MSC_VER) && !defined(__clang__)
23# define NUMERICAL_ALWAYS_INLINE __forceinline
24#else
25# define NUMERICAL_ALWAYS_INLINE [[gnu::always_inline]] inline // NOLINT(cppcoreguidelines-macro-usage): an attribute, which no function can stand for
26#endif
27
28// A lambda inlined into its caller (written after its parameters): a loop's body taken whole into a function built for
29// other instructions (\ref detail::with_fma_instruction).
30#if defined(_MSC_VER) && !defined(__clang__)
31# define NUMERICAL_INLINE_LAMBDA
32#else
33# define NUMERICAL_INLINE_LAMBDA __attribute__((always_inline)) // NOLINT(cppcoreguidelines-macro-usage): an attribute, which no function can stand for
34#endif
35
36namespace numerical::detail {
37
44 NUMERICAL_ALWAYS_INLINE void copy_few_bytes(const unsigned char* from,
45 std::size_t bytes,
46 unsigned char* to) noexcept
47 {
48 if (bytes >= 64) {
49 std::memcpy(to, from, 32);
50 std::memcpy(to + 32, from + 32, 32);
51 std::memcpy(to + bytes - 64, from + bytes - 64, 32);
52 std::memcpy(to + bytes - 32, from + bytes - 32, 32);
53 }
54 else if (bytes >= 32) {
55 std::memcpy(to, from, 32);
56 std::memcpy(to + bytes - 32, from + bytes - 32, 32);
57 }
58 else if (bytes >= 16) {
59 std::memcpy(to, from, 16);
60 std::memcpy(to + bytes - 16, from + bytes - 16, 16);
61 }
62 else if (bytes >= 8) {
63 std::memcpy(to, from, 8);
64 std::memcpy(to + bytes - 8, from + bytes - 8, 8);
65 }
66 else { // 4 to 7 bytes; the last copy's offset kept at 0 or more: for elements of 8 bytes, where this branch is
67 // never taken, GCC 13 and 14 under the sanitizers followed it with no byte and saw an offset of -4
68 // (-Warray-bounds, an error in the tests' build)
69 const std::size_t last {std::max<std::size_t>(bytes, 4) - 4};
70 std::memcpy(to, from, 4);
71 std::memcpy(to + last, from + last, 4);
72 }
73 }
74
76 template <typename T>
78 std::size_t count,
79 T* to) noexcept
80 {
81 static_assert(sizeof(T) >= 4, "an element of four bytes at least");
82 copy_few_bytes(reinterpret_cast<const unsigned char*>(from), count * sizeof(T), reinterpret_cast<unsigned char*>(to)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the elements' bytes, as memcpy reads them
83 }
84
89 inline constexpr std::size_t transpose_tile {32};
90
100 [[nodiscard]] inline std::size_t transpose_piece(std::size_t from_bytes,
101 std::size_t to_bytes) noexcept
102 {
103#if defined(__x86_64__) || defined(_M_X64)
104 constexpr std::size_t aliased {std::size_t {1} << 10U};
105#else
106 constexpr std::size_t aliased {std::size_t {1} << 13U};
107#endif
108 return from_bytes % aliased == 0 || to_bytes % aliased == 0 ? 8 : transpose_tile;
109 }
110
115 template <std::size_t Piece,
116 typename Copy>
117 void transpose_pieces(std::size_t i0,
118 std::size_t i1,
119 std::size_t j0,
120 std::size_t j1,
121 const Copy& copy)
122 {
123 for (std::size_t pi {i0}; pi < i1; pi += Piece) {
124 const std::size_t pi1 {std::min(pi + Piece, i1)};
125 for (std::size_t pj {j0}; pj < j1; pj += Piece) {
126 const std::size_t pj1 {std::min(pj + Piece, j1)};
127 for (std::size_t i {pi}; i < pi1; ++i) {
128 for (std::size_t j {pj}; j < pj1; ++j) {
129 copy(i, j);
130 }
131 }
132 }
133 }
134 }
135
141 template <typename Copy>
142 void transpose_walk(std::size_t i0,
143 std::size_t i1,
144 std::size_t j0,
145 std::size_t j1,
146 std::size_t piece,
147 const Copy& copy)
148 {
149 if (piece == 8) {
151 }
152 else {
154 }
155 }
156
158 inline constexpr std::size_t symmetric_row_group {4};
159
164 template <typename T>
165 [[nodiscard]] std::array<T, symmetric_row_group> symmetric_rows_tail(const T * rows,
166 std::size_t n,
167 const T * v,
168 T * out,
169 std::size_t i,
170 std::size_t done,
171 std::array<T, symmetric_row_group> sums)
172 {
173 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
174 const T* row {rows + (r * n)};
175 const T factor {v[i + r]};
176 for (std::size_t k {done}; k < i + r; ++k) {
177 sums[r] += row[k] * v[k];
178 out[k] += row[k] * factor;
179 }
180 sums[r] += row[i + r] * factor;
181 }
182 return sums;
183 }
184
187 template <typename T>
188 concept split_complex = std::is_same_v<T, std::complex<float>> || std::is_same_v<T, std::complex<double>>;
189
190#if defined(__GNUC__)
192# if defined(__AVX__)
193 inline constexpr std::size_t native_vector_bytes {32};
194# else
195 inline constexpr std::size_t native_vector_bytes {16};
196# endif
197
199 template <typename T>
200 struct native_vector;
201
203 template <>
204 struct native_vector<float>
205 {
206 using type = float __attribute__((vector_size(native_vector_bytes)));
207 };
208
210 template <>
211 struct native_vector<double>
212 {
213 using type = double __attribute__((vector_size(native_vector_bytes)));
214 };
215
218 template <>
219 struct native_vector<std::uint64_t>
220 {
221 using type = std::uint64_t __attribute__((vector_size(native_vector_bytes)));
222 };
223
225 template <typename T>
226 concept native_scalar = std::is_same_v<T, float> || std::is_same_v<T, double>;
227
233 template <native_scalar T>
234 [[nodiscard]] T dot_product(const T* x,
235 const T* y,
236 std::size_t count)
237 {
238 using lane = typename native_vector<T>::type;
239 constexpr std::size_t width {native_vector_bytes / sizeof(T)};
240 constexpr std::size_t lanes {4};
241 std::array<lane, lanes> sum {};
242 const std::size_t done {count - (count % (width * lanes))};
243 for (std::size_t i {0}; i < done; i += width * lanes) {
244# pragma GCC unroll 4 // the sums in registers: GCC keeps a rolled array of vectors in memory
245 for (std::size_t l {0}; l < lanes; ++l) {
246 lane xv;
247 lane yv;
248 std::memcpy(&xv, x + i + (l * width), sizeof(lane));
249 std::memcpy(&yv, y + i + (l * width), sizeof(lane));
250 sum[l] += xv * yv;
251 }
252 }
253 T total {};
254 for (std::size_t l {0}; done != 0 && l < lanes; ++l) { // no block read: the lanes' +0s leave +0
255 for (std::size_t k {0}; k < width; ++k) {
256 total += sum[l][k];
257 }
258 }
259 for (std::size_t i {done}; i < count; ++i) {
260 total += x[i] * y[i];
261 }
262 return total;
263 }
264
274 template <split_complex C>
275 [[nodiscard]] std::array<typename C::value_type, 4> complex_dot_sums(const C* x,
276 const C* y,
277 std::size_t count)
278 {
279 using R = typename C::value_type;
280 using lane = typename native_vector<R>::type;
281 constexpr std::size_t width {native_vector_bytes / sizeof(R)}; // parts in a vector: an even count
282 constexpr std::size_t lanes {4};
283 const auto* const a {reinterpret_cast<const R*>(x)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is an array of its two parts
284 const auto* const b {reinterpret_cast<const R*>(y)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): as above
285 const std::size_t parts {2 * count};
286 std::array<lane, lanes> same {};
287 std::array<lane, lanes> cross {};
288 const std::size_t done {parts - (parts % (width * lanes))};
289 for (std::size_t i {0}; i < done; i += width * lanes) {
290# pragma GCC unroll 4 // the sums in registers
291 for (std::size_t l {0}; l < lanes; ++l) {
292 lane av;
293 lane bv;
294 std::memcpy(&av, a + i + (l * width), sizeof(lane));
295 std::memcpy(&bv, b + i + (l * width), sizeof(lane));
296 std::array<R, width> pairs_swapped {}; // each pair's parts swapped, then loaded: GCC 15 takes a lane set element by element for uninitialised
297 for (std::size_t k {0}; k < width; ++k) {
298 pairs_swapped[k] = bv[k ^ 1U];
299 }
301 std::memcpy(&swapped, pairs_swapped.data(), sizeof(lane));
302 same[l] += av * bv;
303 cross[l] += av * swapped;
304 }
305 }
306 R rr {};
307 R ii {};
308 R ri {};
309 R ir {};
310 for (std::size_t l {0}; l < lanes; ++l) {
311 for (std::size_t k {0}; k < width; k += 2) {
312 rr += same[l][k];
313 ii += same[l][k + 1];
314 ri += cross[l][k];
315 ir += cross[l][k + 1];
316 }
317 }
318 for (std::size_t i {done}; i < parts; i += 2) {
319 rr += a[i] * b[i];
320 ii += a[i + 1] * b[i + 1];
321 ri += a[i] * b[i + 1];
322 ir += a[i + 1] * b[i];
323 }
324 return {rr, ii, ri, ir};
325 }
326
328 template <split_complex C>
329 [[nodiscard]] C dot_product(const C* x,
330 const C* y,
331 std::size_t count)
332 {
333 const auto [rr, ii, ri, ir] {complex_dot_sums(x, y, count)};
334 return C {rr - ii, ri + ir};
335 }
336
339 template <split_complex C>
340 [[nodiscard]] C conjugate_dot_product(const C* x,
341 const C* y,
342 std::size_t count)
343 {
344 const auto [rr, ii, ri, ir] {complex_dot_sums(x, y, count)};
345 return C {rr + ii, ri - ir};
346 }
347
355 template <native_scalar T>
356 void row_dots(const T* m,
357 std::size_t ld,
358 std::size_t rows,
359 const T* x,
360 std::size_t count,
361 T* out)
362 {
363 using lane = typename native_vector<T>::type;
364 constexpr std::size_t width {native_vector_bytes / sizeof(T)};
365 constexpr std::size_t lanes {4};
366 constexpr std::size_t group {4};
367 const std::size_t done {count - (count % (width * lanes))};
368 std::size_t r0 {0};
369 for (; r0 + group <= rows; r0 += group) {
370 std::array<std::array<lane, lanes>, group> sum {};
371 for (std::size_t i {0}; i < done; i += width * lanes) {
372# pragma GCC unroll 4 // the sums in registers: GCC keeps a rolled array of vectors in memory
373 for (std::size_t l {0}; l < lanes; ++l) {
374 lane xv;
375 std::memcpy(&xv, x + i + (l * width), sizeof(lane));
376# pragma GCC unroll 4
377 for (std::size_t r {0}; r < group; ++r) {
378 lane mv;
379 std::memcpy(&mv, m + ((r0 + r) * ld) + i + (l * width), sizeof(lane));
380 sum[r][l] += mv * xv;
381 }
382 }
383 }
384 std::array<T, group> total {};
385 for (std::size_t l {0}; done != 0 && l < lanes; ++l) { // as dot_product's, the four rows side by side
386 for (std::size_t k {0}; k < width; ++k) {
387# pragma GCC unroll 4
388 for (std::size_t r {0}; r < group; ++r) {
389 total[r] += sum[r][l][k];
390 }
391 }
392 }
393 for (std::size_t i {done}; i < count; ++i) { // the remainders' four chains side by side, each in its order
394# pragma GCC unroll 4
395 for (std::size_t r {0}; r < group; ++r) {
396 total[r] += m[((r0 + r) * ld) + i] * x[i];
397 }
398 }
399 for (std::size_t r {0}; r < group; ++r) {
400 out[r0 + r] = total[r];
401 }
402 }
403 for (; r0 < rows; ++r0) {
404 out[r0] = dot_product(m + (r0 * ld), x, count);
405 }
406 }
407
416 template <native_scalar T>
417 void columns_row_dots(const T* a,
418 std::size_t lda,
419 std::size_t rows,
420 std::size_t cols,
421 const T* x,
422 T* y)
423 {
424 using lane = typename native_vector<T>::type;
425 constexpr std::size_t width {native_vector_bytes / sizeof(T)};
426 constexpr std::size_t partials {4 * width};
427 const std::size_t done {cols - (cols % partials)};
428 std::array<lane, partials> sum {};
429 std::array<T, width> part {};
430 const auto finish {[&](std::size_t r0, std::size_t count) { // rows r0… of the vector, from sum
431 for (std::size_t r {0}; r < count; ++r) {
432 T total {};
433 for (std::size_t p {0}; done != 0 && p < partials; ++p) {
434 total += sum[p][r];
435 }
436 y[r0 + r] = total;
437 }
438 }};
439 std::size_t r0 {0};
440 for (; r0 + width <= rows; r0 += width) {
441 sum = {};
442 for (std::size_t j {0}; j < done; j += partials) {
443# pragma GCC unroll 16 // the sums in registers
444 for (std::size_t p {0}; p < partials; ++p) {
445 lane mv;
446 std::memcpy(&mv, a + ((j + p) * lda) + r0, sizeof(lane));
447 lane xv {};
448 xv += x[j + p];
449 sum[p] += mv * xv;
450 }
451 }
452 finish(r0, width);
453 }
454 if (r0 < rows) { // the last rows, short of a vector: through one all the same, the same expression rounding them
455 sum = {};
456 for (std::size_t j {0}; j < done; j += partials) {
457 for (std::size_t p {0}; p < partials; ++p) {
458 std::fill(part.begin(), part.end(), T {});
459 std::copy(a + ((j + p) * lda) + r0, a + ((j + p) * lda) + rows, part.begin());
460 lane mv;
461 std::memcpy(&mv, part.data(), sizeof(lane));
462 lane xv {};
463 xv += x[j + p];
464 sum[p] += mv * xv;
465 }
466 }
467 finish(r0, rows - r0);
468 }
469 for (std::size_t j {done}; j < cols; ++j) {
470 const T* const col {a + (j * lda)};
471 for (std::size_t r {0}; r < rows; ++r) {
472 y[r] += col[r] * x[j];
473 }
474 }
475 }
476
485 template <split_complex C>
486 void columns_complex_row_dots(const C* a,
487 std::size_t lda,
488 std::size_t rows,
489 std::size_t cols,
490 const C* x,
491 C* y)
492 {
493 using R = typename C::value_type;
494 using lane = typename native_vector<R>::type;
495 constexpr std::size_t width {native_vector_bytes / sizeof(R)}; // parts in a vector: an even count
496 constexpr std::size_t per {width / 2}; // rows in a vector
497 constexpr std::size_t slots {2 * width}; // the elements of a row the kernel takes a step
498 const std::size_t done {cols - (cols % slots)};
499 const auto* const parts {reinterpret_cast<const R*>(a)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is an array of its two parts
500 std::vector<R> ahead (2 * width * done); // x[j]'s parts in a vector, then swapped
501 for (std::size_t j {0}; j < done; ++j) {
502 for (std::size_t k {0}; k < width; ++k) {
503 ahead[(2 * j * width) + k] = (k % 2 == 0) ? x[j].real() : x[j].imag();
504 ahead[(((2 * j) + 1) * width) + k] = (k % 2 == 0) ? x[j].imag() : x[j].real();
505 }
506 }
507 std::vector<R> sums (4 * rows); // each row's rr, ii, ri and ir
508 std::array<lane, slots> same {};
509 std::array<lane, slots> cross {};
510 std::array<R, width> part {};
511 const auto step {[&](std::size_t j, std::size_t q, const R* column) { // element j + q of the vector's rows
512 lane mv;
513 lane xs;
514 lane xc;
515 std::memcpy(&mv, column, sizeof(lane));
516 std::memcpy(&xs, ahead.data() + (2 * (j + q) * width), sizeof(lane));
517 std::memcpy(&xc, ahead.data() + (((2 * (j + q)) + 1) * width), sizeof(lane));
518 same[q] += mv * xs;
519 cross[q] += mv * xc;
520 }};
521 for (std::size_t r0 {0}; r0 < rows; r0 += per) {
522 const std::size_t count {std::min(per, rows - r0)};
523 same = {};
524 cross = {};
525 if (count == per) {
526 for (std::size_t j {0}; j < done; j += slots) {
527# pragma GCC unroll 16 // the slots in registers
528 for (std::size_t q {0}; q < slots; ++q) {
529 step(j, q, parts + (2 * (((j + q) * lda) + r0)));
530 }
531 }
532 }
533 else { // the last rows, short of a vector: through one all the same, the same expression rounding them
534 for (std::size_t j {0}; j < done; j += slots) {
535 for (std::size_t q {0}; q < slots; ++q) {
536 std::fill(part.begin(), part.end(), R {});
537 std::copy(parts + (2 * (((j + q) * lda) + r0)), parts + (2 * (((j + q) * lda) + r0 + count)), part.begin());
538 step(j, q, part.data());
539 }
540 }
541 }
542 for (std::size_t v {0}; v < count; ++v) {
543 R rr {};
544 R ii {};
545 R ri {};
546 R ir {};
547 for (std::size_t q {0}; q < slots; ++q) {
548 rr += same[q][2 * v];
549 ii += same[q][(2 * v) + 1];
550 ri += cross[q][2 * v];
551 ir += cross[q][(2 * v) + 1];
552 }
553 sums[r0 + v] = rr;
554 sums[rows + r0 + v] = ii;
555 sums[(2 * rows) + r0 + v] = ri;
556 sums[(3 * rows) + r0 + v] = ir;
557 }
558 }
559 for (std::size_t j {done}; j < cols; ++j) {
560 const R* const col {parts + (2 * j * lda)};
561 const R xr {x[j].real()};
562 const R xi {x[j].imag()};
563 for (std::size_t r {0}; r < rows; ++r) {
564 sums[r] += col[2 * r] * xr;
565 sums[rows + r] += col[(2 * r) + 1] * xi;
566 sums[(2 * rows) + r] += col[2 * r] * xi;
567 sums[(3 * rows) + r] += col[(2 * r) + 1] * xr;
568 }
569 }
570 for (std::size_t r {0}; r < rows; ++r) {
571 y[r] = C {sums[r] - sums[rows + r], sums[(2 * rows) + r] + sums[(3 * rows) + r]};
572 }
573 }
574
580 template <native_scalar T>
581 [[nodiscard]] T symmetric_row(const T* row,
582 const T* v,
583 const T& f,
584 T* out,
585 std::size_t count)
586 {
587 using lane = typename native_vector<T>::type;
588 constexpr std::size_t width {native_vector_bytes / sizeof(T)};
589 constexpr std::size_t lanes {4};
590 const T factor {f}; // read once: it may be an element of v
591 const lane splat = factor - lane {}; // factor in every lane
592 std::array<lane, lanes> sum {};
593 const std::size_t done {count - (count % (width * lanes))};
594 for (std::size_t i {0}; i < done; i += width * lanes) {
595# pragma GCC unroll 4 // the sums in registers: GCC keeps a rolled array of vectors in memory
596 for (std::size_t l {0}; l < lanes; ++l) {
597 lane rv;
598 lane vv;
599 lane ov;
600 std::memcpy(&rv, row + i + (l * width), sizeof(lane));
601 std::memcpy(&vv, v + i + (l * width), sizeof(lane));
602 std::memcpy(&ov, out + i + (l * width), sizeof(lane));
603 sum[l] += rv * vv;
604 ov += rv * splat;
605 std::memcpy(out + i + (l * width), &ov, sizeof(lane));
606 }
607 }
608 T total {};
609 for (std::size_t l {0}; done != 0 && l < lanes; ++l) { // as dot_product's
610 for (std::size_t k {0}; k < width; ++k) {
611 total += sum[l][k];
612 }
613 }
614 for (std::size_t i {done}; i < count; ++i) {
615 total += row[i] * v[i];
616 out[i] += row[i] * factor;
617 }
618 return total;
619 }
620
627 template <native_scalar T>
628 [[nodiscard]] std::array<T, symmetric_row_group> symmetric_rows(const T* rows,
629 std::size_t n,
630 const T* v,
631 T* out,
632 std::size_t i)
633 {
634 using lane = typename native_vector<T>::type;
635 constexpr std::size_t width {native_vector_bytes / sizeof(T)};
636 std::array<lane, symmetric_row_group> sum {};
637 std::array<lane, symmetric_row_group> splat {};
638 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
639 splat[r] = v[i + r] - lane {}; // v[i + r] in every lane
640 }
641 const std::size_t done {i - (i % width)};
642 for (std::size_t j {0}; j < done; j += width) {
643 lane vv;
644 lane ov;
645 std::memcpy(&vv, v + j, sizeof(lane));
646 std::memcpy(&ov, out + j, sizeof(lane));
647# pragma GCC unroll 4 // the sums in registers: GCC keeps a rolled array of vectors in memory
648 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
649 lane rv;
650 std::memcpy(&rv, rows + (r * n) + j, sizeof(lane));
651 sum[r] += rv * vv;
652 ov += rv * splat[r];
653 }
654 std::memcpy(out + j, &ov, sizeof(lane));
655 }
656 std::array<T, symmetric_row_group> sums {};
657 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
658 for (std::size_t k {0}; k < width; ++k) {
659 sums[r] += sum[r][k];
660 }
661 }
662 return symmetric_rows_tail(rows, n, v, out, i, done, sums);
663 }
664
679 template <native_scalar T>
680 std::size_t largest_magnitude_vectors(const T* x,
681 std::size_t count,
682 T& best,
683 std::size_t& at)
684 {
685 using lane = typename native_vector<T>::type;
686 using mask = decltype(lane {} > lane {});
687 using index = std::remove_cvref_t<decltype(mask {}[0])>;
688 constexpr std::size_t width {native_vector_bytes / sizeof(T)};
689 constexpr std::size_t ways {4}; // independent chains: one waits a compare and a select per vector
690 if (count < 1 + (ways * width)) {
691 return 1;
692 }
693 const mask magnitude {~std::bit_cast<mask>(T {-0.0} - lane {})}; // every bit but the sign's
694 std::array<lane, ways> top {};
695 std::array<mask, ways> where {}; // index 0 in every lane
696 std::array<mask, ways> next {}; // each chain's next indices
697 for (std::size_t w {0}; w < ways; ++w) {
698 top[w] = best - lane {}; // best in every lane
699 for (std::size_t k {0}; k < width; ++k) {
700 next[w][k] = static_cast<index>(1 + (w * width) + k);
701 }
702 }
703 std::size_t i {1};
704 for (; i + (ways * width) <= count; i += ways * width) { // selects by masks: each lane takes its own side
705# pragma GCC unroll 4 // the chains in registers: GCC keeps a rolled array of vectors in memory
706 for (std::size_t w {0}; w < ways; ++w) {
707 lane v;
708 std::memcpy(&v, x + i + (w * width), sizeof(lane));
709 const lane v_abs {std::bit_cast<lane>(std::bit_cast<mask>(v) & magnitude)};
710 const mask larger {v_abs > top[w]};
711 top[w] = std::bit_cast<lane>((std::bit_cast<mask>(v_abs) & larger) | (std::bit_cast<mask>(top[w]) & ~larger));
712 where[w] = (next[w] & larger) | (where[w] & ~larger);
713 next[w] += static_cast<index>(ways * width);
714 }
715 }
716 for (std::size_t w {0}; w < ways; ++w) { // each lane holds its subsequence's first largest
717 for (std::size_t k {0}; k < width; ++k) {
718 const auto lane_at {static_cast<std::size_t>(where[w][k])};
719 if (top[w][k] > best || (top[w][k] == best && lane_at < at)) {
720 best = top[w][k];
721 at = lane_at;
722 }
723 }
724 }
725 return i;
726 }
727
728#endif
729
731 inline constexpr std::size_t product_rows {8};
732
733#if defined(__GNUC__) && !defined(__clang__)
734
736 template <native_scalar T>
737 std::size_t subtract_scaled_vectors(T* y,
738 const T* x,
739 const T& f,
740 std::size_t count)
741 {
742 using lane = typename native_vector<T>::type;
743 constexpr std::size_t width {native_vector_bytes / sizeof(T)};
744 const std::size_t done {count - (count % width)};
745 for (std::size_t i {0}; i < done; i += width) {
746 lane yv;
747 lane xv;
748 std::memcpy(&yv, y + i, sizeof(lane));
749 std::memcpy(&xv, x + i, sizeof(lane));
750 yv -= f * xv;
751 std::memcpy(y + i, &yv, sizeof(lane));
752 }
753 return done;
754 }
755
764 template <native_scalar T>
765 std::size_t subtract_combined_vectors(T* y,
766 const T* f,
767 const T* x,
768 std::size_t ldx,
769 std::size_t terms,
770 std::size_t count)
771 {
772 using lane = typename native_vector<T>::type;
773 constexpr std::size_t lanes {4};
774 constexpr std::size_t width {native_vector_bytes / sizeof(T)};
775 const std::size_t done {count - (count % (lanes * width))};
776 for (std::size_t i {0}; i < done; i += lanes * width) {
777 std::array<lane, lanes> held {};
778# pragma GCC unroll 4
779 for (std::size_t u {0}; u < lanes; ++u) {
780 std::memcpy(&held[u], y + i + (u * width), sizeof(lane));
781 }
782 for (std::size_t t {0}; t < terms; ++t) {
783 const T factor {f[t]};
784# pragma GCC unroll 4
785 for (std::size_t u {0}; u < lanes; ++u) {
786 lane row;
787 std::memcpy(&row, x + (t * ldx) + i + (u * width), sizeof(lane));
788 held[u] -= factor * row;
789 }
790 }
791# pragma GCC unroll 4
792 for (std::size_t u {0}; u < lanes; ++u) {
793 std::memcpy(y + i + (u * width), &held[u], sizeof(lane));
794 }
795 }
796 return done;
797 }
798
804 template <native_scalar T>
805 std::size_t subtract_rows_vectors(T* y,
806 std::size_t ldy,
807 const T* f,
808 std::size_t ldf,
809 const T* x,
810 std::size_t ldx,
811 std::size_t terms,
812 std::size_t count)
813 {
814 using lane = typename native_vector<T>::type;
815 constexpr std::size_t width {native_vector_bytes / sizeof(T)};
816 const std::size_t done {count - (count % width)};
817 for (std::size_t i {0}; i < done; i += width) {
818 std::array<lane, product_rows> held {};
819# pragma GCC unroll 8
820 for (std::size_t r {0}; r < product_rows; ++r) {
821 std::memcpy(&held[r], y + (r * ldy) + i, sizeof(lane));
822 }
823 for (std::size_t t {0}; t < terms; ++t) {
824 lane row;
825 std::memcpy(&row, x + (t * ldx) + i, sizeof(lane));
826# pragma GCC unroll 8
827 for (std::size_t r {0}; r < product_rows; ++r) {
828 const T factor {f[(r * ldf) + t]};
829 held[r] -= factor * row;
830 }
831 }
832# pragma GCC unroll 8
833 for (std::size_t r {0}; r < product_rows; ++r) {
834 std::memcpy(y + (r * ldy) + i, &held[r], sizeof(lane));
835 }
836 }
837 return done;
838 }
839
841 template <native_scalar T>
842 std::size_t add_scaled_vectors(T* y,
843 const T* x,
844 const T& f,
845 std::size_t count)
846 {
847 using lane = typename native_vector<T>::type;
848 constexpr std::size_t width {native_vector_bytes / sizeof(T)};
849 const std::size_t done {count - (count % width)};
850 for (std::size_t i {0}; i < done; i += width) {
851 lane yv;
852 lane xv;
853 std::memcpy(&yv, y + i, sizeof(lane));
854 std::memcpy(&xv, x + i, sizeof(lane));
855 yv += f * xv;
856 std::memcpy(y + i, &yv, sizeof(lane));
857 }
858 return done;
859 }
860
867 template <split_complex C>
868 std::size_t add_scaled_vectors(C* y,
869 const C* x,
870 const C& f,
871 std::size_t count)
872 {
873 using R = C::value_type;
874 using lane = typename native_vector<R>::type;
875 constexpr std::size_t width {native_vector_bytes / sizeof(R)}; // parts in a vector: an even count
876 auto* const out {reinterpret_cast<R*>(y)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is an array of its two parts
877 const auto* const in {reinterpret_cast<const R*>(x)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): as above
880 lane sign;
881 for (std::size_t k {0}; k < width; ++k) {
882 real_part[k] = f.real();
883 imaginary_part[k] = f.imag();
884 sign[k] = (k % 2 == 0) ? R {-1} : R {1};
885 }
886 const std::size_t parts {2 * count};
887 const std::size_t done {parts - (parts % width)};
888 for (std::size_t i {0}; i < done; i += width) {
889 lane yv;
890 lane xv;
891 std::memcpy(&yv, out + i, sizeof(lane));
892 std::memcpy(&xv, in + i, sizeof(lane));
893 std::array<R, width> pairs_swapped {}; // as complex_dot_sums's
894 for (std::size_t k {0}; k < width; ++k) {
895 pairs_swapped[k] = xv[k ^ 1U];
896 }
898 std::memcpy(&swapped, pairs_swapped.data(), sizeof(lane));
899 const lane straight {real_part * xv}; // re·re, re·im
900 const lane crossed {imaginary_part * swapped}; // im·im, im·re
901 const lane signed_crossed {sign * crossed}; // −im·im, +im·re
902 const lane product {straight + signed_crossed}; // re·re − im·im, re·im + im·re
903 yv = yv + product;
904 std::memcpy(out + i, &yv, sizeof(lane));
905 }
906 return done / 2;
907 }
908
909#endif
910
917 template <std::size_t Rows,
918 std::size_t Cols,
919 typename T,
920 typename Full>
922 std::size_t ldc,
923 std::size_t rows,
924 std::size_t cols,
925 Full full) noexcept
926 {
927 if (rows == Rows && cols == Cols) {
928 full(c, ldc);
929 return;
930 }
931 std::array<T, Rows * Cols> edge {};
932 for (std::size_t i {0}; i < rows; ++i) {
933 for (std::size_t j {0}; j < cols; ++j) {
934 edge[(i * Cols) + j] = c[(i * ldc) + j];
935 }
936 }
937 full(edge.data(), Cols);
938 for (std::size_t i {0}; i < rows; ++i) {
939 for (std::size_t j {0}; j < cols; ++j) {
940 c[(i * ldc) + j] = edge[(i * Cols) + j];
941 }
942 }
943 }
944
946 template <std::size_t R,
947 typename T,
948 typename Full>
949 void rows_of_tile(std::size_t rows,
950 T* c,
951 std::size_t ldc,
952 Full& full) noexcept
953 {
954 if constexpr (R > 1) {
955 if (rows < R) {
956 rows_of_tile<R - 1>(rows, c, ldc, full);
957 return;
958 }
959 }
960 full.template operator()<R>(c, ldc);
961 }
962
970 template <std::size_t Rows,
971 std::size_t Cols,
972 typename T,
973 typename Full>
975 std::size_t ldc,
976 std::size_t rows,
977 std::size_t cols,
978 Full full) noexcept
979 {
980 if (cols == Cols) {
981 rows_of_tile<Rows>(rows, c, ldc, full);
982 return;
983 }
984 through_edge<Rows, Cols>(c, ldc, rows, cols, [&](T* tile, std::size_t ld) noexcept { rows_of_tile<Rows>(rows, tile, ld, full); });
985 }
986
988 template <typename T>
989 std::size_t subtract_scaled_vectors(T* /*y*/,
990 const T* /*x*/,
991 const T& /*f*/,
992 std::size_t /*count*/)
993 {
994 return 0;
995 }
996
998 template <typename T,
999 typename Magnitude>
1000 std::size_t largest_magnitude_vectors(const T* /*x*/,
1001 std::size_t /*count*/,
1002 Magnitude& /*best*/,
1003 std::size_t& /*at*/)
1004 {
1005 return 1;
1006 }
1007
1010 template <typename T>
1011 std::size_t subtract_combined_vectors(T* /*y*/,
1012 const T* /*f*/,
1013 const T* /*x*/,
1014 std::size_t /*ldx*/,
1015 std::size_t /*terms*/,
1016 std::size_t /*count*/)
1017 {
1018 return 0;
1019 }
1020
1023 template <typename T>
1024 std::size_t subtract_rows_vectors(T* /*y*/,
1025 std::size_t /*ldy*/,
1026 const T* /*f*/,
1027 std::size_t /*ldf*/,
1028 const T* /*x*/,
1029 std::size_t /*ldx*/,
1030 std::size_t /*terms*/,
1031 std::size_t /*count*/)
1032 {
1033 return 0;
1034 }
1035
1037 template <typename T>
1038 std::size_t add_scaled_vectors(T* /*y*/,
1039 const T* /*x*/,
1040 const T& /*f*/,
1041 std::size_t /*count*/)
1042 {
1043 return 0;
1044 }
1045
1047 template <typename T>
1049 const T* v,
1050 const T& f,
1051 T* out,
1052 std::size_t count)
1053 {
1054 const T factor {f}; // read once: it may be an element of v
1055 T total {};
1056 for (std::size_t i {0}; i < count; ++i) {
1057 total += row[i] * v[i];
1058 out[i] += row[i] * factor;
1059 }
1060 return total;
1061 }
1062
1064 template <typename T>
1065 [[nodiscard]] std::array<T, symmetric_row_group> symmetric_rows(const T* rows,
1066 std::size_t n,
1067 const T* v,
1068 T* out,
1069 std::size_t i)
1070 {
1071 return symmetric_rows_tail(rows, n, v, out, i, 0, {});
1072 }
1073
1075 template <typename T>
1076 [[nodiscard]] T dot_product(const T* x,
1077 const T* y,
1078 std::size_t count)
1079 {
1080 T total {};
1081 for (std::size_t i {0}; i < count; ++i) {
1082 total += x[i] * y[i];
1083 }
1084 return total;
1085 }
1086
1088 template <typename T>
1090 const T* y,
1091 std::size_t count)
1092 {
1093 T total {};
1094 for (std::size_t i {0}; i < count; ++i) {
1095 total += std::conj(x[i]) * y[i];
1096 }
1097 return total;
1098 }
1099
1101 template <typename T>
1102 void row_dots(const T* m,
1103 std::size_t ld,
1104 std::size_t rows,
1105 const T* x,
1106 std::size_t count,
1107 T* out)
1108 {
1109 for (std::size_t r {0}; r < rows; ++r) {
1110 out[r] = dot_product(m + (r * ld), x, count);
1111 }
1112 }
1113
1121 template <typename T>
1123 const T& b)
1124 {
1125 return a * b;
1126 }
1127
1128#if defined(_LIBCPP_VERSION)
1130 template <typename R>
1131 [[nodiscard]] NUMERICAL_ALWAYS_INLINE std::complex<R> operator_times(const std::complex<R>& a,
1132 const std::complex<R>& b)
1133 {
1134 const R ac {a.real() * b.real()};
1135 const R bd {a.imag() * b.imag()};
1136 const R ad {a.real() * b.imag()};
1137 const R bc {a.imag() * b.real()};
1138 const R x {ac - bd};
1139 const R y {ad + bc};
1140 if (std::isnan(x) && std::isnan(y)) [[unlikely]] {
1141 return a * b;
1142 }
1143 return {x, y};
1144 }
1145
1146#endif
1147
1158 template <typename T>
1160 const T& f,
1161 const T* x,
1162 std::size_t count)
1163 {
1164 const T factor {f}; // read once, as a scalar loop would: it may be an element of y's array
1165 for (std::size_t i {subtract_scaled_vectors(y, x, factor, count)}; i < count; ++i) {
1166 y[i] -= operator_times(factor, x[i]);
1167 }
1168 }
1169
1179 template <typename T>
1181 const T& factor,
1182 const T& x)
1183 {
1184#if defined(__GNUC__) && !defined(__clang__)
1185 const T product {factor * x};
1186 return held - product;
1187#else
1188 return held - (factor * x);
1189#endif
1190 }
1191
1197 template <typename T>
1199 const T* f,
1200 const T* x,
1201 std::size_t ldx,
1202 std::size_t terms,
1203 std::size_t count)
1204 {
1205 for (std::size_t t {0}; t < terms; ++t) {
1206 subtract_product(y, f[t], x + (t * ldx), count);
1207 }
1208 }
1209
1218 template <typename T>
1219 requires std::is_floating_point_v<T>
1221 const T* f,
1222 const T* x,
1223 std::size_t ldx,
1224 std::size_t terms,
1225 std::size_t count)
1226 {
1227 std::size_t i {subtract_combined_vectors(y, f, x, ldx, terms, count)};
1228 // Blocks of 32, then of 8, 4, 2 and 1, each element a chain of its own through the terms: four multiply-adds a
1229 // cycle wait on their last one, and 8 elements held four chains (arm64, an inverse of order 128: 151 → 137 µs);
1230 // a narrower rest still takes the terms across its elements at once, where element by element each waited on
1231 // the last term (four columns of an LU solve of order 128 on arm64: 112 → 92 µs, the factorisation 75).
1232 const auto blocks = [&]<std::size_t Width>(std::integral_constant<std::size_t, Width>) {
1233 for (; i + Width <= count; i += Width) {
1234 std::array<T, Width> held {};
1235 std::copy_n(y + i, Width, held.begin());
1236 for (std::size_t t {0}; t < terms; ++t) {
1237 const T factor {f[t]};
1238 const T* const row {x + (t * ldx) + i};
1239 for (std::size_t u {0}; u < Width; ++u) {
1240 held[u] = less_product(held[u], factor, row[u]);
1241 }
1242 }
1243 std::copy_n(held.begin(), Width, y + i);
1244 }
1245 };
1246 blocks(std::integral_constant<std::size_t, 32> {});
1247 blocks(std::integral_constant<std::size_t, 8> {});
1248 blocks(std::integral_constant<std::size_t, 4> {});
1249 blocks(std::integral_constant<std::size_t, 2> {});
1250 blocks(std::integral_constant<std::size_t, 1> {});
1251 }
1252
1260 template <typename T>
1261 requires std::is_floating_point_v<T>
1263 std::size_t ldy,
1264 const T* f,
1265 std::size_t ldf,
1266 const T* x,
1267 std::size_t ldx,
1268 std::size_t terms,
1269 std::size_t count)
1270 {
1271 constexpr std::size_t width {4}; // four columns of eight rows: 32 values, which arm64's registers hold
1272 std::size_t i {subtract_rows_vectors(y, ldy, f, ldf, x, ldx, terms, count)};
1273 const auto columns = [&]<std::size_t Width>(std::integral_constant<std::size_t, Width>) {
1274 for (; i + Width <= count; i += Width) {
1275 std::array<std::array<T, Width>, product_rows> held {};
1276 for (std::size_t r {0}; r < product_rows; ++r) {
1277 std::copy_n(y + (r * ldy) + i, Width, held[r].begin());
1278 }
1279 for (std::size_t t {0}; t < terms; ++t) {
1280 const T* const row {x + (t * ldx) + i};
1281 for (std::size_t r {0}; r < product_rows; ++r) {
1282 const T factor {f[(r * ldf) + t]};
1283 for (std::size_t u {0}; u < Width; ++u) {
1284 held[r][u] = less_product(held[r][u], factor, row[u]);
1285 }
1286 }
1287 }
1288 for (std::size_t r {0}; r < product_rows; ++r) {
1289 std::copy_n(held[r].begin(), Width, y + (r * ldy) + i);
1290 }
1291 }
1292 };
1293 columns(std::integral_constant<std::size_t, width> {});
1294 columns(std::integral_constant<std::size_t, 1> {});
1295 }
1296
1298 template <typename T>
1300 const T* x,
1301 const T& f,
1302 std::size_t count)
1303 {
1304 const T factor {f}; // read once, as a scalar loop would: it may be an element of y's array
1305 for (std::size_t i {subtract_scaled_vectors(y, x, factor, count)}; i < count; ++i) {
1306 y[i] -= operator_times(x[i], factor);
1307 }
1308 }
1309
1312 template <split_complex C>
1314 const C& f,
1315 const C& x)
1316 {
1317 using R = C::value_type;
1318 const R ac {f.real() * x.real()};
1319 const R bd {f.imag() * x.imag()};
1320 const R ad {f.real() * x.imag()};
1321 const R bc {f.imag() * x.real()};
1322 const R re {ac - bd};
1323 const R im {ad + bc};
1324 y = C {y.real() + re, y.imag() + im};
1325 }
1326
1329 template <split_complex C>
1331 const C& f,
1332 const C* x,
1333 std::size_t count)
1334 {
1335 const C factor {f}; // read once, as a scalar loop would: it may be an element of y's array
1336 for (std::size_t i {add_scaled_vectors(y, x, factor, count)}; i < count; ++i) {
1337 complex_scaled_add(y[i], factor, x[i]);
1338 }
1339 }
1340
1342 template <typename T>
1344 const T& f,
1345 const T& x)
1346 {
1347 y += operator_times(f, x);
1348 }
1349
1351 template <split_complex C>
1353 const C& f,
1354 const C& x)
1355 {
1356 complex_scaled_add(y, f, x);
1357 }
1358
1360 template <typename T>
1362 const T& f,
1363 const T* x,
1364 std::size_t count)
1365 {
1366 const T factor {f}; // read once, as a scalar loop would: it may be an element of y's array
1367 for (std::size_t i {add_scaled_vectors(y, x, factor, count)}; i < count; ++i) {
1368 y[i] += operator_times(factor, x[i]);
1369 }
1370 }
1371
1372#if defined(__GNUC__)
1375 using finite_lanes_64 = std::uint64_t __attribute__((vector_size(16)));
1377 using finite_lanes_32 = std::uint32_t __attribute__((vector_size(16)));
1378#endif
1379
1390 template <typename R>
1391 requires std::is_same_v<R, double> || std::is_same_v<R, float>
1392 [[nodiscard]] bool all_finite(const R* x,
1393 std::size_t count) noexcept
1394 {
1395 using bits_t = std::conditional_t<sizeof(R) == 8, std::uint64_t, std::uint32_t>;
1396 constexpr unsigned fraction {static_cast<unsigned>(std::numeric_limits<R>::digits - 1)}; // the fraction's bits, below the exponent's
1397 constexpr bits_t carry {bits_t {1} << fraction};
1398 constexpr bits_t exponent {((bits_t {1} << ((8U * sizeof(R)) - fraction - 1U)) - 1U) << fraction};
1399 constexpr unsigned sign {(8U * sizeof(R)) - 1U};
1400 bits_t flagged {0};
1401 std::size_t i {0};
1402#if defined(__GNUC__)
1403 using lanes = std::conditional_t<sizeof(R) == 8, finite_lanes_64, finite_lanes_32>;
1404 constexpr std::size_t width {16 / sizeof(R)};
1405 constexpr lanes exponents {lanes {} + exponent};
1406 constexpr lanes carries {lanes {} + carry};
1407 lanes first {};
1408 lanes second {};
1409 lanes third {};
1410 lanes fourth {};
1411 for (; i + (4 * width) <= count; i += 4 * width) {
1412 std::array<lanes, 4> bits {};
1413 std::memcpy(bits.data(), x + i, sizeof(bits));
1414 first |= (bits[0] & exponents) + carries;
1415 second |= (bits[1] & exponents) + carries;
1416 third |= (bits[2] & exponents) + carries;
1417 fourth |= (bits[3] & exponents) + carries;
1418 }
1419 const lanes all {first | second | third | fourth};
1420 for (std::size_t lane {0}; lane < width; ++lane) {
1421 flagged |= all[lane];
1422 }
1423#endif
1424 for (; i < count; ++i) {
1425 bits_t bits {};
1426 std::memcpy(&bits, x + i, sizeof(bits));
1427 flagged |= (bits & exponent) + carry;
1428 }
1429 return (flagged >> sign) == 0;
1430 }
1431} // namespace numerical::detail
1432
1433#endif // NUMERICAL_SIMD_HPP
A complex number of float or double, which gemm_split multiplies through the real kernel and the kern...
Definition simd.hpp:188
T symmetric_row(const T *row, const T *v, const T &f, T *out, std::size_t count)
Σ row[i] · v[i] and out[i] += row[i] · f, in order, for a type no native vector holds.
Definition simd.hpp:1048
void transpose_walk(std::size_t i0, std::size_t i1, std::size_t j0, std::size_t j1, std::size_t piece, const Copy &copy)
transpose_pieces of side piece (transpose_piece's 8 or the whole tile), each side its own instance: a...
Definition simd.hpp:142
void subtract_rows(double *y, std::size_t ldy, const double *f, std::size_t ldf, const double *x, std::size_t ldx, std::size_t terms, std::size_t count)
subtract_rows for double, the units chosen as subtract_products chooses them: each element the value ...
Definition gemm.hpp:551
T conjugate_dot_product(const T *x, const T *y, std::size_t count)
Σ conj(x[i]) · y[i], in order, where no native vector holds the parts.
Definition simd.hpp:1089
void copy_few(const T *from, std::size_t count, T *to) noexcept
copy_few_bytes of count elements of T, one at least and at most 128 bytes.
Definition simd.hpp:77
constexpr std::size_t transpose_tile
The side of the tiles a transposing copy goes by (an einsum permutation of two axes,...
Definition simd.hpp:89
void add_one_product(T &y, const T &f, const T &x)
y += f · x of one element, as add_product computes each of its elements.
Definition simd.hpp:1343
std::size_t subtract_rows_vectors(T *, std::size_t, const T *, std::size_t, const T *, std::size_t, std::size_t, std::size_t)
No leading columns in vectors, for a type no native vector holds or a compiler that vectorises the sc...
Definition simd.hpp:1024
constexpr std::size_t symmetric_row_group
The rows of a lower triangle symmetric_rows takes at once.
Definition simd.hpp:158
carried_t< T > real_part(const T &x)
The real part of a real (or general) scalar, in carried_t.
T operator_times(const T &a, const T &b)
a · b, as std::operator* computes it, in line. libc++'s complex product (the four products and their ...
Definition simd.hpp:1122
std::array< T, symmetric_row_group > symmetric_rows_tail(const T *rows, std::size_t n, const T *v, T *out, std::size_t i, std::size_t done, std::array< T, symmetric_row_group > sums)
The columns done…i+r of symmetric_rows, one by one: for each row r in order, out[k] += row_r[k] · v[i...
Definition simd.hpp:165
void add_product(double *y, const double &f, const double *x, std::size_t count)
y[i] += f · x[i] for double, as subtract_product.
Definition gemm.hpp:648
std::size_t subtract_scaled_vectors(T *, const T *, const T &, std::size_t)
No leading elements in vectors, for a type no native vector holds.
Definition simd.hpp:989
void row_dots(const std::complex< double > *m, std::size_t ld, std::size_t rows, const std::complex< double > *x, std::size_t count, std::complex< double > *out)
row_dots for std::complex<double>, through complex_row_dots.
Definition gemm.hpp:802
void subtract_products(double *y, const double *f, const double *x, std::size_t ldx, std::size_t terms, std::size_t count)
y[i] -= Σₜ f[t] · x[(t · ldx) + i] for double, the units chosen as subtract_product chooses them,...
Definition gemm.hpp:522
void through_rows(T *c, std::size_t ldc, std::size_t rows, std::size_t cols, Full full) noexcept
through_edge for a tile full computes for any count of its rows up to Rows, its template argument: a ...
Definition simd.hpp:974
void subtract_product(double *y, const double &f, const double *x, std::size_t count)
y[i] −= f · x[i] for double on the widest x86-64 units the matrix product's kernel may use (gemm_path...
Definition gemm.hpp:485
std::array< T, symmetric_row_group > symmetric_rows(const T *rows, std::size_t n, const T *v, T *out, std::size_t i)
symmetric_rows one column at a time, for a type no native vector holds.
Definition simd.hpp:1065
void rows_of_tile(std::size_t rows, T *c, std::size_t ldc, Full &full) noexcept
full's tile of its first R rows, or of rows of them where fewer (through_rows).
Definition simd.hpp:949
std::size_t transpose_piece(std::size_t from_bytes, std::size_t to_bytes) noexcept
The side of the square pieces a transposing copy walks a tile by, from the bytes between rows of its ...
Definition simd.hpp:100
T less_product(T held, const T &factor, const T &x)
held − factor · x for the scalar blocks beside GCC's vectors, rounded as those vectors round it: unde...
Definition simd.hpp:1180
void transpose_pieces(std::size_t i0, std::size_t i1, std::size_t j0, std::size_t j1, const Copy &copy)
copy(i, j) for every i in [i0, i1) and j in [j0, j1), by square pieces of side Piece,...
Definition simd.hpp:117
void copy_few_bytes(const unsigned char *from, std::size_t bytes, unsigned char *to) noexcept
Copies bytes bytes, 4 to 128, from from to to (apart): by two or four copies of one fixed size that o...
Definition simd.hpp:44
std::size_t subtract_combined_vectors(T *, const T *, const T *, std::size_t, std::size_t, std::size_t)
No leading blocks in vectors, for a type no native vector holds or a compiler that vectorises the sca...
Definition simd.hpp:1011
bool all_finite(const R *x, std::size_t count) noexcept
Whether the count values at x are all finite: a NaN or an infinity has every exponent bit set,...
Definition simd.hpp:1392
double dot_product(const double *x, const double *y, std::size_t count)
Σ x[i] · y[i] for double, as subtract_product chooses its units (in another order of summation there,...
Definition gemm.hpp:715
void complex_scaled_add(C &y, const C &f, const C &x)
y += f · x of one complex number, its part products rounded apart as libc++'s operator computes them ...
Definition simd.hpp:1313
constexpr std::size_t product_rows
The rows of y subtract_rows takes at once.
Definition simd.hpp:731
std::size_t add_scaled_vectors(T *, const T *, const T &, std::size_t)
No leading elements in vectors, for a type no native vector holds.
Definition simd.hpp:1038
void through_edge(T *c, std::size_t ldc, std::size_t rows, std::size_t cols, Full full) noexcept
One tile of a matrix product kernel, rows × cols of C (at most Rows × Cols, leading dimension ldc),...
Definition simd.hpp:921
std::size_t largest_magnitude_vectors(const T *, std::size_t, Magnitude &, std::size_t &)
Only x[0], the one best and at come in with, for a type no native vector holds.
Definition simd.hpp:1000
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
constexpr fft_norm swapped(fft_norm norm) noexcept
norm with its forward and backward exchanged, as NumPy's hfft and ihfft take it.
Definition fft.hpp:3175
tensor< T > full(std::vector< std::size_t > shape, const T &value)
A tensor of shape with every element value.
#define NUMERICAL_ALWAYS_INLINE
Definition simd.hpp:25