6#ifndef NUMERICAL_SIMD_HPP
7#define NUMERICAL_SIMD_HPP
22#if defined(_MSC_VER) && !defined(__clang__)
23# define NUMERICAL_ALWAYS_INLINE __forceinline
25# define NUMERICAL_ALWAYS_INLINE [[gnu::always_inline]] inline
30#if defined(_MSC_VER) && !defined(__clang__)
31# define NUMERICAL_INLINE_LAMBDA
33# define NUMERICAL_INLINE_LAMBDA __attribute__((always_inline))
46 unsigned char*
to)
noexcept
50 std::memcpy(
to + 32,
from + 32, 32);
51 std::memcpy(
to + bytes - 64,
from + bytes - 64, 32);
52 std::memcpy(
to + bytes - 32,
from + bytes - 32, 32);
54 else if (bytes >= 32) {
56 std::memcpy(
to + bytes - 32,
from + bytes - 32, 32);
58 else if (bytes >= 16) {
60 std::memcpy(
to + bytes - 16,
from + bytes - 16, 16);
62 else if (bytes >= 8) {
64 std::memcpy(
to + bytes - 8,
from + bytes - 8, 8);
69 const std::size_t
last {std::max<std::size_t>(bytes, 4) - 4};
81 static_assert(
sizeof(
T) >= 4,
"an element of four bytes at least");
82 copy_few_bytes(
reinterpret_cast<const unsigned char*
>(
from), count *
sizeof(
T),
reinterpret_cast<unsigned char*
>(
to));
103#if defined(__x86_64__) || defined(_M_X64)
104 constexpr std::size_t
aliased {std::size_t {1} << 10U};
106 constexpr std::size_t
aliased {std::size_t {1} << 13U};
115 template <std::size_t
Piece,
127 for (std::size_t i {
pi}; i <
pi1; ++i) {
128 for (std::size_t j {
pj}; j <
pj1; ++j) {
141 template <
typename Copy>
164 template <
typename T>
171 std::array<T, symmetric_row_group>
sums)
174 const T*
row {rows + (r * n)};
175 const T factor {v[i + r]};
176 for (std::size_t k {done}; k < i + r; ++k) {
178 out[k] +=
row[k] * factor;
180 sums[r] +=
row[i + r] * factor;
187 template <
typename T>
188 concept split_complex = std::is_same_v<T, std::complex<float>> || std::is_same_v<T, std::complex<double>>;
199 template <
typename T>
225 template <
typename T>
226 concept native_scalar = std::is_same_v<T, float> || std::is_same_v<T, double>;
233 template <native_scalar T>
238 using lane =
typename native_vector<T>::type;
240 constexpr std::size_t lanes {4};
241 std::array<lane, lanes>
sum {};
242 const std::size_t done {count - (count % (width * lanes))};
243 for (std::size_t i {0}; i < done; i += width * lanes) {
245 for (std::size_t l {0}; l < lanes; ++l) {
248 std::memcpy(&
xv, x + i + (l * width),
sizeof(
lane));
249 std::memcpy(&
yv,
y + i + (l * width),
sizeof(
lane));
254 for (std::size_t l {0}; done != 0 && l < lanes; ++l) {
255 for (std::size_t k {0}; k < width; ++k) {
259 for (std::size_t i {done}; i < count; ++i) {
274 template <split_complex C>
279 using R =
typename C::value_type;
280 using lane =
typename native_vector<R>::type;
282 constexpr std::size_t lanes {4};
283 const auto*
const a {
reinterpret_cast<const R*
>(x)};
284 const auto*
const b {
reinterpret_cast<const R*
>(
y)};
285 const std::size_t parts {2 * count};
286 std::array<lane, lanes>
same {};
287 std::array<lane, lanes>
cross {};
288 const std::size_t done {parts - (parts % (width * lanes))};
289 for (std::size_t i {0}; i < done; i += width * lanes) {
291 for (std::size_t l {0}; l < lanes; ++l) {
294 std::memcpy(&
av, a + i + (l * width),
sizeof(
lane));
295 std::memcpy(&
bv, b + i + (l * width),
sizeof(
lane));
297 for (std::size_t k {0}; k < width; ++k) {
310 for (std::size_t l {0}; l < lanes; ++l) {
311 for (std::size_t k {0}; k < width; k += 2) {
315 ir +=
cross[l][k + 1];
318 for (std::size_t i {done}; i < parts; i += 2) {
320 ii += a[i + 1] * b[i + 1];
321 ri += a[i] * b[i + 1];
322 ir += a[i + 1] * b[i];
328 template <split_complex C>
334 return C {
rr -
ii,
ri + ir};
339 template <split_complex C>
345 return C {
rr +
ii,
ri - ir};
355 template <native_scalar T>
363 using lane =
typename native_vector<T>::type;
365 constexpr std::size_t lanes {4};
366 constexpr std::size_t
group {4};
367 const std::size_t done {count - (count % (width * lanes))};
370 std::array<std::array<lane, lanes>,
group>
sum {};
371 for (std::size_t i {0}; i < done; i += width * lanes) {
373 for (std::size_t l {0}; l < lanes; ++l) {
375 std::memcpy(&
xv, x + i + (l * width),
sizeof(
lane));
377 for (std::size_t r {0}; r <
group; ++r) {
379 std::memcpy(&
mv, m + ((
r0 + r) * ld) + i + (l * width),
sizeof(
lane));
384 std::array<T, group>
total {};
385 for (std::size_t l {0}; done != 0 && l < lanes; ++l) {
386 for (std::size_t k {0}; k < width; ++k) {
388 for (std::size_t r {0}; r <
group; ++r) {
393 for (std::size_t i {done}; i < count; ++i) {
395 for (std::size_t r {0}; r <
group; ++r) {
396 total[r] += m[((
r0 + r) * ld) + i] * x[i];
399 for (std::size_t r {0}; r <
group; ++r) {
403 for (;
r0 < rows; ++
r0) {
416 template <native_scalar T>
424 using lane =
typename native_vector<T>::type;
426 constexpr std::size_t
partials {4 * width};
427 const std::size_t done {cols - (cols %
partials)};
428 std::array<lane, partials>
sum {};
429 std::array<T, width>
part {};
430 const auto finish {[&](std::size_t
r0, std::size_t count) {
431 for (std::size_t r {0}; r < count; ++r) {
433 for (std::size_t p {0}; done != 0 && p <
partials; ++p) {
440 for (;
r0 + width <= rows;
r0 += width) {
442 for (std::size_t j {0}; j < done; j +=
partials) {
443# pragma GCC unroll 16
444 for (std::size_t p {0}; p <
partials; ++p) {
446 std::memcpy(&
mv, a + ((j + p) * lda) +
r0,
sizeof(
lane));
456 for (std::size_t j {0}; j < done; j +=
partials) {
457 for (std::size_t p {0}; p <
partials; ++p) {
458 std::fill(
part.begin(),
part.end(),
T {});
459 std::copy(a + ((j + p) * lda) +
r0, a + ((j + p) * lda) + rows,
part.begin());
467 finish(
r0, rows -
r0);
469 for (std::size_t j {done}; j < cols; ++j) {
470 const T*
const col {a + (j * lda)};
471 for (std::size_t r {0}; r < rows; ++r) {
472 y[r] +=
col[r] * x[j];
485 template <split_complex C>
493 using R =
typename C::value_type;
494 using lane =
typename native_vector<R>::type;
496 constexpr std::size_t
per {width / 2};
497 constexpr std::size_t
slots {2 * width};
498 const std::size_t done {cols - (cols %
slots)};
499 const auto*
const parts {
reinterpret_cast<const R*
>(a)};
500 std::vector<R>
ahead (2 * width * done);
501 for (std::size_t j {0}; j < done; ++j) {
502 for (std::size_t k {0}; k < width; ++k) {
503 ahead[(2 * j * width) + k] = (k % 2 == 0) ? x[j].real() : x[j].imag();
504 ahead[(((2 * j) + 1) * width) + k] = (k % 2 == 0) ? x[j].imag() : x[j].real();
507 std::vector<R>
sums (4 * rows);
508 std::array<lane, slots>
same {};
509 std::array<lane, slots>
cross {};
510 std::array<R, width>
part {};
511 const auto step {[&](std::size_t j, std::size_t q,
const R*
column) {
516 std::memcpy(&
xs,
ahead.data() + (2 * (j + q) * width),
sizeof(
lane));
517 std::memcpy(&
xc,
ahead.data() + (((2 * (j + q)) + 1) * width),
sizeof(
lane));
521 for (std::size_t
r0 {0};
r0 < rows;
r0 +=
per) {
522 const std::size_t count {std::min(
per, rows -
r0)};
526 for (std::size_t j {0}; j < done; j +=
slots) {
527# pragma GCC unroll 16
528 for (std::size_t q {0}; q <
slots; ++q) {
529 step(j, q, parts + (2 * (((j + q) * lda) +
r0)));
534 for (std::size_t j {0}; j < done; j +=
slots) {
535 for (std::size_t q {0}; q <
slots; ++q) {
536 std::fill(
part.begin(),
part.end(), R {});
537 std::copy(parts + (2 * (((j + q) * lda) +
r0)), parts + (2 * (((j + q) * lda) +
r0 + count)),
part.begin());
542 for (std::size_t v {0}; v < count; ++v) {
547 for (std::size_t q {0}; q <
slots; ++q) {
549 ii +=
same[q][(2 * v) + 1];
551 ir +=
cross[q][(2 * v) + 1];
556 sums[(3 * rows) +
r0 + v] = ir;
559 for (std::size_t j {done}; j < cols; ++j) {
560 const R*
const col {parts + (2 * j * lda)};
561 const R
xr {x[j].real()};
562 const R
xi {x[j].imag()};
563 for (std::size_t r {0}; r < rows; ++r) {
567 sums[(3 * rows) + r] +=
col[(2 * r) + 1] *
xr;
570 for (std::size_t r {0}; r < rows; ++r) {
571 y[r] = C {
sums[r] -
sums[rows + r],
sums[(2 * rows) + r] +
sums[(3 * rows) + r]};
580 template <native_scalar T>
587 using lane =
typename native_vector<T>::type;
589 constexpr std::size_t lanes {4};
592 std::array<lane, lanes>
sum {};
593 const std::size_t done {count - (count % (width * lanes))};
594 for (std::size_t i {0}; i < done; i += width * lanes) {
596 for (std::size_t l {0}; l < lanes; ++l) {
600 std::memcpy(&
rv,
row + i + (l * width),
sizeof(
lane));
601 std::memcpy(&
vv, v + i + (l * width),
sizeof(
lane));
602 std::memcpy(&
ov, out + i + (l * width),
sizeof(
lane));
605 std::memcpy(out + i + (l * width), &
ov,
sizeof(
lane));
609 for (std::size_t l {0}; done != 0 && l < lanes; ++l) {
610 for (std::size_t k {0}; k < width; ++k) {
614 for (std::size_t i {done}; i < count; ++i) {
616 out[i] +=
row[i] * factor;
627 template <native_scalar T>
634 using lane =
typename native_vector<T>::type;
636 std::array<lane, symmetric_row_group>
sum {};
637 std::array<lane, symmetric_row_group>
splat {};
641 const std::size_t done {i - (i % width)};
642 for (std::size_t j {0}; j < done; j += width) {
645 std::memcpy(&
vv, v + j,
sizeof(
lane));
646 std::memcpy(&
ov, out + j,
sizeof(
lane));
650 std::memcpy(&
rv, rows + (r * n) + j,
sizeof(
lane));
654 std::memcpy(out + j, &
ov,
sizeof(
lane));
656 std::array<T, symmetric_row_group>
sums {};
658 for (std::size_t k {0}; k < width; ++k) {
679 template <native_scalar T>
685 using lane =
typename native_vector<T>::type;
687 using index = std::remove_cvref_t<
decltype(
mask {}[0])>;
689 constexpr std::size_t
ways {4};
690 if (count < 1 + (
ways * width)) {
694 std::array<lane, ways>
top {};
695 std::array<mask, ways>
where {};
696 std::array<mask, ways> next {};
697 for (std::size_t w {0}; w <
ways; ++w) {
699 for (std::size_t k {0}; k < width; ++k) {
700 next[w][k] =
static_cast<index
>(1 + (w * width) + k);
704 for (; i + (
ways * width) <= count; i +=
ways * width) {
706 for (std::size_t w {0}; w <
ways; ++w) {
708 std::memcpy(&v, x + i + (w * width),
sizeof(
lane));
709 const lane v_abs {std::bit_cast<lane>(std::bit_cast<mask>(v) & magnitude)};
711 top[w] = std::bit_cast<lane>((std::bit_cast<mask>(
v_abs) & larger) | (std::bit_cast<mask>(
top[w]) & ~larger));
712 where[w] = (next[w] & larger) | (
where[w] & ~larger);
713 next[w] +=
static_cast<index
>(
ways * width);
716 for (std::size_t w {0}; w <
ways; ++w) {
717 for (std::size_t k {0}; k < width; ++k) {
718 const auto lane_at {
static_cast<std::size_t
>(
where[w][k])};
733#if defined(__GNUC__) && !defined(__clang__)
736 template <native_scalar T>
742 using lane =
typename native_vector<T>::type;
744 const std::size_t done {count - (count % width)};
745 for (std::size_t i {0}; i < done; i += width) {
748 std::memcpy(&
yv,
y + i,
sizeof(
lane));
749 std::memcpy(&
xv, x + i,
sizeof(
lane));
751 std::memcpy(
y + i, &
yv,
sizeof(
lane));
764 template <native_scalar T>
772 using lane =
typename native_vector<T>::type;
773 constexpr std::size_t lanes {4};
775 const std::size_t done {count - (count % (lanes * width))};
776 for (std::size_t i {0}; i < done; i += lanes * width) {
777 std::array<lane, lanes>
held {};
779 for (std::size_t u {0}; u < lanes; ++u) {
780 std::memcpy(&
held[u],
y + i + (u * width),
sizeof(
lane));
782 for (std::size_t t {0}; t < terms; ++t) {
783 const T factor {f[t]};
785 for (std::size_t u {0}; u < lanes; ++u) {
787 std::memcpy(&
row, x + (t *
ldx) + i + (u * width),
sizeof(
lane));
792 for (std::size_t u {0}; u < lanes; ++u) {
793 std::memcpy(
y + i + (u * width), &
held[u],
sizeof(
lane));
804 template <native_scalar T>
814 using lane =
typename native_vector<T>::type;
816 const std::size_t done {count - (count % width)};
817 for (std::size_t i {0}; i < done; i += width) {
818 std::array<lane, product_rows>
held {};
823 for (std::size_t t {0}; t < terms; ++t) {
825 std::memcpy(&
row, x + (t *
ldx) + i,
sizeof(
lane));
828 const T factor {f[(r *
ldf) + t]};
841 template <native_scalar T>
847 using lane =
typename native_vector<T>::type;
849 const std::size_t done {count - (count % width)};
850 for (std::size_t i {0}; i < done; i += width) {
853 std::memcpy(&
yv,
y + i,
sizeof(
lane));
854 std::memcpy(&
xv, x + i,
sizeof(
lane));
856 std::memcpy(
y + i, &
yv,
sizeof(
lane));
867 template <split_complex C>
873 using R = C::value_type;
874 using lane =
typename native_vector<R>::type;
876 auto*
const out {
reinterpret_cast<R*
>(
y)};
877 const auto*
const in {
reinterpret_cast<const R*
>(x)};
881 for (std::size_t k {0}; k < width; ++k) {
884 sign[k] = (k % 2 == 0) ? R {-1} : R {1};
886 const std::size_t parts {2 * count};
887 const std::size_t done {parts - (parts % width)};
888 for (std::size_t i {0}; i < done; i += width) {
891 std::memcpy(&
yv, out + i,
sizeof(
lane));
892 std::memcpy(&
xv,
in + i,
sizeof(
lane));
894 for (std::size_t k {0}; k < width; ++k) {
904 std::memcpy(out + i, &
yv,
sizeof(
lane));
917 template <std::size_t
Rows,
931 std::array<T, Rows * Cols>
edge {};
932 for (std::size_t i {0}; i < rows; ++i) {
933 for (std::size_t j {0}; j < cols; ++j) {
934 edge[(i *
Cols) + j] = c[(i * ldc) + j];
938 for (std::size_t i {0}; i < rows; ++i) {
939 for (std::size_t j {0}; j < cols; ++j) {
940 c[(i * ldc) + j] =
edge[(i *
Cols) + j];
946 template <std::size_t R,
954 if constexpr (R > 1) {
970 template <std::size_t
Rows,
988 template <
typename T>
998 template <
typename T,
1010 template <
typename T>
1023 template <
typename T>
1037 template <
typename T>
1047 template <
typename T>
1056 for (std::size_t i {0}; i < count; ++i) {
1058 out[i] +=
row[i] * factor;
1064 template <
typename T>
1075 template <
typename T>
1081 for (std::size_t i {0}; i < count; ++i) {
1088 template <
typename T>
1094 for (std::size_t i {0}; i < count; ++i) {
1095 total += std::conj(x[i]) *
y[i];
1101 template <
typename T>
1109 for (std::size_t r {0}; r < rows; ++r) {
1121 template <
typename T>
1128#if defined(_LIBCPP_VERSION)
1130 template <
typename R>
1132 const std::complex<R>& b)
1134 const R
ac {a.real() * b.real()};
1135 const R
bd {a.imag() * b.imag()};
1136 const R
ad {a.real() * b.imag()};
1137 const R
bc {a.imag() * b.real()};
1138 const R x {
ac -
bd};
1139 const R
y {
ad +
bc};
1140 if (std::isnan(x) && std::isnan(
y)) [[
unlikely]] {
1158 template <
typename T>
1179 template <
typename T>
1184#if defined(__GNUC__) && !defined(__clang__)
1188 return held - (factor * x);
1197 template <
typename T>
1205 for (std::size_t t {0}; t < terms; ++t) {
1218 template <
typename T>
1219 requires std::is_floating_point_v<T>
1232 const auto blocks = [&]<std::size_t
Width>(std::integral_constant<std::size_t, Width>) {
1234 std::array<T, Width>
held {};
1236 for (std::size_t t {0}; t < terms; ++t) {
1237 const T factor {f[t]};
1238 const T*
const row {x + (t *
ldx) + i};
1239 for (std::size_t u {0}; u <
Width; ++u) {
1246 blocks(std::integral_constant<std::size_t, 32> {});
1247 blocks(std::integral_constant<std::size_t, 8> {});
1248 blocks(std::integral_constant<std::size_t, 4> {});
1249 blocks(std::integral_constant<std::size_t, 2> {});
1250 blocks(std::integral_constant<std::size_t, 1> {});
1260 template <
typename T>
1261 requires std::is_floating_point_v<T>
1271 constexpr std::size_t width {4};
1273 const auto columns = [&]<std::size_t
Width>(std::integral_constant<std::size_t, Width>) {
1279 for (std::size_t t {0}; t < terms; ++t) {
1280 const T*
const row {x + (t *
ldx) + i};
1282 const T factor {f[(r *
ldf) + t]};
1283 for (std::size_t u {0}; u <
Width; ++u) {
1293 columns(std::integral_constant<std::size_t, width> {});
1294 columns(std::integral_constant<std::size_t, 1> {});
1298 template <
typename T>
1312 template <split_complex C>
1317 using R = C::value_type;
1318 const R
ac {f.real() * x.real()};
1319 const R
bd {f.imag() * x.imag()};
1320 const R
ad {f.real() * x.imag()};
1321 const R
bc {f.imag() * x.real()};
1322 const R re {
ac -
bd};
1323 const R im {
ad +
bc};
1324 y = C {
y.real() + re,
y.imag() + im};
1329 template <split_complex C>
1342 template <
typename T>
1351 template <split_complex C>
1360 template <
typename T>
1372#if defined(__GNUC__)
1390 template <
typename R>
1391 requires std::is_same_v<R, double> || std::is_same_v<R, float>
1393 std::size_t count)
noexcept
1395 using bits_t = std::conditional_t<
sizeof(R) == 8, std::uint64_t, std::uint32_t>;
1396 constexpr unsigned fraction {
static_cast<unsigned>(std::numeric_limits<R>::digits - 1)};
1399 constexpr unsigned sign {(8U *
sizeof(R)) - 1U};
1402#if defined(__GNUC__)
1404 constexpr std::size_t width {16 /
sizeof(R)};
1405 constexpr lanes
exponents {lanes {} + exponent};
1411 for (; i + (4 * width) <= count; i += 4 * width) {
1412 std::array<lanes, 4>
bits {};
1413 std::memcpy(
bits.data(), x + i,
sizeof(
bits));
1424 for (; i < count; ++i) {
1426 std::memcpy(&
bits, x + i,
sizeof(
bits));
A complex number of float or double, which gemm_split multiplies through the real kernel and the kern...
T symmetric_row(const T *row, const T *v, const T &f, T *out, std::size_t count)
Σ row[i] · v[i] and out[i] += row[i] · f, in order, for a type no native vector holds.
void transpose_walk(std::size_t i0, std::size_t i1, std::size_t j0, std::size_t j1, std::size_t piece, const Copy ©)
transpose_pieces of side piece (transpose_piece's 8 or the whole tile), each side its own instance: a...
void subtract_rows(double *y, std::size_t ldy, const double *f, std::size_t ldf, const double *x, std::size_t ldx, std::size_t terms, std::size_t count)
subtract_rows for double, the units chosen as subtract_products chooses them: each element the value ...
T conjugate_dot_product(const T *x, const T *y, std::size_t count)
Σ conj(x[i]) · y[i], in order, where no native vector holds the parts.
void copy_few(const T *from, std::size_t count, T *to) noexcept
copy_few_bytes of count elements of T, one at least and at most 128 bytes.
constexpr std::size_t transpose_tile
The side of the tiles a transposing copy goes by (an einsum permutation of two axes,...
void add_one_product(T &y, const T &f, const T &x)
y += f · x of one element, as add_product computes each of its elements.
std::size_t subtract_rows_vectors(T *, std::size_t, const T *, std::size_t, const T *, std::size_t, std::size_t, std::size_t)
No leading columns in vectors, for a type no native vector holds or a compiler that vectorises the sc...
constexpr std::size_t symmetric_row_group
The rows of a lower triangle symmetric_rows takes at once.
carried_t< T > real_part(const T &x)
The real part of a real (or general) scalar, in carried_t.
T operator_times(const T &a, const T &b)
a · b, as std::operator* computes it, in line. libc++'s complex product (the four products and their ...
std::array< T, symmetric_row_group > symmetric_rows_tail(const T *rows, std::size_t n, const T *v, T *out, std::size_t i, std::size_t done, std::array< T, symmetric_row_group > sums)
The columns done…i+r of symmetric_rows, one by one: for each row r in order, out[k] += row_r[k] · v[i...
void add_product(double *y, const double &f, const double *x, std::size_t count)
y[i] += f · x[i] for double, as subtract_product.
std::size_t subtract_scaled_vectors(T *, const T *, const T &, std::size_t)
No leading elements in vectors, for a type no native vector holds.
void row_dots(const std::complex< double > *m, std::size_t ld, std::size_t rows, const std::complex< double > *x, std::size_t count, std::complex< double > *out)
row_dots for std::complex<double>, through complex_row_dots.
void subtract_products(double *y, const double *f, const double *x, std::size_t ldx, std::size_t terms, std::size_t count)
y[i] -= Σₜ f[t] · x[(t · ldx) + i] for double, the units chosen as subtract_product chooses them,...
void through_rows(T *c, std::size_t ldc, std::size_t rows, std::size_t cols, Full full) noexcept
through_edge for a tile full computes for any count of its rows up to Rows, its template argument: a ...
void subtract_product(double *y, const double &f, const double *x, std::size_t count)
y[i] −= f · x[i] for double on the widest x86-64 units the matrix product's kernel may use (gemm_path...
std::array< T, symmetric_row_group > symmetric_rows(const T *rows, std::size_t n, const T *v, T *out, std::size_t i)
symmetric_rows one column at a time, for a type no native vector holds.
void rows_of_tile(std::size_t rows, T *c, std::size_t ldc, Full &full) noexcept
full's tile of its first R rows, or of rows of them where fewer (through_rows).
std::size_t transpose_piece(std::size_t from_bytes, std::size_t to_bytes) noexcept
The side of the square pieces a transposing copy walks a tile by, from the bytes between rows of its ...
T less_product(T held, const T &factor, const T &x)
held − factor · x for the scalar blocks beside GCC's vectors, rounded as those vectors round it: unde...
void transpose_pieces(std::size_t i0, std::size_t i1, std::size_t j0, std::size_t j1, const Copy ©)
copy(i, j) for every i in [i0, i1) and j in [j0, j1), by square pieces of side Piece,...
void copy_few_bytes(const unsigned char *from, std::size_t bytes, unsigned char *to) noexcept
Copies bytes bytes, 4 to 128, from from to to (apart): by two or four copies of one fixed size that o...
std::size_t subtract_combined_vectors(T *, const T *, const T *, std::size_t, std::size_t, std::size_t)
No leading blocks in vectors, for a type no native vector holds or a compiler that vectorises the sca...
bool all_finite(const R *x, std::size_t count) noexcept
Whether the count values at x are all finite: a NaN or an infinity has every exponent bit set,...
double dot_product(const double *x, const double *y, std::size_t count)
Σ x[i] · y[i] for double, as subtract_product chooses its units (in another order of summation there,...
void complex_scaled_add(C &y, const C &f, const C &x)
y += f · x of one complex number, its part products rounded apart as libc++'s operator computes them ...
constexpr std::size_t product_rows
The rows of y subtract_rows takes at once.
std::size_t add_scaled_vectors(T *, const T *, const T &, std::size_t)
No leading elements in vectors, for a type no native vector holds.
void through_edge(T *c, std::size_t ldc, std::size_t rows, std::size_t cols, Full full) noexcept
One tile of a matrix product kernel, rows × cols of C (at most Rows × Cols, leading dimension ldc),...
std::size_t largest_magnitude_vectors(const T *, std::size_t, Magnitude &, std::size_t &)
Only x[0], the one best and at come in with, for a type no native vector holds.
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
constexpr fft_norm swapped(fft_norm norm) noexcept
norm with its forward and backward exchanged, as NumPy's hfft and ihfft take it.
tensor< T > full(std::vector< std::size_t > shape, const T &value)
A tensor of shape with every element value.
#define NUMERICAL_ALWAYS_INLINE