11#ifndef NUMERICAL_X86_HPP
12#define NUMERICAL_X86_HPP
26#if defined(__x86_64__) || defined(_M_X64)
27# define NUMERICAL_X86_COMPILED
28# include <immintrin.h>
29# if defined(_MSC_VER) && !defined(__clang__)
31# define NUMERICAL_TARGET(isa)
32# define NUMERICAL_UNROLL_TILE
33# define NUMERICAL_UNROLL_DEPTH
34# define NUMERICAL_UNROLL_DEPTH_AVX2
37# define NUMERICAL_TARGET(isa) __attribute__((target(isa)))
39# define NUMERICAL_UNROLL_TILE _Pragma("GCC unroll 24")
42# define NUMERICAL_UNROLL_DEPTH _Pragma("GCC unroll 2")
45# define NUMERICAL_UNROLL_DEPTH_AVX2 _Pragma("GCC unroll 4")
51#ifdef NUMERICAL_X86_COMPILED
61 std::uint32_t
sub)
noexcept
63 std::array<std::uint32_t, 4> r {};
64# if defined(_MSC_VER) && !defined(__clang__)
65 std::array<int, 4>
regs {};
67 for (std::size_t i {0}; i < 4; ++i) {
68 r[i] =
static_cast<std::uint32_t
>(
regs[i]);
82 return static_cast<std::uint64_t
>(
_xgetbv(0));
88 std::uint64_t
state)
noexcept
103 const std::array<std::uint32_t, 4>
leaf1 {
cpuid(1, 0)};
106 const std::array<std::uint32_t, 4>
leaf7 {
cpuid(0, 0)[0] >= 7U ?
cpuid(7, 0)
121 template <
typename Body>
148 inline constexpr std::size_t
avx512_mr {12};
151 inline constexpr std::size_t
avx512_nr {16};
154 inline constexpr std::size_t
avx2_mr {6};
157 inline constexpr std::size_t
avx2_nr {8};
165 template <std::
size_t Rows = avx512_mr>
174 std::array<zmm_register, Rows * 2>
acc {};
176 for (std::size_t i {0}; i <
Rows; ++i) {
181 for (std::size_t p {0}; p <
kc; ++p) {
185 for (std::size_t i {0}; i <
Rows; ++i) {
192 for (std::size_t i {0}; i <
Rows; ++i) {
211 through_rows<avx512_mr, avx512_nr>(c, ldc, rows, cols, [&]<std::size_t
Rows>(
double* tile, std::size_t ld)
noexcept {
avx512_full_tile_f64<Rows>(
kc,
ap,
bp,
ldb, tile, ld,
fresh); });
221 template <std::
size_t Rows = avx2_mr>
230 std::array<ymm_register, Rows * 2>
acc {};
232 for (std::size_t i {0}; i <
Rows; ++i) {
237 for (std::size_t p {0}; p <
kc; ++p) {
245 for (std::size_t i {0}; i <
Rows; ++i) {
252 for (std::size_t i {0}; i <
Rows; ++i) {
271 through_rows<avx2_mr, avx2_nr>(c, ldc, rows, cols, [&]<std::size_t
Rows>(
double* tile, std::size_t ld)
noexcept {
avx2_full_tile_f64<Rows>(
kc,
ap,
bp,
ldb, tile, ld,
fresh); });
293 template <std::
size_t Rows = avx512_mr>
302 std::array<zmm_register_f32, Rows * 2>
acc {};
304 for (std::size_t i {0}; i <
Rows; ++i) {
309 for (std::size_t p {0}; p <
kc; ++p) {
313 for (std::size_t i {0}; i <
Rows; ++i) {
320 for (std::size_t i {0}; i <
Rows; ++i) {
338 through_rows<avx512_mr, avx512_nr_f32>(c, ldc, rows, cols, [&]<std::size_t
Rows>(
float* tile, std::size_t ld)
noexcept {
avx512_full_tile_f32<Rows>(
kc,
ap,
bp,
ldb, tile, ld,
fresh); });
342 template <std::
size_t Rows = avx2_mr>
351 std::array<ymm_register_f32, Rows * 2>
acc {};
353 for (std::size_t i {0}; i <
Rows; ++i) {
358 for (std::size_t p {0}; p <
kc; ++p) {
364 for (std::size_t i {0}; i <
Rows; ++i) {
371 for (std::size_t i {0}; i <
Rows; ++i) {
389 through_rows<avx2_mr, avx2_nr_f32>(c, ldc, rows, cols, [&]<std::size_t
Rows>(
float* tile, std::size_t ld)
noexcept {
avx2_full_tile_f32<Rows>(
kc,
ap,
bp,
ldb, tile, ld,
fresh); });
408 const std::complex<
double>*
ap,
409 const std::complex<
double>*
bp,
411 std::complex<
double>* tile,
414 std::
size_t a_row = 1,
419 const auto*
const a {
reinterpret_cast<const double*
>(
ap)};
420 const auto*
const b {
reinterpret_cast<const double*
>(
bp)};
421 auto*
const c {
reinterpret_cast<double*
>(tile)};
422 std::array<ymm_register, Rows * 4>
acc {};
424 for (std::size_t p {0}; p <
kc; ++p) {
428 for (std::size_t i {0}; i <
Rows; ++i) {
438 for (std::size_t i {0}; i <
Rows; ++i) {
439 double*
const row {c + (2 * i * ld)};
451 const std::complex<double>*
ap,
452 const std::complex<double>*
bp,
454 std::complex<double>* c,
468 const std::complex<double>* a,
471 const std::complex<double>*
bp,
473 std::complex<double> * c,
498 const std::complex<
double>*
ap,
499 const std::complex<
double>*
bp,
501 std::complex<
double>* tile,
504 std::
size_t a_row = 1,
509 const auto*
const a {
reinterpret_cast<const double*
>(
ap)};
510 const auto*
const b {
reinterpret_cast<const double*
>(
bp)};
511 auto*
const c {
reinterpret_cast<double*
>(tile)};
512 std::array<zmm_register, Rows * 4>
acc {};
514 for (std::size_t p {0}; p <
kc; ++p) {
518 for (std::size_t i {0}; i <
Rows; ++i) {
529 for (std::size_t i {0}; i <
Rows; ++i) {
530 double*
const row {c + (2 * i * ld)};
540 const std::complex<double>*
ap,
541 const std::complex<double>*
bp,
543 std::complex<double>* c,
557 const std::complex<double>* a,
560 const std::complex<double>*
bp,
562 std::complex<double> * c,
586 std::complex<
float>* tile,
589 std::
size_t a_row = 1,
594 const auto*
const a {
reinterpret_cast<const float*
>(
ap)};
595 const auto*
const b {
reinterpret_cast<const float*
>(
bp)};
596 auto*
const c {
reinterpret_cast<float*
>(tile)};
597 std::array<ymm_register_f32, Rows * 4>
acc {};
599 for (std::size_t p {0}; p <
kc; ++p) {
603 for (std::size_t i {0}; i <
Rows; ++i) {
613 for (std::size_t i {0}; i <
Rows; ++i) {
614 float*
const row {c + (2 * i * ld)};
625 const std::complex<float>*
ap,
626 const std::complex<float>*
bp,
628 std::complex<float>* c,
642 const std::complex<float>* a,
645 const std::complex<float>*
bp,
647 std::complex<float> * c,
669 std::complex<
float>* tile,
672 std::
size_t a_row = 1,
677 const auto*
const a {
reinterpret_cast<const float*
>(
ap)};
678 const auto*
const b {
reinterpret_cast<const float*
>(
bp)};
679 auto*
const c {
reinterpret_cast<float*
>(tile)};
680 std::array<zmm_register_f32, Rows * 4>
acc {};
682 for (std::size_t p {0}; p <
kc; ++p) {
686 for (std::size_t i {0}; i <
Rows; ++i) {
695 const __m512 signs {
_mm512_set_ps(1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F, 1.0F, -1.0F)};
697 for (std::size_t i {0}; i <
Rows; ++i) {
698 float*
const row {c + (2 * i * ld)};
708 const std::complex<float>*
ap,
709 const std::complex<float>*
bp,
711 std::complex<float>* c,
725 const std::complex<float>* a,
728 const std::complex<float>*
bp,
730 std::complex<float> * c,
782 std::size_t
apart)
noexcept
784 const auto v {[&](std::size_t k) {
return lane[k *
apart]; }};
785 return ((v(0) + v(4)) + (v(2) + v(6))) + ((v(1) + v(5)) + (v(3) + v(7)));
790 std::size_t
apart)
noexcept
792 const auto p {[&](std::size_t k) {
return lane[k *
apart] +
lane[(k + 8) *
apart]; }};
793 return ((p(0) + p(4)) + (p(2) + p(6))) + ((p(1) + p(5)) + (p(3) + p(7)));
807 constexpr std::size_t width {8};
808 constexpr std::size_t lanes {4};
810 std::array<zmm_register, lanes>
sum {};
811 const std::size_t done {count - (count % (width * lanes))};
812 for (std::size_t i {0}; i < done; i += width * lanes) {
814 for (std::size_t l {0}; l < lanes; ++l) {
821 for (std::size_t l {0}; l < lanes; ++l) {
824 for (std::size_t i {done}; i < count; ++i) {
826 out[i] +=
row[i] * f;
840 constexpr std::size_t width {4};
841 constexpr std::size_t lanes {8};
843 std::array<ymm_register, lanes>
sum {};
844 const std::size_t done {count - (count % (width * lanes))};
845 for (std::size_t i {0}; i < done; i += width * lanes) {
847 for (std::size_t l {0}; l < lanes; ++l) {
854 for (std::size_t l {0}; l < lanes; l += 2) {
857 for (std::size_t i {done}; i < count; ++i) {
859 out[i] +=
row[i] * f;
875 constexpr std::size_t width {8};
876 std::array<zmm_register, symmetric_row_group>
sum {};
877 std::array<zmm_register, symmetric_row_group>
splat {};
881 const std::size_t done {i - (i % width)};
882 for (std::size_t j {0}; j < done; j += width) {
893 std::array<double, symmetric_row_group>
sums {};
909 constexpr std::size_t width {8};
910 constexpr std::size_t half {4};
911 std::array<ymm_register, symmetric_row_group> low {};
912 std::array<ymm_register, symmetric_row_group> high {};
913 std::array<ymm_register, symmetric_row_group>
splat {};
917 const std::size_t done {i - (i % width)};
918 for (std::size_t j {0}; j < done; j += width) {
935 std::array<double, symmetric_row_group>
sums {};
951 const std::size_t done {count - (count % 8)};
952 for (std::size_t i {0}; i < done; i += 8) {
955 for (std::size_t i {done}; i < count; ++i) {
956 y[i] = std::fma(f, x[i],
y[i]);
974 for (; i + 32 <= count; i += 32) {
979 for (std::size_t t {0}; t < terms; ++t) {
981 const double*
const row {x + (t *
ldx) + i};
992 for (; i + 8 <= count; i += 8) {
994 for (std::size_t t {0}; t < terms; ++t) {
999 for (; i < count; ++i) {
1000 for (std::size_t t {0}; t < terms; ++t) {
1001 y[i] = std::fma(-f[t], x[(t *
ldx) + i],
y[i]);
1025 for (std::size_t i {0}; i < count; i += 8) {
1026 const auto mask {
static_cast<__mmask8>(count - i >= 8 ? 0xFFU : (1U << (count - i)) - 1U)};
1027 std::array<lane, 8>
held {};
1028 for (std::size_t r {0}; r < 8; ++r) {
1031 for (std::size_t t {0}; t < terms; ++t) {
1033 for (std::size_t r {0}; r < 8; ++r) {
1037 for (std::size_t r {0}; r < 8; ++r) {
1059 for (; i + 4 <= count; i += 4) {
1060 std::array<lane, 8>
held {};
1061 for (std::size_t r {0}; r < 8; ++r) {
1064 for (std::size_t t {0}; t < terms; ++t) {
1066 for (std::size_t r {0}; r < 8; ++r) {
1070 for (std::size_t r {0}; r < 8; ++r) {
1074 for (; i < count; ++i) {
1075 std::array<double, 8>
held {};
1076 for (std::size_t r {0}; r < 8; ++r) {
1079 for (std::size_t t {0}; t < terms; ++t) {
1080 const double value {x[(t *
ldx) + i]};
1081 for (std::size_t r {0}; r < 8; ++r) {
1082 held[r] = std::fma(-f[(r *
ldf) + t], value,
held[r]);
1085 for (std::size_t r {0}; r < 8; ++r) {
1101 for (; i + 16 <= count; i += 16) {
1106 for (std::size_t t {0}; t < terms; ++t) {
1108 const double*
const row {x + (t *
ldx) + i};
1119 for (; i + 4 <= count; i += 4) {
1121 for (std::size_t t {0}; t < terms; ++t) {
1126 for (; i < count; ++i) {
1127 for (std::size_t t {0}; t < terms; ++t) {
1128 y[i] = std::fma(-f[t], x[(t *
ldx) + i],
y[i]);
1149 for (; i + 16 <= count; i += 16) {
1150 std::array<zmm_register, 4>
acc {};
1152 for (std::size_t r {0}; r < 4; ++r) {
1155 for (std::size_t t {0}; t < terms; ++t) {
1158 const double*
const row {x + (2 * ((t *
ldx) + i))};
1160 for (std::size_t r {0}; r < 4; ++r) {
1167 for (std::size_t r {0}; r < 4; ++r) {
1171 for (; i + 4 <= count; i += 4) {
1173 for (std::size_t t {0}; t < terms; ++t) {
1180 for (; i < count; ++i) {
1181 double real {
y[2 * i]};
1182 double imag {
y[(2 * i) + 1]};
1183 for (std::size_t t {0}; t < terms; ++t) {
1184 const double xr {x[2 * ((t *
ldx) + i)]};
1185 const double xi {x[(2 * ((t *
ldx) + i)) + 1]};
1186 real = std::fma(-f[2 * t],
xr, real);
1188 real = std::fma(f[(2 * t) + 1],
xi, real);
1192 y[(2 * i) + 1] =
imag;
1207 for (; i + 8 <= count; i += 8) {
1208 std::array<ymm_register, 4>
acc {};
1210 for (std::size_t r {0}; r < 4; ++r) {
1213 for (std::size_t t {0}; t < terms; ++t) {
1216 const double*
const row {x + (2 * ((t *
ldx) + i))};
1218 for (std::size_t r {0}; r < 4; ++r) {
1225 for (std::size_t r {0}; r < 4; ++r) {
1229 for (; i + 2 <= count; i += 2) {
1231 for (std::size_t t {0}; t < terms; ++t) {
1238 for (; i < count; ++i) {
1239 double real {
y[2 * i]};
1240 double imag {
y[(2 * i) + 1]};
1241 for (std::size_t t {0}; t < terms; ++t) {
1242 const double xr {x[2 * ((t *
ldx) + i)]};
1243 const double xi {x[(2 * ((t *
ldx) + i)) + 1]};
1244 real = std::fma(-f[2 * t],
xr, real);
1246 real = std::fma(f[(2 * t) + 1],
xi, real);
1250 y[(2 * i) + 1] =
imag;
1273 for (; j + 4 <= count; j += 4) {
1285 alignas(64) std::array<double, 8>
lanes_same {};
1291 for (std::size_t l {0}; l < 8; l += 2) {
1295 for (; j < count; ++j) {
1296 const double rr {
row[2 * j]};
1297 const double ri {
row[(2 * j) + 1]};
1298 out[2 * j] = std::fma(
vi,
ri, std::fma(
vr,
rr, out[2 * j]));
1299 out[(2 * j) + 1] = std::fma(
vi,
rr, std::fma(-
vr,
ri, out[(2 * j) + 1]));
1300 re += (
rr * v[2 * j]) - (
ri * v[(2 * j) + 1]);
1301 im += (
ri * v[2 * j]) + (
rr * v[(2 * j) + 1]);
1304 return std::complex<double> {re, im};
1318 std::array<ymm_register, 2>
same {};
1319 std::array<ymm_register, 2>
across {};
1321 for (; j + 4 <= count; j += 4) {
1322 for (std::size_t h {0}; h < 2; ++h) {
1323 const std::size_t at {2 * (j + (2 * h))};
1335 alignas(64) std::array<double, 8>
lanes_same {};
1337 for (std::size_t h {0}; h < 2; ++h) {
1343 for (std::size_t l {0}; l < 8; l += 2) {
1347 for (; j < count; ++j) {
1348 const double rr {
row[2 * j]};
1349 const double ri {
row[(2 * j) + 1]};
1350 out[2 * j] = std::fma(
vi,
ri, std::fma(
vr,
rr, out[2 * j]));
1351 out[(2 * j) + 1] = std::fma(
vi,
rr, std::fma(-
vr,
ri, out[(2 * j) + 1]));
1352 re += (
rr * v[2 * j]) - (
ri * v[(2 * j) + 1]);
1353 im += (
ri * v[2 * j]) + (
rr * v[(2 * j) + 1]);
1356 return std::complex<double> {re, im};
1366 template <
bool Maximum>
1372 constexpr std::size_t block {16};
1373 const std::size_t done {count - (count % block)};
1377 std::array<ymm_register, 4>
best {};
1378 std::array<ymm_register, 4>
nans {};
1379 for (std::size_t l {0}; l < 4; ++l) {
1383 for (std::size_t i {0}; i < done; i += block) {
1384 for (std::size_t l {0}; l < 4; ++l) {
1390 std::array<double, block> values {};
1391 std::array<double, block>
flags {};
1392 for (std::size_t l {0}; l < 4; ++l) {
1397 for (std::size_t k {0}; k < block; ++k) {
1398 nan =
nan || std::bit_cast<std::uint64_t>(
flags[k]) != 0
U;
1405 template <
bool Maximum>
1411 constexpr std::size_t block {32};
1412 const std::size_t done {count - (count % block)};
1416 std::array<ymm_register_f32, 4>
best {};
1417 std::array<ymm_register_f32, 4>
nans {};
1418 for (std::size_t l {0}; l < 4; ++l) {
1422 for (std::size_t i {0}; i < done; i += block) {
1423 for (std::size_t l {0}; l < 4; ++l) {
1429 std::array<float, block> values {};
1430 std::array<float, block>
flags {};
1431 for (std::size_t l {0}; l < 4; ++l) {
1436 for (std::size_t k {0}; k < block; ++k) {
1437 nan =
nan || std::bit_cast<std::uint32_t>(
flags[k]) != 0
U;
1450 const std::size_t done {count - (count % 4)};
1451 for (std::size_t i {0}; i < done; i += 4) {
1454 for (std::size_t i {done}; i < count; ++i) {
1455 y[i] = std::fma(f, x[i],
y[i]);
1467 const std::size_t done {count - (count % 8)};
1468 for (std::size_t i {0}; i < done; i += 8) {
1471 for (std::size_t i {done}; i < count; ++i) {
1472 y[i] = std::fma(f, x[i],
y[i]);
1484 const std::complex<
float>* x,
1485 std::complex<
float> f,
1488 auto*
const out {
reinterpret_cast<float*
>(
y)};
1489 const auto*
const in {
reinterpret_cast<const float*
>(x)};
1492 const std::size_t done {count - (count % 4)};
1493 for (std::size_t i {0}; i < 2 * done; i += 8) {
1504 const std::complex<
double>* x,
1505 std::complex<
double> f,
1508 auto*
const out {
reinterpret_cast<double*
>(
y)};
1509 const auto*
const in {
reinterpret_cast<const double*
>(x)};
1512 const std::size_t done {count - (count % 2)};
1513 for (std::size_t i {0}; i < 2 * done; i += 4) {
1523 template <std::size_t
Width,
1533 std::array<T, Width>
sums {};
1534 for (std::size_t k {0}; k < inner; ++k) {
1537 for (std::size_t c {0}; c <
Width; ++c) {
1541 std::copy(
sums.begin(),
sums.end(),
y + j);
1551 template <
typename T>
1561 constexpr std::size_t
group {8};
1562 std::size_t j {first};
1584 constexpr std::size_t width {8};
1585 constexpr std::size_t lanes {4};
1586 std::array<zmm_register, lanes>
sum {};
1587 for (std::size_t i {0}; i < done; i += width * lanes) {
1589 for (std::size_t l {0}; l < lanes; ++l) {
1594 for (std::size_t l {0}; done != 0 && l < lanes; ++l) {
1605 const std::size_t done {count - (count % 32)};
1607 for (std::size_t i {done}; i < count; ++i) {
1620 constexpr std::size_t width {4};
1621 constexpr std::size_t lanes {8};
1622 std::array<ymm_register, lanes>
sum {};
1623 for (std::size_t i {0}; i < done; i += width * lanes) {
1625 for (std::size_t l {0}; l < lanes; ++l) {
1630 for (std::size_t l {0}; done != 0 && l < lanes; l += 2) {
1641 const std::size_t done {count - (count % 32)};
1643 for (std::size_t i {done}; i < count; ++i) {
1656 constexpr std::size_t width {16};
1657 constexpr std::size_t lanes {4};
1658 std::array<zmm_register_f32, lanes>
sum {};
1659 for (std::size_t i {0}; i < done; i += width * lanes) {
1661 for (std::size_t l {0}; l < lanes; ++l) {
1666 for (std::size_t l {0}; done != 0 && l < lanes; ++l) {
1677 const std::size_t done {count - (count % 64)};
1679 for (std::size_t i {done}; i < count; ++i) {
1692 constexpr std::size_t width {8};
1693 constexpr std::size_t lanes {8};
1694 std::array<ymm_register_f32, lanes>
sum {};
1695 for (std::size_t i {0}; i < done; i += width * lanes) {
1697 for (std::size_t l {0}; l < lanes; ++l) {
1702 for (std::size_t l {0}; done != 0 && l < lanes; l += 2) {
1713 const std::size_t done {count - (count % 64)};
1715 for (std::size_t i {done}; i < count; ++i) {
1729 template <std::
size_t Partials>
1737 for (std::size_t j {0}; j < done; ++j) {
1739 const double*
const col {a + (j * lda)};
1742 for (; r + 4 <= rows; r += 4) {
1745 for (; r < rows; ++r) {
1746 s[r] = std::fma(
col[r], x[j], s[r]);
1769 constexpr std::size_t width {8};
1770 constexpr std::size_t lanes {4};
1771 for (std::size_t r {0}; r < rows; ++r) {
1773 for (std::size_t l {0}; done != 0 && l < lanes; ++l) {
1778 for (std::size_t j {done}; j < cols; ++j) {
1779 const double*
const col {a + (j * lda)};
1780 for (std::size_t r {0}; r < rows; ++r) {
1781 y[r] +=
col[r] * x[j];
1791 template <std::
size_t Partials>
1799 for (std::size_t j {0}; j < done; ++j) {
1801 const float*
const col {a + (j * lda)};
1804 for (; r + 8 <= rows; r += 8) {
1807 for (; r < rows; ++r) {
1808 s[r] = std::fma(
col[r], x[j], s[r]);
1825 constexpr std::size_t width {16};
1826 constexpr std::size_t lanes {4};
1827 for (std::size_t r {0}; r < rows; ++r) {
1829 for (std::size_t l {0}; done != 0 && l < lanes; ++l) {
1834 for (std::size_t j {done}; j < cols; ++j) {
1835 const float*
const col {a + (j * lda)};
1836 for (std::size_t r {0}; r < rows; ++r) {
1837 y[r] +=
col[r] * x[j];
1848 template <
typename T, std::
size_t W
idth, T (*Blocks)(const T*, const T*, std::
size_t) noexcept>
1856 const std::size_t done {count - (count %
Width)};
1858 for (; r + 4 <= rows; r += 4) {
1859 const T*
const m0 {m + (r * ld)};
1862 T t2 {done == 0 ?
T {} :
Blocks(
m0 + (2 * ld), x, done)};
1863 T t3 {done == 0 ?
T {} :
Blocks(
m0 + (3 * ld), x, done)};
1864 for (std::size_t i {done}; i < count; ++i) {
1866 t1 +=
m0[ld + i] * x[i];
1867 t2 +=
m0[(2 * ld) + i] * x[i];
1868 t3 +=
m0[(3 * ld) + i] * x[i];
1875 for (; r < rows; ++r) {
1877 for (std::size_t i {done}; i < count; ++i) {
1878 total += m[(r * ld) + i] * x[i];
1893 const std::complex<
double>*
y,
1896 constexpr std::size_t width {4};
1897 constexpr std::size_t lanes {4};
1898 const auto*
const a {
reinterpret_cast<const double*
>(x)};
1899 const auto*
const b {
reinterpret_cast<const double*
>(
y)};
1900 const std::size_t parts {2 * count};
1901 std::array<ymm_register, lanes>
same {};
1902 std::array<ymm_register, lanes>
cross {};
1903 const std::size_t done {parts - (parts % (width * lanes))};
1904 for (std::size_t i {0}; i < done; i += width * lanes) {
1906 for (std::size_t l {0}; l < lanes; ++l) {
1917 for (std::size_t l {0}; l < lanes; ++l) {
1918 alignas(32) std::array<double, width> s {};
1919 alignas(32) std::array<double, width> c {};
1927 for (std::size_t i {done}; i < parts; i += 2) {
1929 ii += a[i + 1] * b[i + 1];
1930 ri += a[i] * b[i + 1];
1931 ir += a[i + 1] * b[i];
1934 return {
rr -
ii,
ri + ir};
1939 const std::complex<
float>*
y,
1942 constexpr std::size_t width {8};
1943 constexpr std::size_t lanes {4};
1944 const auto*
const a {
reinterpret_cast<const float*
>(x)};
1945 const auto*
const b {
reinterpret_cast<const float*
>(
y)};
1946 const std::size_t parts {2 * count};
1947 std::array<ymm_register_f32, lanes>
same {};
1948 std::array<ymm_register_f32, lanes>
cross {};
1949 const std::size_t done {parts - (parts % (width * lanes))};
1950 for (std::size_t i {0}; i < done; i += width * lanes) {
1952 for (std::size_t l {0}; l < lanes; ++l) {
1963 for (std::size_t l {0}; l < lanes; ++l) {
1964 alignas(32) std::array<float, width> s {};
1965 alignas(32) std::array<float, width> c {};
1968 rr += (s[0] + s[2]) + (s[4] + s[6]);
1969 ii += (s[1] + s[3]) + (s[5] + s[7]);
1970 ri += (c[0] + c[2]) + (c[4] + c[6]);
1971 ir += (c[1] + c[3]) + (c[5] + c[7]);
1973 for (std::size_t i {done}; i < parts; i += 2) {
1975 ii += a[i + 1] * b[i + 1];
1976 ri += a[i] * b[i + 1];
1977 ir += a[i + 1] * b[i];
1980 return {
rr -
ii,
ri + ir};
1986 template <
typename C>
1994 for (std::size_t r {0}; r < rows; ++r) {
2052 return _mm256_setr_ps(first, second, first, second, first, second, first, second);
2064 template <std::size_t
Slots,
2076 constexpr std::size_t width {32 /
sizeof(R)};
2077 constexpr std::size_t tile {4 * width};
2078 const std::size_t parts {2 * rows};
2079 for (std::size_t q {0}; q <
Slots; ++q) {
2080 R*
const same {
sums + (2 * q * parts)};
2083 for (; p + tile <= parts; p += tile) {
2085 std::array<held, 4> s {};
2086 std::array<held, 4> c {};
2087 for (std::size_t j {q}; j < done; j +=
Slots) {
2088 const R*
const col {a + (2 * j * lda) + p};
2092 for (std::size_t t {0}; t < 4; ++t) {
2098 for (std::size_t t {0}; t < 4; ++t) {
2103 for (; p < parts; p += 2) {
2104 for (std::size_t j {q}; j < done; j +=
Slots) {
2105 const R*
const col {a + (2 * j * lda) + p};
2106 const R
xr {x[2 * j]};
2107 const R
xi {x[(2 * j) + 1]};
2124 template <std::size_t
Slots,
2135 constexpr std::size_t lanes {4};
2136 constexpr std::size_t
per {
Slots / lanes};
2137 const auto slot {[&](std::size_t q, std::size_t kind, std::size_t r, std::size_t
part) {
2138 return sums[(2 * q * 2 * rows) + (kind * 2 * rows) + (2 * r) +
part];
2140 std::vector<R>
acc (4 * rows);
2141 for (std::size_t r {0}; r < rows; ++r) {
2146 for (std::size_t l {0}; done != 0 && l < lanes; ++l) {
2147 const std::size_t q {l *
per};
2148 if constexpr (
per == 2) {
2149 rr +=
slot(q, 0, r, 0) +
slot(q + 1, 0, r, 0);
2150 ii +=
slot(q, 0, r, 1) +
slot(q + 1, 0, r, 1);
2151 ri +=
slot(q, 1, r, 0) +
slot(q + 1, 1, r, 0);
2152 ir +=
slot(q, 1, r, 1) +
slot(q + 1, 1, r, 1);
2155 rr += (
slot(q, 0, r, 0) +
slot(q + 1, 0, r, 0)) + (
slot(q + 2, 0, r, 0) +
slot(q + 3, 0, r, 0));
2156 ii += (
slot(q, 0, r, 1) +
slot(q + 1, 0, r, 1)) + (
slot(q + 2, 0, r, 1) +
slot(q + 3, 0, r, 1));
2157 ri += (
slot(q, 1, r, 0) +
slot(q + 1, 1, r, 0)) + (
slot(q + 2, 1, r, 0) +
slot(q + 3, 1, r, 0));
2158 ir += (
slot(q, 1, r, 1) +
slot(q + 1, 1, r, 1)) + (
slot(q + 2, 1, r, 1) +
slot(q + 3, 1, r, 1));
2162 acc[(4 * r) + 1] =
ii;
2163 acc[(4 * r) + 2] =
ri;
2164 acc[(4 * r) + 3] = ir;
2166 for (std::size_t j {done}; j < cols; ++j) {
2167 const R*
const col {a + (2 * j * lda)};
2168 const R
xr {x[2 * j]};
2169 const R
xi {x[(2 * j) + 1]};
2170 for (std::size_t r {0}; r < rows; ++r) {
2172 acc[(4 * r) + 1] +=
col[(2 * r) + 1] *
xi;
2173 acc[(4 * r) + 2] +=
col[2 * r] *
xi;
2174 acc[(4 * r) + 3] +=
col[(2 * r) + 1] *
xr;
2177 for (std::size_t r {0}; r < rows; ++r) {
2178 y[r] = {
acc[4 * r] -
acc[(4 * r) + 1],
acc[(4 * r) + 2] +
acc[(4 * r) + 3]};
2212 const auto pair {[](
const float* lo,
const float* hi) {
2227 template <
typename R>
2239 constexpr std::size_t width {32 /
sizeof(R)};
2240 constexpr std::size_t tile {4 * width};
2241 const std::size_t parts {2 * count};
2243 for (std::size_t
pc {0};
pc < inner;
pc += depth) {
2244 const std::size_t end {std::min(
pc + depth, inner)};
2245 const bool first {
pc == 0};
2247 for (; p + tile <= parts; p += tile) {
2248 std::array<held, 4> re {};
2249 std::array<held, 4> im {};
2250 for (std::size_t k {
pc}; k < end; ++k) {
2255 for (std::size_t t {0}; t < 4; ++t) {
2261 for (std::size_t t {0}; t < 4; ++t) {
2262 R*
const at {
y + p + (t * width)};
2266 for (; p < parts; p += 2) {
2272 for (std::size_t k {
pc}; k < end; ++k) {
2275 rr = std::fma(x[2 * k],
br,
rr);
2276 ri = std::fma(x[2 * k],
bi,
ri);
2277 ir = std::fma(x[(2 * k) + 1],
br, ir);
2278 ii = std::fma(x[(2 * k) + 1],
bi,
ii);
2280 y[p] = (first ? R {0} :
y[p]) + (
rr -
ii);
2281 y[p + 1] = (first ? R {0} :
y[p + 1]) + (
ri + ir);
2401 template <
typename R>
2405 const R
ii {w.imag() * v.imag()};
2406 const R ir {w.imag() * v.real()};
2407 return {std::fma(w.real(), v.real(), -
ii), std::fma(w.real(), v.imag(), ir)};
2423 template <
typename V>
2487 template <
typename R>
2492 return {std::fma(c, v.real(), s.real()), std::fma(c, v.imag(), s.imag())};
2497 template <
typename R>
2500 const std::complex<R> * twiddles,
2506 const std::complex<R> w {twiddles[((k - 1) * run) + p - 1]};
2517 template <
typename R>
2521 std::complex<R>* out,
2522 const std::complex<R>* twiddles,
2525 using C = std::complex<R>;
2526 using vec =
decltype(
avx2_load(
static_cast<const R*
>(
nullptr)));
2527 constexpr std::size_t width {32 /
sizeof(C)};
2529 const std::size_t run {
part - 1};
2533 for (std::size_t p {0}; p <
part; ++p) {
2535 C*
const to {out + (
stride * 3 * p)};
2538 const C
w1 {p == 0 ? C {1} : twiddles[p - 1]};
2539 const C
w2 {p == 0 ? C {1} : twiddles[run + p - 1]};
2544 for (; q + width <=
stride; q += width) {
2545 const R*
const at {
reinterpret_cast<const R*
>(
in_p + q)};
2558 R*
const place {
reinterpret_cast<R*
>(
to + q)};
2564 for (; q <
stride; ++q) {
2568 const C t {
x1 +
x2};
2569 const C v {
x0 - C {R(0.5) * t.real(), R(0.5) * t.imag()}};
2570 const C d {
x1 -
x2};
2571 const C u {half_root_three * d.real(), half_root_three * d.imag()};
2572 const C
iu {-u.imag(), u.real()};
2585 template <
typename R>
2589 std::complex<R>* out,
2590 const std::complex<R>* twiddles,
2596 using C = std::complex<R>;
2597 using vec =
decltype(
avx2_load(
static_cast<const R*
>(
nullptr)));
2598 constexpr std::size_t width {32 /
sizeof(C)};
2600 const std::size_t run {
part - 1};
2607 for (std::size_t p {0}; p <
part; ++p) {
2609 C*
const to {out + (
stride * 5 * p)};
2612 for (; q + width <=
stride; q += width) {
2613 const R*
const at {
reinterpret_cast<const R*
>(
in_p + q)};
2627 R*
const place {
reinterpret_cast<R*
>(
to + q)};
2635 for (; q <
stride; ++q) {
2649 const C
ib1 {-
b1.imag(),
b1.real()};
2650 const C
ib2 {-
b2.imag(),
b2.real()};
2653 for (std::size_t j {1}; j < 5; ++j) {
2705 template <
typename R>
2709 std::complex<R>* out,
2710 const std::complex<R>* twiddles,
2713 using vec =
decltype(
avx2_load(
static_cast<const R*
>(
nullptr)));
2714 constexpr std::size_t width {32 /
sizeof(std::complex<R>)};
2716 const std::size_t run {
part - 1};
2719 const R*
const from {
reinterpret_cast<const R*
>(
in)};
2720 R*
const into {
reinterpret_cast<R*
>(out)};
2721 for (std::size_t p {0}; p <
part; ++p) {
2723 R*
const to {into + (2 *
stride * 8 * p)};
2724 for (std::size_t q {0}; q + width <=
stride; q += width) {
2725 const R*
const at {
in_p + (2 * q)};
2750 avx2_four_point(
avx2_add(
x0,
x4),
avx2_add(
x1,
x5),
avx2_add(
x2,
x6),
avx2_add(
x3,
x7),
e0,
e1,
e2,
e3);
2752 R*
const place {
to + (2 * q)};
2796 template <
typename R>
2798 const std::complex<R>* z,
2799 const std::complex<R>* roots,
2802 using vec =
decltype(
avx2_load(
static_cast<const R*
>(
nullptr)));
2803 constexpr std::size_t width {32 /
sizeof(std::complex<R>)};
2805 const R*
const from {
reinterpret_cast<const R*
>(z)};
2806 const R*
const root {
reinterpret_cast<const R*
>(roots)};
2807 R*
const into {
reinterpret_cast<R*
>(
X)};
2809 for (; k + width <= h; k += width) {
2827 template <
typename R>
2829 const std::complex<R>*
X,
2830 const std::complex<R>* roots,
2833 using vec =
decltype(
avx2_load(
static_cast<const R*
>(
nullptr)));
2834 constexpr std::size_t width {32 /
sizeof(std::complex<R>)};
2836 const R*
const from {
reinterpret_cast<const R*
>(
X)};
2837 const R*
const root {
reinterpret_cast<const R*
>(roots)};
2838 R*
const into {
reinterpret_cast<R*
>(z)};
2840 for (; k + width <= h; k += width) {
2875 template <
typename Isa,
3227 double*
const into {
reinterpret_cast<double*
>(
to)};
constexpr std::size_t symmetric_row_group
The rows of a lower triangle symmetric_rows takes at once.
std::array< T, symmetric_row_group > symmetric_rows_tail(const T *rows, std::size_t n, const T *v, T *out, std::size_t i, std::size_t done, std::array< T, symmetric_row_group > sums)
The columns done…i+r of symmetric_rows, one by one: for each row r in order, out[k] += row_r[k] · v[i...
T root(const T &x)
The square root of a scalar, as std::sqrt gives it.
@ zero
C = A·B: C's elements before the product are not read.
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
The native vector of float and double the kernels hold their lanes in (GNU vector extensions,...