18#ifndef NUMERICAL_GEMM_HPP
19#define NUMERICAL_GEMM_HPP
36#if defined(__GNUC__) && defined(__aarch64__)
38# define NUMERICAL_NEON_COMPILED
50#if !defined(NUMERICAL_OUT_OF_LINE)
51# if defined(_MSC_VER) && !defined(__clang__)
52# define NUMERICAL_OUT_OF_LINE __declspec(noinline)
54# define NUMERICAL_OUT_OF_LINE [[gnu::noinline]]
62# define NUMERICAL_SCALAR_DEPTH _Pragma("clang loop vectorize(disable) interleave(disable)")
64# define NUMERICAL_SCALAR_DEPTH
70# define NUMERICAL_UNROLL_BLOCK _Pragma("GCC unroll 32")
72# define NUMERICAL_UNROLL_BLOCK
95#ifdef NUMERICAL_NEON_COMPILED
105#ifdef NUMERICAL_X86_COMPILED
118 for (
auto p {
static_cast<int>(
most)}; p > 0; --p) {
138 static std::atomic<std::uint8_t>
found {0xFF};
150 const unsigned cap {
static_cast<unsigned>(
gemm_path_cap().load(std::memory_order_relaxed))};
156 gemm_path_found().store(
static_cast<std::uint8_t
>((
cap << 3U) |
static_cast<unsigned>(
path)), std::memory_order_relaxed);
170#ifdef NUMERICAL_NEON_COMPILED
177#ifdef NUMERICAL_NEON_COMPILED
184 std::size_t count)
noexcept
188 for (; i + 8 <= count; i += 8) {
194 for (; i + 2 <= count; i += 2) {
197 for (; i < count; ++i) {
198 y[i] = std::fma(f, x[i],
y[i]);
208 std::size_t count)
noexcept
211# if !defined(__clang__)
213 for (; i + 16 <= count; i += 16) {
219 for (; i + 4 <= count; i += 4) {
223 for (; i < count; ++i) {
224 y[i] = std::fma(f, x[i],
y[i]);
236 std::size_t count)
noexcept
240 for (; i + 32 <= count; i += 32) {
257 for (std::size_t t {0}; t < terms; ++t) {
259 const double*
const row {x + (t *
ldx) + i};
294 for (; i + 16 <= count; i += 16) {
303 for (std::size_t t {0}; t < terms; ++t) {
305 const double*
const row {x + (t *
ldx) + i};
324 for (; i + 8 <= count; i += 8) {
329 for (std::size_t t {0}; t < terms; ++t) {
331 const double*
const row {x + (t *
ldx) + i};
342 for (; i + 4 <= count; i += 4) {
345 for (std::size_t t {0}; t < terms; ++t) {
347 const double*
const row {x + (t *
ldx) + i};
354 for (; i + 2 <= count; i += 2) {
356 for (std::size_t t {0}; t < terms; ++t) {
361 for (; i < count; ++i) {
363 for (std::size_t t {0}; t < terms; ++t) {
364 sum = std::fma(-f[t], x[(t *
ldx) + i],
sum);
380 std::size_t count)
noexcept
382 static_assert(
product_rows == 8,
"eight rows, sixteen named registers");
384 for (; i + 4 <= count; i += 4) {
401 for (std::size_t t {0}; t < terms; ++t) {
446 for (; i < count; ++i) {
447 double s0 {
y[(0 *
ldy) + i]};
448 double s1 {
y[(1 *
ldy) + i]};
449 double s2 {
y[(2 *
ldy) + i]};
450 double s3 {
y[(3 *
ldy) + i]};
451 double s4 {
y[(4 *
ldy) + i]};
452 double s5 {
y[(5 *
ldy) + i]};
453 double s6 {
y[(6 *
ldy) + i]};
454 double s7 {
y[(7 *
ldy) + i]};
455 for (std::size_t t {0}; t < terms; ++t) {
456 const double v {x[(t *
ldx) + i]};
457 s0 = std::fma(-f[(0 *
ldf) + t], v,
s0);
458 s1 = std::fma(-f[(1 *
ldf) + t], v, s1);
459 s2 = std::fma(-f[(2 *
ldf) + t], v, s2);
460 s3 = std::fma(-f[(3 *
ldf) + t], v,
s3);
461 s4 = std::fma(-f[(4 *
ldf) + t], v,
s4);
462 s5 = std::fma(-f[(5 *
ldf) + t], v,
s5);
463 s6 = std::fma(-f[(6 *
ldf) + t], v,
s6);
464 s7 = std::fma(-f[(7 *
ldf) + t], v,
s7);
467 y[(1 *
ldy) + i] = s1;
468 y[(2 *
ldy) + i] = s2;
490 const double factor {f};
491#ifdef NUMERICAL_X86_COMPILED
502#ifdef NUMERICAL_NEON_COMPILED
529#ifdef NUMERICAL_X86_COMPILED
540#ifdef NUMERICAL_NEON_COMPILED
560#ifdef NUMERICAL_X86_COMPILED
571#ifdef NUMERICAL_NEON_COMPILED
587 const std::complex<double>* f,
588 const std::complex<double>* x,
593#ifdef NUMERICAL_X86_COMPILED
597 auto*
const into {
reinterpret_cast<double*
>(
y)};
598 const auto*
const scales {
reinterpret_cast<const double*
>(f)};
599 const auto*
const rows {
reinterpret_cast<const double*
>(x)};
615 const std::complex<double>& f,
616 const std::complex<double>* x,
619 const std::complex<double> factor {f};
620#ifdef NUMERICAL_X86_COMPILED
632 const std::complex<double>* x,
633 const std::complex<double>& f,
636 const std::complex<double> factor {f};
637#ifdef NUMERICAL_X86_COMPILED
653 const double factor {f};
654#ifdef NUMERICAL_X86_COMPILED
665#ifdef NUMERICAL_NEON_COMPILED
677 const std::complex<float>& f,
678 const std::complex<float>* x,
681 const std::complex<float> factor {f};
682#ifdef NUMERICAL_X86_COMPILED
696 const std::complex<double>& f,
697 const std::complex<double>* x,
700 const std::complex<double> factor {f};
701#ifdef NUMERICAL_X86_COMPILED
719#ifdef NUMERICAL_X86_COMPILED
737#ifdef NUMERICAL_X86_COMPILED
752 template <split_complex C>
757#ifdef NUMERICAL_X86_COMPILED
767 const std::complex<double>*
y,
775 const std::complex<float>*
y,
782 template <split_complex C>
790#ifdef NUMERICAL_X86_COMPILED
796 for (std::size_t r {0}; r < rows; ++r) {
802 inline void row_dots(
const std::complex<double>* m,
805 const std::complex<double>* x,
807 std::complex<double>* out)
816 const std::complex<float>* x,
818 std::complex<float>* out)
832#ifdef NUMERICAL_X86_COMPILED
855#ifdef NUMERICAL_X86_COMPILED
874 template <
typename T>
875 inline constexpr std::size_t
gemm_nr {
sizeof(
T) >= 16 ? std::size_t {6} : std::size_t {96} /
sizeof(
T)};
896 template <
typename T>
906#ifdef NUMERICAL_X86_COMPILED
917 template <
typename T>
928 template <
typename T>
936 for (std::size_t p {0}; p <
kc; ++p) {
937 for (std::size_t i {0}; i <
gemm_mr; ++i) {
939 for (std::size_t j {0}; j < nr; ++j) {
940 tile[(i * nr) + j] +=
ai *
bp[(p *
ldb) + j];
953 template <
typename T>
954 requires std::is_same_v<T, float> || std::is_same_v<T, double> || std::is_same_v<T, std::uint64_t>
961 using lane =
typename native_vector<T>::type;
967 std::array<lane, gemm_mr * lanes>
acc {};
968# pragma GCC unroll 32
969 for (std::size_t q {0}; q <
acc.size(); ++q) {
976 for (std::size_t p {0}; p <
kc; ++p) {
977 std::array<lane, lanes> b {};
978 std::memcpy(b.data(),
bp + (p *
ldb),
sizeof(b));
980 for (std::size_t i {0}; i <
gemm_mr; ++i) {
983 for (std::size_t j {0}; j < lanes; ++j) {
984 acc[(i * lanes) + j] +=
ai * b[j];
988# pragma GCC unroll 32
989 for (std::size_t q {0}; q <
acc.size(); ++q) {
997#ifdef NUMERICAL_NEON_COMPILED
1011 static_assert(
gemm_mr == 4 &&
gemm_nr<double> == 12,
"two rows of A a vector, twelve columns six vectors");
1012 constexpr std::size_t lanes {6};
1013 std::array<float64x2_t, 4 * lanes>
acc {};
1015 for (std::size_t q {0}; q <
acc.size(); ++q) {
1018 for (std::size_t p {0}; p <
kc; ++p) {
1022 for (std::size_t j {0}; j < lanes; ++j) {
1031 for (std::size_t q {0}; q <
acc.size(); ++q) {
1044 static_assert(
gemm_mr == 4 &&
gemm_nr<float> == 24,
"four rows of A a vector, 24 columns six vectors");
1045 constexpr std::size_t lanes {6};
1046 std::array<float32x4_t, 4 * lanes>
acc {};
1048 for (std::size_t q {0}; q <
acc.size(); ++q) {
1051 for (std::size_t p {0}; p <
kc; ++p) {
1054 for (std::size_t j {0}; j < lanes; ++j) {
1063 for (std::size_t q {0}; q <
acc.size(); ++q) {
1074 template <std::size_t
Mr,
1082 for (std::size_t ir {0}; ir < rows; ir +=
Mr) {
1083 const std::size_t mr {std::min(
Mr, rows - ir)};
1086 for (std::size_t p {0}; p <
kc; ++p) {
1087 for (std::size_t i {0}; i <
Mr; ++i) {
1088 sliver[(p *
Mr) + i] = a[((ir + i) * lda) + p];
1093 for (std::size_t p {0}; p <
kc; ++p) {
1094 for (std::size_t i {0}; i <
Mr; ++i) {
1095 sliver[(p *
Mr) + i] = i < mr ? a[((ir + i) * lda) + p] :
T {};
1105 template <std::size_t
Nr,
1115 for (std::size_t
jr {0};
jr < cols;
jr +=
Nr) {
1116 const std::size_t width {std::min(
Nr, cols -
jr)};
1118 for (std::size_t p {0}; p <
kc; ++p) {
1120 std::copy_n(b + (p *
ldb) +
jr, width,
row);
1130 template <std::size_t
Nr,
1138 static_assert(
Nr % 2 == 0,
"a sliver's columns go by pairs");
1139 for (std::size_t
jr {0};
jr < cols;
jr +=
Nr) {
1140 const std::size_t width {std::min(
Nr, cols -
jr)};
1145 for (std::size_t j {0}; j <
Nr; j += 2) {
1146 const T*
const first {
bt + ((
jr + j) *
ldbt)};
1147 const T*
const second {first +
ldbt};
1149 if constexpr (std::same_as<T, double>) {
1150#if defined(NUMERICAL_NEON_COMPILED)
1151 for (; p + 2 <=
kc; p += 2) {
1157#elif defined(NUMERICAL_X86_COMPILED)
1158 for (; p + 2 <=
kc; p += 2) {
1166 for (; p <
kc; ++p) {
1168 sliver[(p *
Nr) + j + 1] = second[p];
1173 for (std::size_t j {0}; j <
Nr; ++j) {
1175 for (std::size_t p {0}; p <
kc; ++p) {
1183 template <
typename T>
1186 std::size_t&
ld_out)
noexcept
1194 template <
typename T>
1207 transpose_walk(0, rows, 0, cols,
transpose_piece(ld *
sizeof(
T), cols *
sizeof(
T)), [out, x, ld, cols](std::size_t i, std::size_t j) {
1208 out[(i * cols) + j] = x[(j * ld) + i];
1218 template <std::size_t
Nr,
1232 template <
typename Kernel>
1235 return Kernel::reads_b_in_place(
ldb);
1239 template <
typename T>
1251 template <
typename T>
1255 std::size_t j)
noexcept
1257 return b[(p *
ldb) + j];
1266 template <std::size_t
Nr,
1280 template <
typename Kernel>
1288 template <
typename T>
1300 template <
typename T>
1304 std::size_t j)
noexcept
1306 return bt[(j *
ldbt) + p];
1315 template <std::size_t
Mr,
1323 for (std::size_t ir {0}; ir < rows; ir +=
Mr) {
1324 const std::size_t mr {std::min(
Mr, rows - ir)};
1326 for (std::size_t p {0}; p <
kc; ++p) {
1328 for (std::size_t i {0}; i <
Mr; ++i) {
1339 template <
typename Kernel,
1349 Kernel::pack_a(a + (
i0 * lda) +
p0, lda, rows,
kc, out);
1352#ifdef NUMERICAL_X86_COMPILED
1355 template <typename T>
1359 std::size_t p)
noexcept
1361 return a + (i * lda) + p;
1365 [[
nodiscard]]
static constexpr std::size_t
step(std::size_t )
noexcept
1371 [[
nodiscard]]
static constexpr std::size_t
across(std::size_t lda)
noexcept
1379 template <
typename T>
1391 template <
typename T>
1395 std::size_t p)
noexcept
1397 return a[(i * lda) + p];
1406 template <
typename Kernel,
1419#ifdef NUMERICAL_X86_COMPILED
1421 template <typename T>
1425 std::size_t p)
noexcept
1427 return at + (p *
ldat) + i;
1437 [[
nodiscard]]
static constexpr std::size_t
across(std::size_t )
noexcept
1445 template <
typename T>
1457 template <
typename T>
1461 std::size_t p)
noexcept
1471 template <
typename T>
1519 for (std::size_t i {0}; i < rows; ++i) {
1520 std::copy_n(c + (i * ldc), cols,
accumulators.data() + (i *
nr));
1523 for (std::size_t i {0}; i < rows; ++i) {
1524 std::copy_n(
accumulators.data() + (i *
nr), cols, c + (i * ldc));
1529#ifdef NUMERICAL_NEON_COMPILED
1531 template <
typename T>
1544 std::size_t cols)
noexcept
1548 for (std::size_t i {0}; i < rows; ++i) {
1549 std::copy_n(c + (i * ldc), cols,
accumulators.data() + (i * nr));
1552 for (std::size_t i {0}; i < rows; ++i) {
1553 std::copy_n(
accumulators.data() + (i * nr), cols, c + (i * ldc));
1559#ifdef NUMERICAL_AMX_COMPILED
1564 static constexpr std::size_t mr {
amx_mr};
1565 static constexpr std::size_t nr {
amx_nr};
1568 static constexpr std::size_t depth {512};
1569 static constexpr std::size_t block_rows {
gemm_mc};
1572 static constexpr std::size_t b_in_place_rows {block_rows};
1578 static constexpr std::size_t serial_below {std::size_t {1} << 24};
1582 static constexpr bool reads_b_in_place(std::size_t
ldb)
noexcept
1591 static void pack_a(
const double* a,
1595 double* out)
noexcept
1601 static void tile(std::size_t
kc,
1608 std::size_t cols)
noexcept
1619 static constexpr std::size_t depth {512};
1620 static constexpr std::size_t block_rows {
gemm_mc};
1621 static constexpr std::size_t b_in_place_rows {block_rows};
1623 static constexpr std::size_t serial_below {std::size_t {1} << 24};
1626 static constexpr bool reads_b_in_place(std::size_t
ldb)
noexcept
1634 static void pack_a(
const float* a,
1638 float* out)
noexcept
1643 static void tile(std::size_t
kc,
1650 std::size_t cols)
noexcept
1662#ifdef NUMERICAL_X86_COMPILED
1680#if defined(NUMERICAL_X86_COMPILED) || !defined(__clang__)
1705#ifdef NUMERICAL_X86_COMPILED
1716 template <
typename T>
1721#if defined(__GNUC__) && !defined(__clang__)
1725 return sum + (a * b);
1731 template <
typename R>
1734 std::complex<R> b)
noexcept
1762#if defined(NUMERICAL_X86_COMPILED) || !defined(__clang__)
1786 template <term_rounding Mode>
1790 template <
typename T>
1800 template <
typename T>
1806 return std::fma(a, b,
sum);
1819#ifdef NUMERICAL_X86_COMPILED
1822 template <std::
size_t R, std::
size_t N,
typename T>
1832 std::array<T, R * N>
sums {};
1835 for (std::size_t e {0}; e < R *
N; ++e) {
1836 sums[e] = c[((e /
N) * ldc) + (e %
N)];
1839 for (std::size_t p {0}; p < k; ++p) {
1841 for (std::size_t r {0}; r < R; ++r) {
1842 const T ap {a[(r * lda) + p]};
1844 for (std::size_t j {0}; j <
N; ++j) {
1845 sums[(r *
N) + j] = std::fma(
ap, b[(p *
ldb) + j],
sums[(r *
N) + j]);
1850 for (std::size_t e {0}; e < R *
N; ++e) {
1851 c[((e /
N) * ldc) + (e %
N)] =
sums[e];
1868 constexpr std::size_t width {4};
1871 if (k == 0 &&
kept) {
1875 for (; j + (4 * width) <= n; j += 4 * width) {
1880 for (std::size_t p {0}; p < k; ++p) {
1882 const double *
row {b + (p *
ldb) + j};
1893 for (; j + width <= n; j += width) {
1895 for (std::size_t p {0}; p < k; ++p) {
1900 for (; j < n; ++j) {
1901 double sum {
kept ? c[j] :
double {}};
1902 for (std::size_t p {0}; p < k; ++p) {
1903 sum = std::fma(a[p], b[(p *
ldb) + j],
sum);
1919 constexpr std::size_t width {8};
1922 if (k == 0 &&
kept) {
1926 for (; j + (4 * width) <= n; j += 4 * width) {
1931 for (std::size_t p {0}; p < k; ++p) {
1933 const float *
row {b + (p *
ldb) + j};
1944 for (; j + width <= n; j += width) {
1946 for (std::size_t p {0}; p < k; ++p) {
1951 for (; j < n; ++j) {
1952 float sum {
kept ? c[j] :
float {}};
1953 for (std::size_t p {0}; p < k; ++p) {
1954 sum = std::fma(a[p], b[(p *
ldb) + j],
sum);
1963#ifdef NUMERICAL_X86_COMPILED
1971 template <std::
size_t R, std::
size_t V>
1973 const std::complex<
double>* a,
1975 const std::complex<
double>* b,
1977 std::complex<
double>* c,
1981 static_assert(
V == 1 ||
V == 2,
"two or four complex columns, two to a vector");
1982 std::array<ymm_register, R * V>
sums {};
1984 auto*
const cs {
reinterpret_cast<double*
>(c)};
1985 const auto*
const as {
reinterpret_cast<const double*
>(a)};
1986 const auto*
const bs {
reinterpret_cast<const double*
>(b)};
1988 for (std::size_t r {0}; r < R; ++r) {
1990 for (std::size_t v {0}; v <
V; ++v) {
1994 for (std::size_t p {0}; p < k; ++p) {
1995 const double*
const row {
bs + (2 * p *
ldb)};
1996 std::array<ymm_register, V>
bw {};
1997 std::array<ymm_register, V>
bx {};
1999 for (std::size_t v {0}; v <
V; ++v) {
2004 for (std::size_t r {0}; r < R; ++r) {
2005 const double*
const ap {
as + (2 * ((r * lda) + p))};
2009 for (std::size_t v {0}; v <
V; ++v) {
2019 for (std::size_t r {0}; r < R; ++r) {
2021 for (std::size_t v {0}; v <
V; ++v) {
2031 template <std::
size_t R>
2033 const std::complex<
double>* a,
2035 const std::complex<
double>* b,
2037 std::complex<
double>* c,
2041 std::array<xmm_register, R>
sums {};
2043 auto*
const cs {
reinterpret_cast<double*
>(c)};
2044 const auto*
const as {
reinterpret_cast<const double*
>(a)};
2045 const auto*
const bs {
reinterpret_cast<const double*
>(b)};
2047 for (std::size_t r {0}; r < R; ++r) {
2050 for (std::size_t p {0}; p < k; ++p) {
2054 for (std::size_t r {0}; r < R; ++r) {
2055 const double*
const ap {
as + (2 * ((r * lda) + p))};
2064 for (std::size_t r {0}; r < R; ++r) {
2076 template <std::
size_t R, std::
size_t N>
2078 const std::complex<double>* a,
2080 const std::complex<double>* b,
2082 std::complex<double>* c,
2086 if constexpr (
N == 1) {
2089 else if constexpr (
N == 2 ||
N == 4) {
2093 constexpr std::size_t w {
N > 4 ? 4 : 2};
2094 std::array<std::complex<double>, R * w> first {};
2096 for (std::size_t e {0}; e < R * w; ++e) {
2097 first[e] = c[((e / w) * ldc) + (e % w)];
2104 for (std::size_t e {0}; e < R * w; ++e) {
2105 c[((e / w) * ldc) + (e % w)] = first[e];
2118 template <std::
size_t R, std::
size_t N, term_rounding Mode,
typename T>
2128#ifdef NUMERICAL_X86_COMPILED
2135#ifdef NUMERICAL_X86_COMPILED
2143 std::array<T, R * N>
sums {};
2146 for (std::size_t e {0}; e < R *
N; ++e) {
2147 sums[e] = c[((e /
N) * ldc) + (e %
N)];
2151 for (std::size_t p {0}; p < k; ++p) {
2153 for (std::size_t r {0}; r < R; ++r) {
2154 const T ap {a[(r * lda) + p]};
2156 for (std::size_t j {0}; j <
N; ++j) {
2162 for (std::size_t e {0}; e < R *
N; ++e) {
2163 c[((e /
N) * ldc) + (e %
N)] =
sums[e];
2170 template <std::
size_t R, term_rounding Mode,
typename T>
2181 static_assert(
narrow_columns == 8,
"the switch below names each narrower last block");
2199 template <
typename T>
2205 std::size_t
to)
noexcept
2208 std::fill(c + (first * ldc), c + (
last * ldc),
T {});
2211 for (std::size_t i {first}; i <
last; ++i) {
2212 std::fill(c + (i * ldc) +
from, c + (i * ldc) +
to,
T {});
2218 template <
typename T>
2243 template <term_rounding Mode,
typename T>
2262 for (std::size_t i {first}; i < m; ++i) {
2269#ifdef NUMERICAL_X86_COMPILED
2279 for (std::size_t p {0}; p < k; ++p) {
2280 const T aip {a[(i * lda) + p]};
2281 for (std::size_t j {0}; j < n; ++j) {
2294 template <std::
floating_po
int T>
2297#ifdef NUMERICAL_X86_COMPILED
2298 if constexpr (std::is_same_v<T, double> || std::is_same_v<T, float>) {
2323#ifdef NUMERICAL_X86_COMPILED
2335 template <
typename T>
2347#ifdef NUMERICAL_X86_COMPILED
2348 if constexpr (std::is_same_v<T, double> || std::is_same_v<T, float>) {
2356 if constexpr (std::is_floating_point_v<T>) {
2370 template <
typename T>
2373 const std::size_t
miss {
static_cast<std::size_t
>(std::bit_cast<std::uintptr_t>(at) %
packed_line)};
2387 template <
typename T>
2393 static constexpr std::size_t
kept_limit {(std::size_t {32} << 20U) /
sizeof(
T)};
2399 a_ {aligned(storage_.data())},
2400 b_ {aligned(storage_.data() +
a_count + padding)}
2410 kept() = storage_.size() <= kept_limit ? std::move(storage_) :
buffer<T> {};
2458 template <
typename Kernel>
2461 std::size_t k)
noexcept
2463 return m * n * k >= Kernel::serial_below;
2469 template <
typename Kernel>
2471 Kernel::prefetch_rows(c, ldc);
2475 template <
typename Kernel,
2478 std::size_t )
noexcept
2483 template <
typename Kernel,
2487 std::size_t ldc)
noexcept
2489 Kernel::prefetch_rows(c, ldc);
2494 template <
typename Kernel,
2497 Kernel::tile(z, x, x, z,
y, z, z, z,
true);
2501 template <
typename Kernel,
2504 Kernel::tile_a_in_place(z, x, z, z, x, z,
y, z, z, z);
2509 template <
typename Kernel>
2513 template <
typename Kernel,
2517 std::size_t )
noexcept
2524 template <
typename Kernel,
2528 std::size_t n)
noexcept
2530 return a_packed ==
nullptr && n > Kernel::nr;
2534 template <
typename Kernel,
2543 template <
typename Kernel,
2552 template <
typename Kernel>
2555 return Kernel::block_rows;
2559 template <
typename Kernel>
2560 requires requires(std::size_t
kc)
2562 Kernel::task_rows_for(
kc);
2567 return Kernel::task_rows_for(
kc);
2579 template <
typename Kernel,
2581 typename B = b_rows,
2582 typename A = a_rows>
2595 constexpr std::size_t mr {Kernel::mr};
2596 constexpr std::size_t nr {Kernel::nr};
2597 constexpr std::size_t
nc_max {nr * 128};
2601#ifdef NUMERICAL_X86_COMPILED
2604 const std::size_t
a_rows {((m + mr) - 1) / mr * mr};
2605 const std::size_t
b_cols {((std::min(
nc_max, n) + nr) - 1) / nr * nr};
2607 const std::size_t
a_blocks {
a_own ? ((m + Kernel::block_rows) - 1) / Kernel::block_rows : 0};
2619 const std::size_t
nc {std::min(
nc_max, n -
jc)};
2620 const std::size_t
slivers {((
nc + nr) - 1) / nr};
2621 for (std::size_t
pc {0};
pc < k;
pc += Kernel::depth) {
2622 const std::size_t
kc {std::min(Kernel::depth, k -
pc)};
2626 const std::size_t
tiles {((m + mr) - 1) / mr};
2636#ifdef NUMERICAL_X86_COMPILED
2650 const std::size_t
whole {in_place ? width - (width % nr) : 0};
2654 else if (
whole < width &&
pc +
kc == k) {
2659 const std::size_t
ic {(
piece -
groups) * Kernel::block_rows};
2675 for (std::size_t ir {
ic}; ir <
ic_end; ir += mr) {
2676 for (std::size_t s {first}; s <
last; ++s) {
2677 const std::size_t
jr {s * nr};
2678 const bool direct {in_place &&
jr + nr <=
nc};
2685 T*
const tile {c + (ir * ldc) +
jc +
jr};
2689 Kernel::tile(1,
a_tiles + (ir *
kc) + (
ahead * mr), packed.b() + (
jr *
kc), nr, tile, ldc,
2690 std::min(mr, m - ir),
nc -
jr);
2697#ifdef NUMERICAL_X86_COMPILED
2700 Kernel::tile_a_in_place(
kc, A::element(a, lda, ir,
pc), A::step(lda), A::across(lda),
b_tile,
b_ld,
2701 c + (ir * ldc) +
jc +
jr, ldc, std::min(mr, m - ir), std::min(nr,
nc -
jr),
fresh);
2712 std::min(nr,
nc -
jr));
2728 template <
typename Kernel,
2730 typename B = b_rows,
2731 typename A = a_rows>
2742 constexpr std::size_t mr {Kernel::mr};
2743 constexpr std::size_t nr {Kernel::nr};
2744 constexpr std::size_t
nc_max {nr * 128};
2745 const std::size_t
a_rows {((n + mr) - 1) / mr * mr};
2746 const std::size_t
b_cols {((std::min(
nc_max, n) + nr) - 1) / nr * nr};
2748 const std::size_t
a_blocks {((n + Kernel::block_rows) - 1) / Kernel::block_rows};
2752 const auto meets = [lower](std::size_t
r0, std::size_t
r1, std::size_t
j0, std::size_t
j1) {
return lower ?
j0 <
r1 :
r0 <
j1; };
2753 const auto inside = [lower](std::size_t
r0, std::size_t
r1, std::size_t
j0, std::size_t
j1) {
return lower ?
j1 <=
r0 + 1 :
r1 <=
j0 + 1; };
2755 const std::size_t
nc {std::min(
nc_max, n -
jc)};
2756 const std::size_t
slivers {((
nc + nr) - 1) / nr};
2757 for (std::size_t
pc {0};
pc < k;
pc += Kernel::depth) {
2758 const std::size_t
kc {std::min(Kernel::depth, k -
pc)};
2764 const std::size_t
whole {in_place ? width - (width % nr) : 0};
2765 if (
whole < width) {
2770 const std::size_t
ic {(
piece -
groups) * Kernel::block_rows};
2771 const std::size_t rows {std::min(Kernel::block_rows, n -
ic)};
2776 const std::size_t
tiles {((n + mr) - 1) / mr};
2786 for (std::size_t ir {
ic}; ir <
ic_end; ir += mr) {
2787 const std::size_t rows {std::min(mr, n - ir)};
2788 for (std::size_t s {first}; s <
last; ++s) {
2789 const std::size_t
jr {s * nr};
2790 const std::size_t
j0 {
jc +
jr};
2791 const std::size_t cols {std::min(nr,
nc -
jr)};
2792 if (!
meets(ir, ir + rows,
j0,
j0 + cols)) {
2795 const bool direct {in_place &&
jr + nr <=
nc};
2798 T*
const block {c + (ir * ldc) +
j0};
2800 Kernel::tile(
kc, packed.a() + (ir *
kc),
bp,
ldbp, block, ldc, rows, cols);
2803 std::array<T, mr * nr>
copy {};
2804 for (std::size_t i {0}; i < rows; ++i) {
2805 std::copy_n(block + (i * ldc), cols,
copy.data() + (i * nr));
2807 Kernel::tile(
kc, packed.a() + (ir *
kc),
bp,
ldbp,
copy.data(), nr, rows, cols);
2808 for (std::size_t i {0}; i < rows; ++i) {
2809 for (std::size_t j {0}; j < cols; ++j) {
2811 block[(i * ldc) + j] =
copy[(i * nr) + j];
2822#ifdef NUMERICAL_X86_COMPILED
2826 static constexpr std::size_t mr {
avx512_mr};
2827 static constexpr std::size_t nr {
avx512_nr};
2828 static constexpr std::size_t depth {
gemm_kc};
2829 static constexpr std::size_t block_rows {96};
2830 static constexpr std::size_t b_in_place_rows {mr};
2834 static constexpr std::size_t serial_below {1};
2842 static constexpr bool reads_b_in_place(std::size_t
ldb)
noexcept
2848 static void pack_a(
const double* a,
2858 static void tile(std::size_t
kc,
2875 static constexpr std::size_t mr {
avx2_mr};
2876 static constexpr std::size_t nr {
avx2_nr};
2877 static constexpr std::size_t depth {
gemm_kc};
2878 static constexpr std::size_t block_rows {72};
2879 static constexpr std::size_t b_in_place_rows {mr};
2888 return kc <= 128 ? 36 : block_rows;
2897 std::size_t ldc)
noexcept
2899 for (std::size_t r {0}; r < mr; ++r) {
2907 static constexpr std::size_t serial_below {1};
2915 static constexpr bool reads_b_in_place(std::size_t
ldb)
noexcept
2921 static void pack_a(
const double* a,
2931 static void tile(std::size_t
kc,
2947 static constexpr std::size_t mr {
avx512_mr};
2949 static constexpr std::size_t depth {
gemm_kc};
2950 static constexpr std::size_t block_rows {96};
2951 static constexpr std::size_t b_in_place_rows {mr};
2953 static constexpr std::size_t serial_below {1};
2958 static constexpr bool reads_b_in_place(std::size_t
ldb)
noexcept
2963 static void pack_a(
const float* a,
2972 static void tile(std::size_t
kc,
2989 static constexpr std::size_t mr {
avx2_mr};
2991 static constexpr std::size_t depth {
gemm_kc};
2992 static constexpr std::size_t block_rows {72};
2993 static constexpr std::size_t b_in_place_rows {mr};
2995 static constexpr std::size_t serial_below {1};
2999 return kc <= 128 ? 36 : block_rows;
3005 static constexpr bool reads_b_in_place(std::size_t
ldb)
noexcept
3010 static void pack_a(
const float* a,
3019 static void tile(std::size_t
kc,
3038 static constexpr std::size_t depth {
gemm_kc / 2};
3039 static constexpr std::size_t block_rows {96};
3040 static constexpr std::size_t b_in_place_rows {mr};
3045 static constexpr std::size_t serial_below {std::size_t {343000}};
3053 static constexpr bool reads_b_in_place(std::size_t
ldb)
noexcept
3059 static void pack_a(
const std::complex<double>* a,
3063 std::complex<double>* out)
3069 static void tile(std::size_t
kc,
3070 const std::complex<double>*
ap,
3071 const std::complex<double>*
bp,
3073 std::complex<double>* c,
3085 const std::complex<double>* a,
3088 const std::complex<double>*
bp,
3090 std::complex<double> * c,
3096 avx512_tile_c64_a_in_place(
kc, a,
a_step,
a_row,
bp,
ldb, c, ldc, rows, cols,
fresh);
3105 static constexpr std::size_t depth {
gemm_kc / 2};
3106 static constexpr std::size_t block_rows {72};
3107 static constexpr std::size_t b_in_place_rows {mr};
3111 static constexpr std::size_t serial_below {std::size_t {140608}};
3119 static constexpr bool reads_b_in_place(std::size_t
ldb)
noexcept
3125 static void pack_a(
const std::complex<double>* a,
3129 std::complex<double>* out)
3135 static void tile(std::size_t
kc,
3136 const std::complex<double>*
ap,
3137 const std::complex<double>*
bp,
3139 std::complex<double>* c,
3151 const std::complex<double>* a,
3154 const std::complex<double>*
bp,
3156 std::complex<double> * c,
3162 avx2_tile_c64_a_in_place(
kc, a,
a_step,
a_row,
bp,
ldb, c, ldc, rows, cols,
fresh);
3170 static constexpr std::size_t depth {
gemm_kc};
3171 static constexpr std::size_t block_rows {96};
3172 static constexpr std::size_t b_in_place_rows {mr};
3176 static constexpr std::size_t serial_below {std::size_t {592704}};
3181 static constexpr bool reads_b_in_place(std::size_t
ldb)
noexcept
3186 static void pack_a(
const std::complex<float>* a,
3190 std::complex<float>* out)
3195 static void tile(std::size_t
kc,
3196 const std::complex<float>*
ap,
3197 const std::complex<float>*
bp,
3199 std::complex<float>* c,
3211 const std::complex<float>* a,
3214 const std::complex<float>*
bp,
3216 std::complex<float> * c,
3222 avx512_tile_c32_a_in_place(
kc, a,
a_step,
a_row,
bp,
ldb, c, ldc, rows, cols,
fresh);
3231 static constexpr std::size_t depth {
gemm_kc};
3232 static constexpr std::size_t block_rows {72};
3233 static constexpr std::size_t b_in_place_rows {mr};
3237 static constexpr std::size_t serial_below {std::size_t {140608}};
3242 static constexpr bool reads_b_in_place(std::size_t
ldb)
noexcept
3247 static void pack_a(
const std::complex<float>* a,
3251 std::complex<float>* out)
3256 static void tile(std::size_t
kc,
3257 const std::complex<float>*
ap,
3258 const std::complex<float>*
bp,
3260 std::complex<float>* c,
3272 const std::complex<float>* a,
3275 const std::complex<float>*
bp,
3277 std::complex<float> * c,
3283 avx2_tile_c32_a_in_place(
kc, a,
a_step,
a_row,
bp,
ldb, c, ldc, rows, cols,
fresh);
3290 template <
typename T>
3309 template <
typename B,
3310 typename A = a_rows>
3323#ifdef NUMERICAL_AMX_COMPILED
3325 gemm_packed<amx_tile_kernel, double, B, A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
3329#ifdef NUMERICAL_NEON_COMPILED
3331 gemm_packed<neon_tile_kernel<double>,
double,
B,
A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
3335#ifdef NUMERICAL_X86_COMPILED
3337 gemm_packed<avx512_tile_kernel, double, B, A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
3341 gemm_packed<avx2_tile_kernel, double, B, A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
3354 template <
typename A,
3367#ifdef NUMERICAL_AMX_COMPILED
3369 gemm_packed_triangle<amx_tile_kernel, double, B, A>(n, k, a, lda, b,
ldb, c, ldc, lower);
3373#ifdef NUMERICAL_NEON_COMPILED
3375 gemm_packed_triangle<neon_tile_kernel<double>,
double,
B,
A>(n, k, a, lda, b,
ldb, c, ldc, lower);
3379#ifdef NUMERICAL_X86_COMPILED
3381 gemm_packed_triangle<avx512_tile_kernel, double, B, A>(n, k, a, lda, b,
ldb, c, ldc, lower);
3385 gemm_packed_triangle<avx2_tile_kernel, double, B, A>(n, k, a, lda, b,
ldb, c, ldc, lower);
3390 gemm_packed_triangle<vector_tile_kernel<double>,
double,
B,
A>(n, k, a, lda, b,
ldb, c, ldc, lower);
3397 template <
typename A,
3409#ifdef NUMERICAL_X86_COMPILED
3412 gemm_packed_triangle<avx512_tile_kernel_f32, float, B, A>(n, k, a, lda, b,
ldb, c, ldc, lower);
3416 gemm_packed_triangle<avx2_tile_kernel_f32, float, B, A>(n, k, a, lda, b,
ldb, c, ldc, lower);
3420 gemm_packed_triangle<vector_tile_kernel<float>,
float,
B,
A>(n, k, a, lda, b,
ldb, c, ldc, lower);
3441 template <
typename B,
3442 typename A = a_rows>
3454#ifdef NUMERICAL_NEON_COMPILED
3456# ifdef NUMERICAL_AMX_COMPILED
3458 gemm_packed<amx_tile_kernel_f32, float, B, A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
3463 gemm_packed<neon_tile_kernel<float>,
float,
B,
A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
3467#ifdef NUMERICAL_X86_COMPILED
3470 gemm_packed<avx512_tile_kernel_f32, float, B, A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
3474 gemm_packed<avx2_tile_kernel_f32, float, B, A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
3497 template <
typename T>
3498 void gemm(std::size_t m,
3515 template <std::
size_t W
idth, term_rounding Mode,
typename T>
3527 static_assert(
narrow_group == 4,
"the switches below name each count of rows up to narrow_group");
3529 const std::size_t blocks {n /
Width};
3530 const auto block_of {[&](std::size_t first, std::size_t j) {
3531 const T*
const rows {a + (first * lda)};
3532 T*
const out {c + (first * ldc) + j};
3543 if (n %
Width != 0) {
3544 const std::size_t j {blocks *
Width};
3545 for (std::size_t first {0}; first < m; first +=
narrow_group) {
3546 const T*
const rows {a + (first * lda)};
3547 T*
const out {c + (first * ldc) + j};
3558#ifdef NUMERICAL_X86_COMPILED
3563 template <std::
size_t R>
3564 inline constexpr std::size_t
few_rows_columns {R == 1 ? 32 : (R == 2 ? 16 : (R == 3 ? 12 : 8))};
3572 template <std::
size_t R,
typename T>
3583 few_rows<few_rows_columns<R>,
term_rounding::x86_fused>(R, n, k, a, lda, b,
ldb, c, ldc, start);
3595 template <std::
size_t R>
3605 template <
typename Isa,
3618 typename Z::value_type*
sums)
noexcept
3620 using P =
typename Z::value_type;
3623 constexpr std::size_t width {
sizeof(
vec) /
sizeof(
P)};
3624 const std::size_t parts {2 * w};
3625 const std::size_t
whole {parts - (parts % width)};
3626 const auto*
const bs {
reinterpret_cast<const P*
>(b)};
3627 const auto*
const as {
reinterpret_cast<const P*
>(a)};
3628 auto*
const cs {
reinterpret_cast<P*
>(c)};
3629 const auto real_sums {[&](std::size_t r)
noexcept {
return sums + (2 * r * parts); }};
3630 const auto imaginary_sums {[&](std::size_t r)
noexcept {
return sums + (((2 * r) + 1) * parts); }};
3631 for (std::size_t
pc {0};
pc < k;
pc += depth) {
3632 const std::size_t
kc {std::min(depth, k -
pc)};
3633 std::fill(
sums,
sums + (2 * R * parts),
P {});
3635 for (; p + 2 <=
kc; p += 2) {
3638 for (std::size_t r {0}; r < R; ++r) {
3639 const P*
const ar {
as + (2 * ((r * lda) +
pc + p))};
3646 for (std::size_t j {0}; j <
whole; j += width) {
3652 for (std::size_t j {
whole}; j < parts; ++j) {
3660 for (std::size_t r {0}; r < R; ++r) {
3661 const P*
const ar {
as + (2 * ((r * lda) +
pc + p))};
3666 for (std::size_t j {0}; j <
whole; j += width) {
3671 for (std::size_t j {
whole}; j < parts; ++j) {
3679 for (std::size_t r {0}; r < R; ++r) {
3682 P*
const out {cs + (2 * r * ldc)};
3683 for (std::size_t j {0}; j <
whole; j += width) {
3687 for (std::size_t j {
whole}; j < parts; j += 2) {
3698 template <std::size_t R,
3712 few_rows_complex_chunk<fft_avx2, R>(depth, w, k, a, lda, b,
ldb, c, ldc, start,
sums.data());
3717 template <std::size_t R,
3731 few_rows_complex_chunk<fft_avx512, R>(depth, w, k, a, lda, b,
ldb, c, ldc, start,
sums.data());
3742 template <std::size_t R,
3754 constexpr bool doubles {std::is_same_v<Z, std::complex<double>>};
3756 const std::size_t depth {
doubles ? avx512_complex_tile_kernel::depth : avx512_complex_tile_kernel_f32::depth};
3760 const std::size_t chunks {(n + width - 1) / width};
3761 parallel_for(chunks, 4 * R * width * k, [&](std::size_t t) {
3762 const std::size_t j {t * width};
3763 const std::size_t w {std::min(width, n - j)};
3775#ifdef NUMERICAL_NEON_COMPILED
3785 template <
typename T>
3786 requires std::same_as<T, float>
3793 template <
typename T>
3794 requires std::same_as<T, double>
3801 template <
typename T>
3802 concept few_rows_real = std::same_as<T, float> || std::same_as<T, double>;
3805 template <
typename T>
3815 template <
typename T>
3816 requires std::same_as<T, float>
3819 std::size_t k)
noexcept
3821 return n >= 32 && m * n * k >= (std::size_t {1} << 16U);
3826 template <
typename T>
3827 requires std::same_as<T, double>
3830 std::size_t )
noexcept
3836 template <few_rows_real T>
3853 few_rows<few_rows_neon_columns<T>,
term_rounding::fused>(m, n, k, a, lda, b,
ldb, c, ldc, start);
3858 template <
typename T>
3882#ifndef NUMERICAL_X86_COMPILED
3894 template <split_complex C>
3905 using R =
typename C::value_type;
3909 std::array<R, 2 * split_row_width>
sums {};
3910 for (std::size_t c {0};
kept && c < count; ++c) {
3911 sums[2 * c] =
y[j + c].real();
3916 for (std::size_t k {
pc}; k < end; ++k) {
3917 const R f {a[k].real()};
3919 for (std::size_t c {0}; c < count; ++c) {
3921 sums[(2 * c) + 1] = std::fma(f,
row[(c *
step) + 1],
sums[(2 * c) + 1]);
3924 for (std::size_t k {
pc}; k < end; ++k) {
3925 const R g {a[k].imag()};
3927 for (std::size_t c {0}; c < count; ++c) {
3933 for (std::size_t c {0}; c < count; ++c) {
3934 y[j + c] = {
sums[2 * c],
sums[(2 * c) + 1]};
3941#ifdef NUMERICAL_NEON_COMPILED
4038 constexpr std::size_t lanes {16 /
sizeof(R)};
4039 std::array<vec, Registers>
sums {};
4041# pragma GCC unroll 16
4042 for (std::size_t t {0}; t <
Registers; ++t) {
4046 for (std::size_t k {
pc}; k < end; ++k) {
4049# pragma GCC unroll 16
4050 for (std::size_t t {0}; t <
Registers; ++t) {
4054 for (std::size_t k {
pc}; k < end; ++k) {
4057# pragma GCC unroll 16
4058 for (std::size_t t {0}; t <
Registers; ++t) {
4062# pragma GCC unroll 16
4063 for (std::size_t t {0}; t <
Registers; ++t) {
4071 template <split_complex C>
4081 using R =
typename C::value_type;
4082 constexpr std::size_t lanes {16 /
sizeof(R)};
4083 const R*
const parts {
reinterpret_cast<const R*
>(b + first)};
4084 R*
const out {
reinterpret_cast<R*
>(
y + first)};
4085 const std::size_t count {2 * (
last - first)};
4086 const std::size_t
whole {(count / lanes) * lanes};
4091 for (; p + (16 * lanes) <= count; p += 16 * lanes) {
4094 for (; p + (4 * lanes) <= count; p += 4 * lanes) {
4097 switch ((
whole - p) / lanes) {
4118 template <split_complex Z>
4134 for (std::size_t i {0}; i < m; ++i) {
4135 Z*
const row {c + (i * ldc)};
4166#ifdef NUMERICAL_AMX_COMPILED
4184 template <
typename R>
4196 template <
typename R>
4211 template <split_complex
Z,
4212 typename B = b_rows,
4213 typename A = a_rows>
4223 typename Z::value_type*
ar,
4224 typename Z::value_type*
br,
4225 typename Z::value_type*
cr)
4227 using R =
typename Z::value_type;
4229 R*
const row {
ar + (i * 3 *
kb)};
4230 for (std::size_t p {0}; p <
kb; ++p) {
4231 const Z value {A::at(a, lda, i,
pc + p)};
4232 row[p] = value.real();
4233 row[
kb + p] = -value.imag();
4234 row[(2 *
kb) + p] = value.imag();
4238 R*
const row {
br + (p * 2 *
nb)};
4239 for (std::size_t j {0}; j <
nb; ++j) {
4240 const Z value {B::at(b,
ldb,
pc + p,
jc + j)};
4241 row[j] = value.real();
4242 row[
nb + j] = value.imag();
4256 template <split_complex
Z,
4257 typename B = b_rows,
4258 typename A = a_rows>
4268 typename Z::value_type*
ar,
4269 typename Z::value_type*
br,
4270 typename Z::value_type*
cr)
4272 using R =
typename Z::value_type;
4274 for (std::size_t p {0}; p <
kb; ++p) {
4275 const Z value {A::at(a, lda, i,
pc + p)};
4276 ar[(i * 2 *
kb) + p] = value.real();
4277 ar[(i * 2 *
kb) +
kb + p] = -value.imag();
4281 R*
const top {
br + (p * 2 *
nb)};
4283 for (std::size_t j {0}; j <
nb; ++j) {
4284 const Z value {B::at(b,
ldb,
pc + p,
jc + j)};
4285 top[j] = value.real();
4286 top[
nb + j] = value.imag();
4287 bottom[j] = value.imag();
4302 template <split_complex
Z,
4303 typename B = b_rows,
4304 typename A = a_rows>
4315 using R =
typename Z::value_type;
4323 R*
const ar {
kept.a.data()};
4324 R*
const br {
kept.b.data()};
4325 R*
const cr {
kept.c.data()};
4329 for (std::size_t j {0}; j <
nb; ++j) {
4330 cr[(i * 2 *
nb) + j] = c[(i * ldc) +
jc + j].real();
4331 cr[(i * 2 *
nb) +
nb + j] = c[(i * ldc) +
jc + j].imag();
4337 split_parts_block<Z, B, A>(m,
nb,
kb,
pc,
jc, a, lda, b,
ldb,
ar,
br,
cr);
4340 split_expanded_block<Z, B, A>(m,
nb,
kb,
pc,
jc, a, lda, b,
ldb,
ar,
br,
cr);
4344 for (std::size_t j {0}; j <
nb; ++j) {
4345 c[(i * ldc) +
jc + j] =
Z {
cr[(i * 2 *
nb) + j],
cr[(i * 2 *
nb) +
nb + j]};
4374 std::size_t k)
noexcept
4377#ifdef NUMERICAL_NEON_COMPILED
4379# ifdef NUMERICAL_AMX_COMPILED
4383 return k <= 16 || (m <= 64 && k <= 64);
4386#ifdef NUMERICAL_X86_COMPILED
4396 template <
typename T>
4399 std::size_t )
noexcept
4408 std::size_t k)
noexcept
4422 std::size_t k)
noexcept
4429 template <
typename T>
4445#ifdef NUMERICAL_NEON_COMPILED
4451 gemm_packed<vector_tile_kernel<T>,
T>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
4455#ifdef NUMERICAL_X86_COMPILED
4457 template <split_complex
Z,
4467 template <
typename B,
4483 gemm_packed<vector_tile_kernel<T>,
T,
B,
A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
4486#ifdef NUMERICAL_X86_COMPILED
4488 gemm_packed<x86_complex_kernel<T, avx512_complex_tile_kernel, avx512_complex_tile_kernel_f32>,
T,
B,
A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
4492 gemm_packed<x86_complex_kernel<T, avx2_complex_tile_kernel, avx2_complex_tile_kernel_f32>,
T,
B,
A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
4508 const std::size_t blocks {((m + rows) - 1) / rows};
4510 const std::size_t
r0 {block * rows};
4511 gemm_split<T, B, A>(std::min(rows, m -
r0), n, k, &A::at(a, lda,
r0, 0), lda, b,
ldb, c + (
r0 * ldc), ldc);
4516 gemm_split<T, B, A>(m, n, k, a, lda, b,
ldb, c, ldc);
4520 template <split_complex T>
4532 gemm_large_complex<b_rows, a_rows>(m, n, k, a, lda, b,
ldb, c, ldc, start);
4543 if (m == 0 || n == 0 || k == 0) {
4546#if defined(NUMERICAL_X86_COMPILED)
4549#elif defined(NUMERICAL_NEON_COMPILED)
4582 template <
typename T>
4596#ifdef NUMERICAL_X86_COMPILED
4598 static_assert(
narrow_group == 4,
"the switch below names each count of rows up to narrow_group");
4607#ifdef NUMERICAL_NEON_COMPILED
4613 if (m != 0 && n != 0 && k != 0) {
4622#ifdef NUMERICAL_X86_COMPILED
4623 if constexpr (std::is_same_v<T, double> || std::is_same_v<T, float>) {
4626 static_assert(
narrow_group == 4,
"the switch below names each count of rows up to narrow_group");
4648 template <
typename B,
4661 if (!
gemm_double_elsewhere<B, A>(m, n, k, a, lda, b,
ldb, c, ldc, start)) {
4662 gemm_packed<vector_tile_kernel<double>,
double,
B,
A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
4667 template <
typename B,
4680 if (!
gemm_float_elsewhere<B, A>(m, n, k, a, lda, b,
ldb, c, ldc, start)) {
4681 gemm_packed<vector_tile_kernel<float>,
float,
B,
A>(m, n, k, a, lda, b,
ldb, c, ldc,
nullptr, start);
4686 template <
typename B,
4700 gemm_large_complex<B, A>(m, n, k, a, lda, b,
ldb, c, ldc, start);
4713 template <
typename A,
4729#ifdef NUMERICAL_NEON_COMPILED
4736 std::size_t
a_ld {};
4737 std::size_t
b_ld {};
4744 if (m != 0 && n != 0 && k != 0) {
4745 gemm_large_complex<B, A>(m, n, k, a, lda, b,
ldb, c, ldc, start);
4759 std::size_t
a_ld {};
4760 std::size_t
b_ld {};
4766 gemm_large_layouts<B, A>(m, n, k, a, lda, b,
ldb, c, ldc, start);
4770 template <
typename T>
4782 gemm_layouts<a_rows, b_transposed>(m, n, k, a, lda,
bt,
ldbt, c, ldc, start);
4786 template <
typename T>
4798 gemm_layouts<a_transposed, b_rows>(m, n, k, at,
ldat, b,
ldb, c, ldc, start);
4802 template <
typename T>
4814 gemm_layouts<a_transposed, b_transposed>(m, n, k, at,
ldat,
bt,
ldbt, c, ldc, start);
4830 template <
typename T>
4843 template <
typename T>
4847 std::size_t lda)
noexcept
4856 template <
typename T>
4859 std::size_t rows)
noexcept
4870 template <
typename T>
4873 std::size_t )
noexcept
4883 std::size_t k)
noexcept
4889#ifdef NUMERICAL_AMX_COMPILED
4894#ifdef NUMERICAL_NEON_COMPILED
4899#ifdef NUMERICAL_X86_COMPILED
4912 template <
typename Kernel,
4920 const std::size_t
a_rows {((m + Kernel::mr) - 1) / Kernel::mr * Kernel::mr};
4923 const std::size_t blocks {((m + Kernel::block_rows) - 1) / Kernel::block_rows};
4925 const std::size_t
ic {block * Kernel::block_rows};
4926 Kernel::pack_a(a + (
ic * lda), lda, std::min(Kernel::block_rows, m -
ic), k, out + (
ic * k));
4937 template <
typename T>
4958#ifdef NUMERICAL_AMX_COMPILED
4961 left.granule = amx_tile_kernel::mr;
4964#ifdef NUMERICAL_NEON_COMPILED
4967 left.granule = neon_tile_kernel<double>::mr;
4970#ifdef NUMERICAL_X86_COMPILED
4973 left.granule = avx512_tile_kernel::mr;
4977 left.granule = avx2_tile_kernel::mr;
4991 template <
typename T>
5019#ifdef NUMERICAL_AMX_COMPILED
5025#ifdef NUMERICAL_NEON_COMPILED
5031#ifdef NUMERICAL_X86_COMPILED
5045 inline constexpr std::size_t
dot_chunk {std::size_t {1} << 14U};
5064 template <
typename T,
5073 return dot(a, b, count);
5077 const std::size_t first {c *
dot_chunk};
5088 template <
typename T>
5098 template <
typename T>
5107 template <
typename T>
5128#ifdef NUMERICAL_X86_COMPILED
5134#ifdef NUMERICAL_AMX_COMPILED
5137 inline constexpr std::size_t
amx_dot_least {std::size_t {1} << 15U};
5142 inline constexpr std::size_t
amx_dot_most {std::size_t {1} << 18U};
5152 template <
typename T>
5160#ifdef NUMERICAL_AMX_COMPILED
5161 if constexpr (std::is_same_v<T, double>) {
5162 const std::size_t elements {rows * cols};
5180 const std::size_t first {g *
group};
5181 row_dots(a + (first * lda), lda, std::min(
group, rows - first), x, cols,
y + first);
5203 constexpr std::size_t lanes {8};
5205 for (std::size_t j {0}; j < cols; ++j) {
5206 double*
const lane {
sums.data() + ((j % lanes) * rows)};
5207 const double*
const column {a + (j * lda)};
5208 const double factor {x[j]};
5209 for (std::size_t r {0}; r < rows; ++r) {
5213 for (std::size_t r {0}; r < rows; ++r) {
5215 for (std::size_t l {1}; l < lanes; ++l) {
5235 template <
typename T>
5236 requires std::is_same_v<T, double> || std::is_same_v<T, float>
5244#ifdef NUMERICAL_X86_COMPILED
5249 constexpr std::size_t
partials {256 /
sizeof(
T)};
5250 const std::size_t done {cols - (cols %
partials)};
5253 const std::size_t height {std::min(
column_rows, rows - first)};
5255 const T*
const part {a + first};
5260#elif defined(__GNUC__)
5266# ifdef NUMERICAL_AMX_COMPILED
5267 if constexpr (std::is_same_v<T, double>) {
5268 const std::size_t elements {rows * cols};
5284 (
void)a, (
void)lda, (
void)rows, (
void)cols, (
void)x, (
void)
y;
5296 template <split_complex C>
5304#if defined(__GNUC__)
5311 const std::size_t height {std::clamp<std::size_t>((((rows / (2 *
max_threads())) + 15) / 16) * 16, 16,
column_rows)};
5312# ifdef NUMERICAL_X86_COMPILED
5314 using R =
typename C::value_type;
5315 constexpr std::size_t
slots {std::is_same_v<R, double> ? 8 : 16};
5316 const std::size_t done {cols - (cols %
slots)};
5317 const auto*
const parts {
reinterpret_cast<const R*
>(a)};
5318 const auto*
const xs {
reinterpret_cast<const R*
>(x)};
5319 parallel_for((rows + height - 1) / height, height * cols *
weight, [&](std::size_t block) {
5320 const std::size_t first {block * height};
5321 const std::size_t count {std::min(height, rows - first)};
5329 parallel_for((rows + height - 1) / height, height * cols *
weight, [&](std::size_t block) {
5330 const std::size_t first {block * height};
5335 (
void)a, (
void)lda, (
void)rows, (
void)cols, (
void)x, (
void)
y;
5351 detail::gemm_path_cap().store(most, std::memory_order_relaxed);
5352 return detail::best_gemm_path(most);
5358 return detail::gemm_path_current();
The matrix coprocessor of Apple's M-series chips (AMX), for the double matrix product.
numerical::buffer, the storage of the library's vectors, matrices and tensors: a std::vector whose el...
#define NUMERICAL_OUT_OF_LINE
The storage of a product's packed operands, kept by the calling thread from one product to the next.
packing_storage(std::size_t a_count, std::size_t b_count)
Room for a_count packed elements of A and b_count of B.
static buffer< T > grown(buffer< T > storage, std::size_t count)
storage with at least count elements (it only grows, so a reuse writes nothing).
static buffer< T > & kept() noexcept
The storage this thread keeps between products.
T * b() const noexcept
B's packed elements.
static T * aligned(T *at) noexcept
line_aligned.
static constexpr std::size_t padding
Elements enough to reach a line.
packing_storage(packing_storage &&)=delete
static constexpr std::size_t kept_limit
The most elements a thread keeps between products (32 MiB).
T * a() const noexcept
A's packed elements.
packing_storage & operator=(packing_storage &&)=delete
packing_storage(const packing_storage &)=delete
packing_storage & operator=(const packing_storage &)=delete
A complex number of float or double, which gemm_split multiplies through the real kernel and the kern...
#define NUMERICAL_SCALAR_DEPTH
#define NUMERICAL_UNROLL_BLOCK
constexpr std::size_t gemm_packed_threshold
Below this many multiply-adds the plain loop (gemm_rows, by blocks of rows held in registers) runs in...
bool gemm_path_usable(gemm_path path) noexcept
Whether path can run here.
C complex_dot_product(const C *x, const C *y, std::size_t count)
Σ x[i] · y[i] of complex numbers: on the x86-64 units with AVX2 (the AVX-512 path too) their fused ch...
complex_term complex_terms() noexcept
The complex_term of the path in use.
constexpr std::size_t split_columns
The columns of B and C, and the depth, of one real product of gemm_split: its real operands stay a fe...
void gemm_pack_b(const T *b, std::size_t ldb, std::size_t kc, std::size_t cols, T *out)
Copies kc × cols of B (from row-major, leading dimension ldb) into slivers of Nr columns,...
bool on_release(void(*step)()) noexcept
Registers step among the calling thread's thread_releases, while a place is left (a workspace past th...
bool plain_loop_pays(std::size_t m, std::size_t n, std::size_t k) noexcept
Whether a double product of m rows, n columns and depth k, past gemm_rows's threshold,...
constexpr std::size_t gemm_rows_share
The multiply-adds by rows a thread takes at most past gemm_rows_work, the blocks of rows shared out: ...
void parallel_zero(T *out, std::size_t count)
Sets the count elements at out to zero, split across the threads the work pays for: a fresh result's ...
const T * pack_left_for(std::size_t m, std::size_t k, const T *a, std::size_t lda, buffer< T > &storage)
A packed for Kernel into storage, in parallel by its blocks of rows, as gemm_packed packs it.
void gemm_pack_a(const T *a, std::size_t lda, std::size_t rows, std::size_t kc, T *out)
Copies rows × kc of A (from row-major, leading dimension lda) into slivers of Mr rows,...
void gemm_triangle(std::size_t n, std::size_t k, const double *a, std::size_t lda, const double *b, std::size_t ldb, double *c, std::size_t ldc, bool lower)
The triangle of the double product C = A·B (gemm_packed_triangle) on the tile of the path in use,...
void narrow_rows(std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start) noexcept
C += A·B for R whole rows of C of n columns (C = A·B as start says): blocks of narrow_columns,...
void transpose_walk(std::size_t i0, std::size_t i1, std::size_t j0, std::size_t j1, std::size_t piece, const Copy ©)
transpose_pieces of side piece (transpose_piece's 8 or the whole tile), each side its own instance: a...
constexpr bool zeroes_c(bool empty) noexcept
Whether gemm_packed zeroes a C that holds nothing (empty) for Kernel's tiles to add to.
bool split_by_parts(std::size_t m) noexcept
Whether gemm_split runs a product of m rows of A by four real products of each block's parts (split_p...
void gemm_large(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
A product past the plain loop: on the AMX or x86-64 tiles where gemm_elsewhere takes it,...
constexpr bool prefetches_next_row_of_c
Whether Kernel has the C tile of the next row of tiles prefetched while a tile computes (Kernel::pref...
void subtract_rows(double *y, std::size_t ldy, const double *f, std::size_t ldf, const double *x, std::size_t ldx, std::size_t terms, std::size_t count)
subtract_rows for double, the units chosen as subtract_products chooses them: each element the value ...
left_kernel
The tile a product's left operand is packed for (numerical::detail::pack_left): plain where it is not...
@ avx512
The AVX-512 tile.
@ vector
The portable path's vector tile.
@ plain
gemm_rows, or a scalar whose products are not shared: A is read in place.
void gemm_pack_at(const T *at, std::size_t ldat, std::size_t rows, std::size_t kc, T *out)
The slivers of Mr rows of A (rows × kc), column by column, from its transpose at (kc × rows row-major...
T conjugate_dot_product(const T *x, const T *y, std::size_t count)
Σ conj(x[i]) · y[i], in order, where no native vector holds the parts.
constexpr std::size_t dot_chunk_weight(std::size_t chunks) noexcept
The work chunked_dot weighs one of its chunks chunks, in multiply-adds for parallel_for's split: four...
bool amx_usable() noexcept
Whether this chip's AMX can be used: known family and a passing self-test, decided once.
left_kernel left_kernel_for(std::size_t, std::size_t, std::size_t) noexcept
The tile gemm runs a product of m × n by depth k on, for a left operand packed once for many products...
constexpr std::size_t gemm_kernel_speedup
The multiply-adds the register tile does in the time of one in work_per_thread's dependent chain (abo...
void gemm_large_complex(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
A complex product past the plain loop: on the vector tile on the portable path, whole complex product...
void exchange(buffer< T > &a, std::size_t n, std::size_t i, std::size_t j)
Exchanges rows i and j, then columns i and j, of the row-major n × n a: a similarity.
std::size_t parallel_threads(std::size_t count, std::size_t work_per_index)
The threads parallel_for shares count indices of work_per_index out to (threads_for): the hardware's,...
carried_t< T > real_part(const T &x)
The real part of a real (or general) scalar, in carried_t.
void gemm_at(std::size_t m, std::size_t n, std::size_t k, const T *at, std::size_t ldat, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm_layouts with A given as Aᵀ (at, k × m, ldat apart).
T operator_times(const T &a, const T &b)
a · b, as std::operator* computes it, in line. libc++'s complex product (the four products and their ...
constexpr std::size_t vector_speedup
The multiply-adds a vectorised inner loop does in the time of one in work_per_thread's dependent chai...
void zero_c(T *c, std::size_t ldc, std::size_t m, std::size_t n)
Zeroes C's m × n elements, its rows ldc apart: on the threads where they lie together and are many (p...
constexpr std::size_t gemm_mr
The rows of C one register tile covers.
constexpr std::size_t narrow_group
The rows of C one block of gemm_rows takes together: each element a chain of multiply-adds that waits...
void columns_amx_lanes(const double *a, std::size_t lda, std::size_t rows, std::size_t cols, const double *x, double *y)
amx_dot_rows_f64's sums of the rows rows of the column-major a (column j at a + j · lda) without the ...
void add_product(double *y, const double &f, const double *x, std::size_t count)
y[i] += f · x[i] for double, as subtract_product.
packed_left< double > pack_left< double >(std::size_t m, std::size_t n, std::size_t k, const double *a, std::size_t lda, buffer< double > &storage)
pack_left for double, on the tile left_kernel_for chooses.
gemm_path best_gemm_path(gemm_path most) noexcept
The most capable path that can run here, most at the highest.
void row_dots(const std::complex< double > *m, std::size_t ld, std::size_t rows, const std::complex< double > *x, std::size_t count, std::complex< double > *out)
row_dots for std::complex<double>, through complex_row_dots.
void subtract_products(double *y, const double *f, const double *x, std::size_t ldx, std::size_t terms, std::size_t count)
y[i] -= Σₜ f[t] · x[(t · ldx) + i] for double, the units chosen as subtract_product chooses them,...
constexpr std::size_t work_per_thread
The work one thread must have to pay for starting it: about 60 µs of multiply-adds,...
constexpr std::size_t dot_chunk
The elements of one partial sum of chunked_dot: fixed, so the sum does not depend on the threads.
T chunked_conjugate_dot(const T *a, const T *b, std::size_t count)
Σ conj(a[i]) · b[i] (zdotc): chunked_dot of each chunk by conjugate_dot_product, without a conjugated...
void gemm_with(const packed_left< T > &left, std::size_t n, const T *b, std::size_t ldb, T *c, std::size_t ldc)
C += A·B for the packed left (pack_left) and the k × n b, as gemm computes it, A read in place here,...
const T * rows_from_transpose(const T *x, std::size_t ld, std::size_t rows, std::size_t cols, buffer< T > &scratch, std::size_t &ld_out)
A row-major rows × cols operand for the plain loop, copied into scratch from its transpose x (cols × ...
void zero_block(T *c, std::size_t ldc, std::size_t first, std::size_t last, std::size_t from, std::size_t to) noexcept
Zeroes rows first to last (excluded) of C, columns from to to (excluded), its rows ldc apart.
void gemm_layouts(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm with either operand given as its transpose (A: a_rows or a_transposed, B: b_rows or b_transposed...
void gemm_split(std::size_t m, std::size_t n, std::size_t k, const Z *a, std::size_t lda, const Z *b, std::size_t ldb, Z *c, std::size_t ldc)
C += A·B for complex numbers through the real kernel, by blocks of split_columns columns and split_de...
complex_term
How a complex product rounds a term that sums nothing (0 + a·b) on the path in use,...
@ compiler
The vector tile's and the plain loop's: 0 + a·b as the compiler rounds it (rounded_apart_add).
@ tiles
The x86-64 complex tiles': each part's product from zero, re·re − im·im and re·im + im·re,...
@ split
arm64's real kernel's (gemm_split): Re a·Re b from zero, −Im a·Im b fused into it; Re a·Im b,...
void subtract_product(double *y, const double &f, const double *x, std::size_t count)
y[i] −= f · x[i] for double on the widest x86-64 units the matrix product's kernel may use (gemm_path...
void gemm_large_layouts(std::size_t m, std::size_t n, std::size_t k, const double *a, std::size_t lda, const double *b, std::size_t ldb, double *c, std::size_t ldc, c_start start)
A double product past the plain loop, B and A laid out as B and A say (gemm_layouts).
void gemm_atbt(std::size_t m, std::size_t n, std::size_t k, const T *at, std::size_t ldat, const T *bt, std::size_t ldbt, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm_layouts with both operands given as their transposes.
constexpr std::size_t plain_loop_least_width
The fewest columns of a product plain_loop_pays sends to the plain loop on the x86-64 tiles: there an...
packed_left< T > pack_left(std::size_t m, std::size_t, std::size_t k, const T *a, std::size_t lda, buffer< T > &)
The m × k a packed once for products with it of about n columns each (gemm_with), in storage (grown a...
void gemm_bt(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *bt, std::size_t ldbt, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm_layouts with B given as Bᵀ (bt, n × k, ldbt apart).
constexpr std::size_t split_kept
The most elements an operand of gemm_split keeps from one product to the next on a thread: 2 MiB of d...
T rounded_apart_add(T sum, T a, T b) noexcept
sum + a·b as the vector tile rounds it (vector_tile_kernel), so that the plain loop's elements are it...
bool plain_loop_suits(std::size_t, std::size_t, std::size_t) noexcept
plain_loop_pays for a product of T: double's and float's, those it was measured for.
constexpr std::size_t tasks_per_thread
The tasks a product's compute pass hands each thread at least, while its rows allow: a slower core th...
constexpr std::size_t gemm_rows_work
The most multiply-adds of a product gemm runs by rows (gemm_rows) whatever its width: up to there its...
constexpr std::size_t split_depth
The depth of one real product of gemm_split (split_columns).
T * line_aligned(T *at) noexcept
The first element of at's run that starts a cache line (or at's nearest element towards one,...
std::atomic< std::uint8_t > & gemm_path_found() noexcept
The path gemm_path_current found last, packed with the cap it was found under (cap · 8 + path); 0xFF ...
constexpr bool gemm_shares(std::size_t m, std::size_t n, std::size_t k) noexcept
Whether the product's threads pay: below Kernel's serial_below multiply-adds, the product runs on the...
constexpr std::size_t split_row_width
The columns split_row_columns holds at once: a block of B's rows read twice stays in the caches.
constexpr bool neon_compiled
Whether the NEON path's tile is compiled in: on every arm64 target GCC or Clang builds.
void parallel_for(std::size_t count, std::size_t work_per_index, const Body &body)
Runs body(i) for every i in [0, count).
constexpr std::size_t gemm_nr
The columns of C one register tile covers: 96 bytes of elements (12 double, 24 float),...
void matrix_times_vector(const T *a, std::size_t lda, std::size_t rows, std::size_t cols, const T *x, T *y)
y = A x for the row-major rows × cols a, its rows lda apart: each element the dot product of its row ...
constexpr std::size_t split_narrow_area
The largest n·k of a complex product gemm_split shares out by blocks of rows: the real product of its...
constexpr std::size_t gemm_mc
The rows of one task: an A block (gemm_mc × gemm_kc) stays in the L2 cache.
std::size_t transpose_piece(std::size_t from_bytes, std::size_t to_bytes) noexcept
The side of the square pieces a transposing copy walks a tile by, from the bytes between rows of its ...
void split_expanded_block(std::size_t m, std::size_t nb, std::size_t kb, std::size_t pc, std::size_t jc, const Z *a, std::size_t lda, const Z *b, std::size_t ldb, typename Z::value_type *ar, typename Z::value_type *br, typename Z::value_type *cr)
One block of gemm_split (split_parts_block's) by one real product of its expanded operands,...
gemm_path gemm_path_current() noexcept
The path the next double product takes: the most capable one under the cap that runs here (each path'...
bool columns_times_vector(const T *a, std::size_t lda, std::size_t rows, std::size_t cols, const T *x, T *y)
y = A x for the column-major rows × cols a (column j at a + j · lda; NumPy's a.T, a transposed operan...
bool gemm_float_elsewhere(std::size_t m, std::size_t n, std::size_t k, const float *a, std::size_t lda, const float *b, std::size_t ldb, float *c, std::size_t ldc, c_start start=c_start::kept)
A float product on the AMX where it suits the product, on the NEON tile on arm64's fused paths,...
void narrow_block(std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start) noexcept
C += A·B for R rows and N columns of C (C = A·B as start says), each sum in ascending order of the de...
std::integral_constant< term_rounding, Mode > rounding_is
The rounding mode Mode as a type, which picks rounded_term's overload.
constexpr std::size_t packed_line
The cache line a packed operand starts.
constexpr std::size_t narrow_columns
The columns of C one block of gemm_rows keeps in registers over the whole depth.
const T * rows_in_place(const T *x, std::size_t ld, std::size_t &ld_out) noexcept
A row-major rows × cols operand for the plain loop, x itself (ld apart).
void gemm(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
C += A·B, every element of C summed in ascending order of the depth.
void few_rows(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start)
C += A·B (C = A·B as start says) for m rows of A, by groups of at most narrow_group rows and blocks o...
T rounded_term(rounding_is< term_rounding::compiler >, T sum, T a, T b) noexcept
sum + a·b as the compiler rounds it (rounded_apart_add).
bool few_complex_rows(std::size_t m, std::size_t n, std::size_t k) noexcept
Whether gemm runs a complex product of m rows, B k × n, a few rows at a time with B read by rows (few...
bool gemm_elsewhere(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
A product a faster path than the vector one takes, if any: none for this scalar.
constexpr std::size_t matrix_vector_weight
The weight of a matrix element in matrix_times_vector's share of the threads, beside its multiply-add...
std::array< T, gemm_mr *gemm_nr< T > > gemm_tile
constexpr std::size_t narrow_least_depth
The least depth at which rows of C wider than one block (narrow_columns) run by blocks in gemm_rows; ...
packed_left< T > left_rows(const packed_left< T > &left, std::size_t r0, std::size_t rows) noexcept
The rows [r0, r0 + rows) of left, r0 a multiple of left.granule: their slivers in its packing,...
constexpr std::size_t gemm_kc
The depth of one panel: a B sliver (gemm_kc × gemm_nr) stays in the L1 cache.
void gemm_pack_bt(const T *bt, std::size_t ldbt, std::size_t kc, std::size_t cols, T *out)
gemm_pack_b of B = Bᵀᵀ, from the row-major bt (cols × kc, leading dimension ldbt): each column of a s...
void gemm_packed(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, const T *a_packed=nullptr, c_start start=c_start::kept)
The packed product with the tile Kernel computes: for each block of B's columns and each panel of the...
left_kernel left_kernel_for< double >(std::size_t m, std::size_t n, std::size_t k) noexcept
left_kernel_for a double product: past gemm_rows, as gemm_elsewhere chooses, of a depth of one panel;...
double dot_product(const double *x, const double *y, std::size_t count)
Σ x[i] · y[i] for double, as subtract_product chooses its units (in another order of summation there,...
void complex_scaled_add(C &y, const C &f, const C &x)
y += f · x of one complex number, its part products rounded apart as libc++'s operator computes them ...
void split_parts_block(std::size_t m, std::size_t nb, std::size_t kb, std::size_t pc, std::size_t jc, const Z *a, std::size_t lda, const Z *b, std::size_t ldb, typename Z::value_type *ar, typename Z::value_type *br, typename Z::value_type *cr)
One block of gemm_split ([Re C | Im C] at cr, m × nb each, += A·B, A m × kb and B kb × nb) by four re...
void gemm_rows_rounded(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
C += A·B by the plain i-k-j loop, parallel over the rows of C. Past a depth of 1 (narrow_least_depth)...
bool rows_terms_fuse() noexcept
Whether gemm_rows rounds each term c + a·b as one fused multiply-add on the path in use,...
constexpr std::size_t row_group
The rows a thread takes of a matrix times a vector (matrix_times_vector).
constexpr bool reads_a_in_place
Whether Kernel reads A where it lies (Kernel::tile_a_in_place).
constexpr std::size_t split_parts_rows
The most rows of A for which gemm_split runs four real products of a block's parts on the AMX path (s...
constexpr std::size_t task_rows_cap(std::size_t) noexcept
The most rows of a task of Kernel over a panel of depth kc: its block of rows.
c_start
What a product finds in C: the sums it adds to (C += A·B), or nothing it reads (C = A·B)....
@ zero
C = A·B: C's elements before the product are not read.
constexpr std::size_t gemm_rows_weight
The divisor of a block of gemm_rows' rows' multiply-adds as its threads are weighed: on x86-64 its wo...
constexpr std::size_t product_rows
The rows of y subtract_rows takes at once.
packed_left< T > read_in_place(std::size_t m, std::size_t k, const T *a, std::size_t lda) noexcept
The m × k a read in place by gemm_with, unpacked: each product packs it as gemm does.
constexpr std::size_t gemm_min_depth
Below this depth the plain loop runs too: each panel reads and writes its tiles of C once,...
bool plain_loop_suits< float >(std::size_t m, std::size_t n, std::size_t k) noexcept
plain_loop_pays for float: double's rule, where the plain loop outran the packed tiles as for double ...
bool gemm_double_elsewhere(std::size_t m, std::size_t n, std::size_t k, const double *a, std::size_t lda, const double *b, std::size_t ldb, double *c, std::size_t ldc, c_start start=c_start::kept)
A double product on the path in use (numerical::gemm_path_in_use), unless it is the portable one,...
constexpr std::size_t column_rows
The rows of a column-major matrix columns_times_vector takes at once: their partial sums (32 KB on an...
split_operands< R > & split_storage()
The calling thread's split_operands. A product uses them whole on one thread, and the real product it...
void gemm_kernel(std::size_t kc, const T *ap, const T *bp, std::size_t ldb, gemm_tile< T > &tile)
tile += Aₚ·Bₚ over one panel, for any scalar.
bool by_rows(std::size_t m, std::size_t n, std::size_t k)
Whether gemm runs an m × k by k × n product by rows (gemm_rows): gemm_rows_columns columns or fewer; ...
void gemm_packed_triangle(std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, bool lower)
gemm_packed restricted to a triangle of the square C = A·B (n × n): its lower one (j ≤ i) where lower...
constexpr std::size_t gemm_nc
The columns of B packed at once, a whole number of slivers.
std::atomic< gemm_path > & gemm_path_cap() noexcept
The most capable path products may take, as numerical::limit_gemm_path set it: none is probed until a...
constexpr std::size_t gemm_rows_columns
The most columns of C for which gemm runs any product by gemm_rows, whose blocks keep C's rows in reg...
void gemm_with< double >(const packed_left< double > &left, std::size_t n, const double *b, std::size_t ldb, double *c, std::size_t ldc)
gemm_with for double: on the tile A was packed for, A read from its packing, where gemm would choose ...
bool plain_loop_suits< double >(std::size_t m, std::size_t n, std::size_t k) noexcept
plain_loop_pays for double.
bool rows_fuse() noexcept
Whether the tile a double or float product past the plain loop runs on fuses each term (gemm_rows rou...
constexpr std::size_t gemm_slivers_per_task
The B slivers of one task.
void prefetch_next_row(const T *, std::size_t) noexcept
Nothing, for a tile that prefetches no C (prefetches_next_row_of_c).
constexpr bool reads_partial_b_in_place
Whether Kernel reads B's last partial sliver in place (Kernel::partial_b_in_place): a tile that store...
constexpr bool writes_fresh
Whether Kernel's tiles write C as it holds nothing, their sums from zero (Kernel::tile's fresh): C th...
constexpr std::size_t multiply_add_weight
The real multiply-adds one multiply-add of T weighs: 1 for a real number.
T chunked_dot(const T *a, const T *b, std::size_t count, const Dot &dot)
Σ a[i] · b[i] for a plain dot product: chunks of dot_chunk elements in parallel, each by dot,...
constexpr bool packs_a(const T *a_packed, std::size_t) noexcept
Whether gemm_packed packs A for Kernel: unless given packed (a_packed).
void split_row_columns(const C *a, const C *b, std::size_t row_step, std::size_t column_step, std::size_t inner, std::size_t first, std::size_t last, C *y, bool kept=false)
Columns first to last of the complex vector a (inner) times B (B[k][j] at b[(k · row_step) + (j · col...
void gemm_rows(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
C += A·B by the plain i-k-j loop (gemm_rows_rounded), each element rounded as the tile of the path in...
void complex_row_dots(const C *m, std::size_t ld, std::size_t rows, const C *x, std::size_t count, C *out)
row_dots of complex numbers: each row by complex_dot_product, as chunked_dot sums a row.
constexpr std::size_t memory_element_weight
The weight of an element a pass bound by the memory writes or moves, against a multiply-add,...
term_rounding
How numerical::detail::gemm_rows rounds each term sum + a·b, as the tile of the path in use does: as ...
@ x86_fused
std::fma in functions built for AVX2 with FMA.
@ fused
std::fma, where every target has the instruction.
@ compiler
As the compiler rounds sum + a·b (rounded_apart_add).
The NumericAL public API.
std::vector< T, detail::default_init_allocator< T > > buffer
The storage of vector, matrix and tensor, namely a std::vector whose buffer<T>(n) and resize(n) leave...
std::size_t max_threads()
The threads a kernel may use: the cap if one is set, else the hardware threads (starting none).
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
T dot(const vector< T > &a, const vector< T > &b)
Dot product.
gemm_path gemm_path_in_use() noexcept
The path double matrix products take.
gemm_path limit_gemm_path(gemm_path most) noexcept
Keeps double matrix products on most or a more portable path: the most capable path at or below it th...
gemm_path
The kernels a double matrix product can run on (a float one too on arm64), from the most portable....
@ avx512
AVX-512 with fused multiply-adds.
@ portable
The compiler's vectors (16 bytes on x86-64 wheels), as the compiler rounds c + a·b (see gemm).
@ avx2
AVX2 with fused multiply-adds.
@ amx
The Apple AMX coprocessor, fused multiply-adds; the products it does not suit on the NEON path's tile...
@ neon
arm64's vectors with fused multiply-adds, for double and float.
A minimal std::thread parallel-for for the hot kernels.
The native vector of float and double the kernels hold their lanes in (GNU vector extensions,...
#define NUMERICAL_ALWAYS_INLINE
A as gemm_packed reads it by default: row-major, m × k, its rows lda apart, packed by the tile.
static void pack(const T *a, std::size_t lda, std::size_t i0, std::size_t p0, std::size_t rows, std::size_t kc, T *out)
The rows from i0 of A, rows × kc from column p0, packed for Kernel.
static const T & at(const T *a, std::size_t lda, std::size_t i, std::size_t p) noexcept
A's element in row i and column p.
static const T * rows_of(const T *a, std::size_t lda, std::size_t, std::size_t, buffer< T > &, std::size_t &ld_out)
A row-major for the plain loop: itself.
A given as Aᵀ: row-major, k × m, its rows lda apart (C = Aᵀᵀ·B, no copy of the transpose).
static const T * rows_of(const T *at, std::size_t ldat, std::size_t rows, std::size_t cols, buffer< T > &scratch, std::size_t &ld_out)
A row-major for the plain loop, copied from at into scratch.
static const T & at(const T *at_, std::size_t ldat, std::size_t i, std::size_t p) noexcept
A's element in row i and column p: Aᵀ's in row p and column i.
static void pack(const T *at, std::size_t ldat, std::size_t i0, std::size_t p0, std::size_t rows, std::size_t kc, T *out)
The rows from i0 of A, rows × kc from column p0, packed as Kernel lays them out (gemm_pack_at).
B as gemm_packed reads it by default: row-major, k × n, its rows ldb apart.
static void pack(const T *b, std::size_t ldb, std::size_t p0, std::size_t j0, std::size_t kc, std::size_t cols, T *out)
The kc × cols block of B from row p0 and column j0 into slivers of Nr columns.
static constexpr bool in_place(std::size_t ldb) noexcept
Whether Kernel reads B's whole slivers in place.
static const T * rows_of(const T *x, std::size_t ld, std::size_t, std::size_t, buffer< T > &, std::size_t &ld_out)
The rows × cols operand row-major for the plain loop: x itself, ld apart.
static const T & at(const T *b, std::size_t ldb, std::size_t p, std::size_t j) noexcept
B's element in row p and column j.
B given as Bᵀ: row-major, n × k, its rows ldb apart (C = A·Bᵀᵀ, no copy of the transpose).
static const T & at(const T *bt, std::size_t ldbt, std::size_t p, std::size_t j) noexcept
B's element in row p and column j: Bᵀ's in row j and column p.
static constexpr bool in_place(std::size_t) noexcept
Never in place: a sliver's columns lie apart.
static const T * rows_of(const T *x, std::size_t ld, std::size_t rows, std::size_t cols, buffer< T > &scratch, std::size_t &ld_out)
The rows × cols operand row-major for the plain loop, copied from its transpose x (ld apart) into scr...
static void pack(const T *bt, std::size_t ldbt, std::size_t p0, std::size_t j0, std::size_t kc, std::size_t cols, T *out)
The kc × cols block of B from row p0 and column j0 (Bᵀ's rows from j0, its columns from p0) into sliv...
A left operand packed once for several products by it (an LU's −L₂₁ against blocks of its U₁₂): the t...
std::size_t lda
A's row distance.
std::size_t k
A's columns, the depth.
const T * packed
A packed for the tile (null where plain).
left_kernel kernel
The tile.
std::size_t granule
The rows of the tile's slivers: a block of rows from a multiple of it reads the packing.
The real operands of gemm_split, kept by each thread from one product to the next (split_storage): ma...
buffer< R > b
B's parts for a block of columns and of depth: [Re B | Im B], or [[Re B, Im B], [Im B,...
buffer< R > c
[Re C | Im C], a block of columns.
buffer< R > a
A's parts for a block of depth: [Re A | −Im A | Im A], or [Re A | −Im A].
Nothing to turn on around a task.
The vector path's tile: gemm_mr × gemm_nr elements of C in registers (gemm_kernel),...
static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
Whether B's rows ldb apart are read in place: no farther apart than a packed sliver's (and a B that o...
static constexpr std::size_t speedup
The weight of its work (gemm_kernel_speedup).
static constexpr std::size_t b_in_place_rows
The most rows of A for which gemm_packed reads B in place at any distance between its rows (where rea...
static constexpr std::size_t depth
The panel depth.
static void tile(std::size_t kc, const T *ap, const T *bp, std::size_t ldb, T *c, std::size_t ldc, std::size_t rows, std::size_t cols)
C += Aₚ·Bₚ for one tile of C (rows × cols) over a panel of depth kc, B's rows ldb apart.
static constexpr std::size_t mr
The tile's rows.
static constexpr std::size_t serial_below
Every product with work shares it (gemm_shares).
static constexpr std::size_t nr
The tile's columns.
static void pack_a(const T *a, std::size_t lda, std::size_t rows, std::size_t kc, T *out)
A's slivers for this tile (gemm_pack_a).
static constexpr std::size_t block_rows
The x86-64 tiles of the double matrix product: AVX-512 and AVX2 with FMA, chosen at run time.