46#ifdef NUMERICAL_AMX_COMPILED
48 enum class amx_op :
unsigned
64 inline void amx(std::uint64_t operand)
noexcept
66 __asm__
volatile (
".word (0x201000 + (%0 << 5) + 0%1 - ((0%1 >> 4) * 6))"
68 :
"i" (
static_cast<unsigned>(Op)),
"r" (operand)
77 inline void amx_switch()
noexcept
79 __asm__
volatile (
"nop\nnop\nnop\n.word (0x201000 + (17 << 5) + %0)"
81 :
"i" (
static_cast<unsigned>(Off))
90 amx_session()
noexcept
95 amx_session(
const amx_session&) =
delete;
96 amx_session& operator=(
const amx_session&) =
delete;
97 amx_session(amx_session&&) =
delete;
98 amx_session& operator=(amx_session&&) =
delete;
107 [[nodiscard]]
inline std::uint64_t amx_address(
const void * at,
108 std::uint64_t index)
noexcept
110 return (index << 56U) |
reinterpret_cast<std::uintptr_t
>(at);
114 inline constexpr std::size_t amx_dot_group {8};
124 inline void amx_dot_rows_f64(
const double* m,
129 double* out)
noexcept
131 alignas(64)
static constexpr std::array<double, 8>
zero {};
132 alignas(64) std::array<std::array<double, 8>, amx_dot_group> lanes {};
133 const std::size_t whole {count - (count % 8)};
134 for (std::size_t r0 {0}; r0 < rows; r0 += amx_dot_group) {
135 const std::size_t group {std::min(amx_dot_group, rows - r0)};
136 for (std::size_t r {0}; r < group; ++r) {
139 for (std::size_t j {0}; j <
whole; j += 8) {
141 for (std::size_t r {0}; r <
group; ++r) {
143 amx<amx_op::fma64>((std::uint64_t {1} << 63U) | (std::uint64_t {r} << 20U) | (std::uint64_t {r} *64U));
146 for (std::size_t r {0}; r <
group; ++r) {
149 for (std::size_t r {0}; r <
group; ++r) {
150 for (std::size_t j {
whole}; j < count; ++j) {
151 lanes[r][j % 8] = std::fma(m[((
r0 + r) * ld) + j], x[j], lanes[r][j % 8]);
153 double sum {lanes[r][0]};
154 for (std::size_t l {1}; l < 8; ++l) {
163 inline constexpr std::size_t
amx_mr {32};
166 inline constexpr std::size_t
amx_nr {16};
173 inline constexpr std::size_t
amx_least_work {std::size_t {1} << 11U};
184 std::size_t k)
noexcept
202 std::size_t k)
noexcept
215 std::size_t k)
noexcept
218 return (m >= 16 && n >= 16) || (m * n * k >= (std::size_t {1} << 16U) && !
within_one_tile);
236 [[gnu::noinline, gnu::aligned(64)]]
inline void amx_depth_f64(std::size_t
kc,
239 std::size_t
ldb)
noexcept
241 for (std::size_t p {0}; p <
kc; ++p) {
242 const double*
const a {
ap + (p *
amx_mr)};
243 const double*
const b {
bp + (p *
ldb)};
244 for (std::uint64_t
ya {0};
ya < 4; ++
ya) {
249 for (std::uint64_t
ya {0};
ya < 4; ++
ya) {
250 for (std::uint64_t
xb {0};
xb < 2; ++
xb) {
277 std::size_t ld)
noexcept
279 for (std::uint64_t
ya {0};
ya < 4; ++
ya) {
280 for (std::uint64_t
xb {0};
xb < 2; ++
xb) {
281 for (std::uint64_t j {0}; j < 8; ++j) {
287 for (std::uint64_t
ya {0};
ya < 4; ++
ya) {
288 for (std::uint64_t
xb {0};
xb < 2; ++
xb) {
289 for (std::uint64_t j {0}; j < 8; ++j) {
298 [[gnu::noinline, gnu::aligned(64)]]
inline void amx_depth_f32(std::size_t
kc,
301 std::size_t
ldb)
noexcept
303 for (std::size_t p {0}; p <
kc; ++p) {
305 const float*
const b {
bp + (p *
ldb)};
310 for (std::uint64_t
ya {0};
ya < 2; ++
ya) {
311 for (std::uint64_t
xb {0};
xb < 2; ++
xb) {
322 std::size_t
ldb)
noexcept
331 std::size_t
ldb)
noexcept
337 template <
typename T>
351 template <
typename T,
357 std::size_t
ldb)
noexcept
359 constexpr std::size_t lanes {64 /
sizeof(
T)};
360 for (std::size_t p {0}; p <
kc; ++p) {
362 const T*
const b {
bp + (p *
ldb)};
378 template <
typename T,
384 std::uint64_t
halves)
noexcept
401 std::uint64_t
halves)
noexcept
406 else if (
bands == 2) {
409 else if (
bands == 3) {
423 std::uint64_t
halves)
noexcept
448 template <
typename T>
456 std::size_t cols)
noexcept
459 constexpr std::size_t lanes {64 /
sizeof(
T)};
461 alignas(64) std::array<T, amx_mr * lanes>
cut {};
463 const std::size_t
cut_width {cols % lanes};
466 for (std::uint64_t j {0}; j < lanes; ++j) {
467 const std::size_t
row {(
ya * lanes) + j};
471 for (std::uint64_t
xb {0};
xb < 2; ++
xb) {
472 const std::uint64_t z {(j * 2 *
bands) + (
ya * 2) +
xb};
474 at(c + (
row * ldc) + (
xb * lanes), z);
477 at(
cut.data() + (
row * lanes), z);
484 for (std::size_t
row {0};
row < rows; ++
row) {
489 const std::uint64_t
used_bands {(rows + lanes - 1) / lanes};
490 const std::uint64_t
used_halves {std::min<std::uint64_t>(2, (cols + lanes - 1) / lanes)};
499 for (std::size_t
row {0};
row < rows; ++
row) {
512 std::size_t ld)
noexcept
514 for (std::uint64_t
ya {0};
ya < 2; ++
ya) {
515 for (std::uint64_t
xb {0};
xb < 2; ++
xb) {
516 for (std::uint64_t j {0}; j < 16; ++j) {
522 for (std::uint64_t
ya {0};
ya < 2; ++
ya) {
523 for (std::uint64_t
xb {0};
xb < 2; ++
xb) {
524 for (std::uint64_t j {0}; j < 16; ++j) {
544 std::size_t cols)
noexcept
572 std::size_t cols)
noexcept
574 if ((rows <
amx_mr || cols <
amx_nr) && (cols < 8 || cols % 8 == 0)) {
578 through_edge<amx_mr, amx_nr>(c, ldc, rows, cols, [&](
double* tile, std::size_t ld)
noexcept {
amx_full_tile_f64(
kc,
ap,
bp,
ldb, tile, ld); });
598 double* out)
noexcept
603 double*
const sliver {out + (ir *
kc)};
605 for (; p + 2 <=
kc; p += 2) {
606 for (std::size_t i {0}; i <
amx_mr; i += 4) {
607 std::array<pair, 4>
row {};
608 for (std::size_t r {0}; r < 4; ++r) {
609 std::memcpy(&
row.at(r), a + ((ir + i + r) * lda) + p,
sizeof(
pair));
615 __asm__ volatile (
"stnp %q0, %q1, [%2]"
619 __asm__ volatile (
"stnp %q0, %q1, [%2]"
625 for (; p <
kc; ++p) {
626 for (std::size_t i {0}; i <
amx_mr; i += 2) {
627 const double first {a[((ir + i) * lda) + p]};
628 const double second {a[((ir + i + 1) * lda) + p]};
629 __asm__ volatile (
"stnp %d0, %d1, [%2]"
631 :
"w" (first),
"w" (second),
"r" (
sliver + (p *
amx_mr) + i)
637 double*
const sliver {out + (ir *
kc)};
639 for (; p + 2 <=
kc; p += 2) {
640 for (std::size_t i {0}; i <
amx_mr; i += 2) {
643 std::memcpy(&first, a + ((ir + i) * lda) + p,
sizeof(
pair));
644 std::memcpy(&second, a + ((ir + i + 1) * lda) + p,
sizeof(
pair));
651 for (; p <
kc; ++p) {
652 for (std::size_t i {0}; i <
amx_mr; ++i) {
660 double*
const sliver {out + (ir *
kc)};
663 for (; p + 2 <=
kc; p += 2) {
664 for (std::size_t i {0}; i <
amx_mr; i += 4) {
665 std::array<pair, 4>
row {};
666 for (std::size_t r {0}; r < 4 && ir + i + r < rows; ++r) {
667 std::memcpy(&
row.at(r), a + ((ir + i + r) * lda) + p,
sizeof(
pair));
674 __asm__ volatile (
"stnp %q0, %q1, [%2]"
678 __asm__ volatile (
"stnp %q0, %q1, [%2]"
691 for (; p <
kc; ++p) {
692 for (std::size_t i {0}; i <
amx_mr; ++i) {
693 sliver[(p *
amx_mr) + i] = ir + i < rows ? a[((ir + i) * lda) + p] : 0.0;
719 return std::array<quad, 4> {
__builtin_shufflevector(
low01,
low23, 0, 1, 4, 5),
__builtin_shufflevector(
low01,
low23, 2, 3, 6, 7),
720 __builtin_shufflevector(
high01,
high23, 0, 1, 4, 5),
__builtin_shufflevector(
high01,
high23, 2, 3, 6, 7)};
724 __asm__ volatile (
"stnp %q0, %q1, [%2]"
726 :
"w" (
upper),
"w" (lower),
"r" (at)
731 float*
const sliver {out + (ir *
kc)};
733 for (; p + 4 <=
kc; p += 4) {
735 std::array<quad, 8>
row {};
736 for (std::size_t r {0}; r < 8; ++r) {
737 std::memcpy(&
row.at(r), a + ((ir + i + r) * lda) + p,
sizeof(
quad));
740 const std::array<quad, 4> lower {transposed(
row[4],
row[5],
row[6],
row[7])};
748 for (; p <
kc; ++p) {
750 const float first {a[((ir + i) * lda) + p]};
751 const float second {a[((ir + i + 1) * lda) + p]};
752 __asm__ volatile (
"stnp %s0, %s1, [%2]"
763 float*
const sliver {out + (ir *
kc)};
764 const std::size_t height {std::min(
amx_f32_side, rows - ir)};
767 for (; p + 4 <=
kc; p += 4) {
769 std::array<quad, 8>
row {};
770 for (std::size_t r {0}; r < 8 && i + r < height; ++r) {
771 std::memcpy(&
row.at(r), a + ((ir + i + r) * lda) + p,
sizeof(
quad));
774 const std::array<quad, 4> lower {transposed(
row[4],
row[5],
row[6],
row[7])};
776 for (std::size_t c {0}; c < 4; ++c) {
787 for (; p <
kc; ++p) {
816 std::size_t size {
sizeof family};
827 constexpr std::size_t rows {11};
828 constexpr std::size_t count {21};
829 std::array<double, rows * count> m {};
830 std::array<double, count> x {};
831 std::array<double, rows>
got {};
832 std::array<double, rows>
want {};
833 for (std::size_t i {0}; i < m.size(); ++i) {
834 m[i] = std::ldexp(
static_cast<double>((i * 37) % 101) - 50.0, -
static_cast<int>(i % 9));
836 for (std::size_t j {0}; j < count; ++j) {
837 x[j] = 1.0 /
static_cast<double>((j % 13) + 3);
839 for (std::size_t r {0}; r < rows; ++r) {
840 std::array<double, 8>
lane {};
841 for (std::size_t j {0}; j < count; ++j) {
842 lane[j % 8] = std::fma(m[(r * count) + j], x[j],
lane[j % 8]);
845 for (std::size_t l {1}; l < 8; ++l) {
861 constexpr std::size_t depth {7};
863 std::array<float, depth * side> a {};
864 std::array<float, depth * side> b {};
865 std::array<float, side * side> c {};
866 std::array<float, side * side>
want {};
867 for (std::size_t i {0}; i < depth *
side; ++i) {
868 a[i] = std::ldexp(
static_cast<float>((i * 37) % 101) - 50.0F, -
static_cast<int>(i % 9));
869 b[i] = 1.0F /
static_cast<float>((i % 13) + 3);
871 for (std::size_t i {0}; i <
side *
side; ++i) {
872 c[i] =
static_cast<float>(i % 17) * 0.1F;
875 for (
const std::size_t rows : {
side, std::size_t {19}}) {
876 const std::size_t cols {rows ==
side ?
side : std::size_t {11}};
877 for (std::size_t i {0}; i < rows; ++i) {
878 for (std::size_t j {0}; j < cols; ++j) {
879 for (std::size_t p {0}; p < depth; ++p) {
900 constexpr std::size_t depth {7};
901 std::array<double, depth * amx_mr> a {};
902 std::array<double, depth * amx_nr> b {};
903 std::array<double, amx_mr * amx_nr> c {};
904 std::array<double, amx_mr * amx_nr>
want {};
905 for (std::size_t i {0}; i < depth *
amx_mr; ++i) {
906 a[i] = std::ldexp(
static_cast<double>((i * 37) % 101) - 50.0, -
static_cast<int>(i % 9));
908 for (std::size_t i {0}; i < depth *
amx_nr; ++i) {
909 b[i] = 1.0 /
static_cast<double>((i % 13) + 3);
912 c[i] =
static_cast<double>(i % 17) * 0.1;
915 for (
const std::size_t rows : {
amx_mr, std::size_t {19}}) {
916 const std::size_t cols {rows ==
amx_mr ?
amx_nr : std::size_t {11}};
917 for (std::size_t i {0}; i < rows; ++i) {
918 for (std::size_t j {0}; j < cols; ++j) {
919 for (std::size_t p {0}; p < depth; ++p) {
930 return std::count(
passed.begin(),
passed.end(),
true) == 3;
937 std::size_t size {
sizeof present};
945 static std::atomic<bool>
probed {
false};
966#ifdef NUMERICAL_AMX_COMPILED