NumericAL
A header-only C++20 lexer built on REAL
Loading...
Searching...
No Matches
amx.hpp
Go to the documentation of this file.
1
22#ifndef NUMERICAL_AMX_HPP
23#define NUMERICAL_AMX_HPP
24
25#include <algorithm>
26#include <array>
27#include <cstring>
28#include <atomic>
29#include <cmath>
30#include <cstddef>
31#include <cstdint>
32#include <initializer_list>
33
34#include "simd.hpp"
35
36#if defined(__APPLE__)
37# include <TargetConditionals.h>
38#endif
39#if defined(__APPLE__) && defined(__aarch64__) && TARGET_OS_OSX && !defined(NUMERICAL_NO_AMX)
40# define NUMERICAL_AMX_COMPILED // the AMX path is compiled in (tested with #ifdef only)
41# include <sys/sysctl.h>
42#endif
43
45
46#ifdef NUMERICAL_AMX_COMPILED
48 enum class amx_op : unsigned
49 {
50 ldx = 0,
51 ldy = 1,
52 ldz = 4,
53 stz = 5,
54 fma64 = 10,
55 fma32 = 12,
56 };
57
63 template <amx_op Op>
64 inline void amx(std::uint64_t operand) noexcept
65 {
66 __asm__ volatile (".word (0x201000 + (%0 << 5) + 0%1 - ((0%1 >> 4) * 6))" // NOLINT(hicpp-no-assembler): no compiler emits the AMX
67 :
68 : "i" (static_cast<unsigned>(Op)), "r" (operand)
69 : "memory");
70 }
71
76 template <bool Off>
77 inline void amx_switch() noexcept
78 {
79 __asm__ volatile ("nop\nnop\nnop\n.word (0x201000 + (17 << 5) + %0)" // NOLINT(hicpp-no-assembler): no compiler emits the AMX
80 :
81 : "i" (static_cast<unsigned>(Off))
82 : "memory");
83 }
84
86 class amx_session
87 {
88 public:
89
90 amx_session() noexcept
91 {
92 amx_switch<false>();
93 }
94
95 amx_session(const amx_session&) = delete;
96 amx_session& operator=(const amx_session&) = delete;
97 amx_session(amx_session&&) = delete;
98 amx_session& operator=(amx_session&&) = delete;
99
100 ~amx_session()
101 {
102 amx_switch<true>();
103 }
104 };
105
107 [[nodiscard]] inline std::uint64_t amx_address(const void * at,
108 std::uint64_t index) noexcept
109 {
110 return (index << 56U) | reinterpret_cast<std::uintptr_t>(at); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the instruction takes the address as an integer
111 }
112
114 inline constexpr std::size_t amx_dot_group {8};
115
124 inline void amx_dot_rows_f64(const double* m,
125 std::size_t ld,
126 std::size_t rows,
127 const double* x,
128 std::size_t count,
129 double* out) noexcept
130 {
131 alignas(64) static constexpr std::array<double, 8> zero {};
132 alignas(64) std::array<std::array<double, 8>, amx_dot_group> lanes {};
133 const std::size_t whole {count - (count % 8)};
134 for (std::size_t r0 {0}; r0 < rows; r0 += amx_dot_group) {
135 const std::size_t group {std::min(amx_dot_group, rows - r0)};
136 for (std::size_t r {0}; r < group; ++r) {
138 }
139 for (std::size_t j {0}; j < whole; j += 8) {
140 amx<amx_op::ldx>(amx_address(x + j, 0));
141 for (std::size_t r {0}; r < group; ++r) {
142 amx<amx_op::ldy>(amx_address(m + ((r0 + r) * ld) + j, r));
143 amx<amx_op::fma64>((std::uint64_t {1} << 63U) | (std::uint64_t {r} << 20U) | (std::uint64_t {r} *64U)); // vector mode: Z row r += X0 ∘ Y r
144 }
145 }
146 for (std::size_t r {0}; r < group; ++r) { // every row stored before any is read: a read right behind its store waited on it
147 amx<amx_op::stz>(amx_address(lanes[r].data(), r));
148 }
149 for (std::size_t r {0}; r < group; ++r) {
150 for (std::size_t j {whole}; j < count; ++j) {
151 lanes[r][j % 8] = std::fma(m[((r0 + r) * ld) + j], x[j], lanes[r][j % 8]);
152 }
153 double sum {lanes[r][0]};
154 for (std::size_t l {1}; l < 8; ++l) {
155 sum += lanes[r][l];
156 }
157 out[r0 + r] = sum;
158 }
159 }
160 }
161
163 inline constexpr std::size_t amx_mr {32};
164
166 inline constexpr std::size_t amx_nr {16};
167
173 inline constexpr std::size_t amx_least_work {std::size_t {1} << 11U};
174
182 [[nodiscard]] constexpr bool amx_suits(std::size_t m,
183 std::size_t n,
184 std::size_t k) noexcept
185 {
186 const bool within_one_tile {m < amx_mr && n < amx_nr};
187 return m * n * k >= amx_least_work && !within_one_tile;
188 }
189
191 inline constexpr std::size_t amx_f32_side {32};
192
200 [[nodiscard]] constexpr bool amx_suits_f32(std::size_t m,
201 std::size_t n,
202 std::size_t k) noexcept
203 {
204 return m * n * k >= amx_least_work && (m >= 8 || n >= 24);
205 }
206
213 [[nodiscard]] constexpr bool amx_outruns_plain_loop(std::size_t m,
214 std::size_t n,
215 std::size_t k) noexcept
216 {
217 const bool within_one_tile {m < amx_mr && n < amx_nr};
218 return (m >= 16 && n >= 16) || (m * n * k >= (std::size_t {1} << 16U) && !within_one_tile);
219 }
220
224 inline constexpr std::size_t amx_in_place_stride {512};
225
236 [[gnu::noinline, gnu::aligned(64)]] inline void amx_depth_f64(std::size_t kc,
237 const double* ap,
238 const double* bp,
239 std::size_t ldb) noexcept
240 {
241 for (std::size_t p {0}; p < kc; ++p) {
242 const double* const a {ap + (p * amx_mr)};
243 const double* const b {bp + (p * ldb)};
244 for (std::uint64_t ya {0}; ya < 4; ++ya) {
245 amx<amx_op::ldy>(amx_address(a + (ya * 8), ya));
246 }
248 amx<amx_op::ldx>(amx_address(b + 8, 1));
249 for (std::uint64_t ya {0}; ya < 4; ++ya) {
250 for (std::uint64_t xb {0}; xb < 2; ++xb) {
251 // Matrix mode: Z row (2·ya + xb), X register xb (64 bytes each), Y register ya.
252 amx<amx_op::fma64>((((ya * 2) + xb) << 20U) | ((xb * 64) << 10U) | (ya * 64));
253 }
254 }
255 }
256 }
257
272 inline void amx_full_tile_f64(std::size_t kc,
273 const double* ap,
274 const double* bp,
275 std::size_t ldb,
276 double* tile,
277 std::size_t ld) noexcept
278 {
279 for (std::uint64_t ya {0}; ya < 4; ++ya) {
280 for (std::uint64_t xb {0}; xb < 2; ++xb) {
281 for (std::uint64_t j {0}; j < 8; ++j) {
282 amx<amx_op::ldz>(amx_address(tile + (((ya * 8) + j) * ld) + (xb * 8), (j * 8) + (ya * 2) + xb));
283 }
284 }
285 }
287 for (std::uint64_t ya {0}; ya < 4; ++ya) {
288 for (std::uint64_t xb {0}; xb < 2; ++xb) {
289 for (std::uint64_t j {0}; j < 8; ++j) {
290 amx<amx_op::stz>(amx_address(tile + (((ya * 8) + j) * ld) + (xb * 8), (j * 8) + (ya * 2) + xb));
291 }
292 }
293 }
294 }
295
298 [[gnu::noinline, gnu::aligned(64)]] inline void amx_depth_f32(std::size_t kc,
299 const float* ap,
300 const float* bp,
301 std::size_t ldb) noexcept
302 {
303 for (std::size_t p {0}; p < kc; ++p) {
304 const float* const a {ap + (p * amx_f32_side)};
305 const float* const b {bp + (p * ldb)};
307 amx<amx_op::ldy>(amx_address(a + 16, 1));
309 amx<amx_op::ldx>(amx_address(b + 16, 1));
310 for (std::uint64_t ya {0}; ya < 2; ++ya) {
311 for (std::uint64_t xb {0}; xb < 2; ++xb) {
312 amx<amx_op::fma32>((((ya * 2) + xb) << 20U) | ((xb * 64) << 10U) | (ya * 64));
313 }
314 }
315 }
316 }
317
319 inline void amx_depth(std::size_t kc,
320 const double* ap,
321 const double* bp,
322 std::size_t ldb) noexcept
323 {
325 }
326
328 inline void amx_depth(std::size_t kc,
329 const float* ap,
330 const float* bp,
331 std::size_t ldb) noexcept
332 {
334 }
335
337 template <typename T>
338 inline constexpr amx_op amx_multiply_add {amx_op::fma32};
339
341 template <>
342 inline constexpr amx_op amx_multiply_add<double> {amx_op::fma64};
343
351 template <typename T,
352 std::uint64_t Bands,
353 std::uint64_t Halves>
354 [[gnu::noinline, gnu::aligned(64)]] void amx_depth_part(std::size_t kc,
355 const T* ap,
356 const T* bp,
357 std::size_t ldb) noexcept
358 {
359 constexpr std::size_t lanes {64 / sizeof(T)}; // a register's elements
360 for (std::size_t p {0}; p < kc; ++p) {
361 const T* const a {ap + (p * amx_mr)};
362 const T* const b {bp + (p * ldb)};
363 for (std::uint64_t ya {0}; ya < Bands; ++ya) {
364 amx<amx_op::ldy>(amx_address(a + (ya * lanes), ya));
365 }
366 for (std::uint64_t xb {0}; xb < Halves; ++xb) {
367 amx<amx_op::ldx>(amx_address(b + (xb * lanes), xb));
368 }
369 for (std::uint64_t ya {0}; ya < Bands; ++ya) {
370 for (std::uint64_t xb {0}; xb < Halves; ++xb) {
371 amx<amx_multiply_add<T>>((((ya * 2) + xb) << 20U) | ((xb * 64) << 10U) | (ya * 64));
372 }
373 }
374 }
375 }
376
378 template <typename T,
379 std::uint64_t Bands>
380 void amx_depth_part_halves(std::size_t kc,
381 const T* ap,
382 const T* bp,
383 std::size_t ldb,
384 std::uint64_t halves) noexcept
385 {
386 if (halves == 1) {
388 }
389 else {
391 }
392 }
393
396 inline void amx_depth_part(std::size_t kc,
397 const double* ap,
398 const double* bp,
399 std::size_t ldb,
400 std::uint64_t bands,
401 std::uint64_t halves) noexcept
402 {
403 if (bands == 1) {
405 }
406 else if (bands == 2) {
408 }
409 else if (bands == 3) {
411 }
412 else { // every band: a partial tile, so one half (the whole tile goes to amx_depth)
414 }
415 }
416
418 inline void amx_depth_part(std::size_t kc,
419 const float* ap,
420 const float* bp,
421 std::size_t ldb,
422 std::uint64_t bands,
423 std::uint64_t halves) noexcept
424 {
425 if (bands == 1) {
427 }
428 else { // every band: a partial tile, so one half (the whole tile goes to amx_depth)
430 }
431 }
432
448 template <typename T>
449 void amx_partial_tile(std::size_t kc,
450 const T* ap,
451 const T* bp,
452 std::size_t ldb,
453 T* c,
454 std::size_t ldc,
455 std::size_t rows,
456 std::size_t cols) noexcept
457 {
458 static_assert(amx_f32_side == amx_mr, "both tiles have amx_mr rows");
459 constexpr std::size_t lanes {64 / sizeof(T)}; // a half row: one Z row
460 constexpr std::uint64_t bands {amx_mr / lanes}; // accumulators down the tile
461 alignas(64) std::array<T, amx_mr * lanes> cut {}; // the half row the last column cuts, per row
462 const std::uint64_t whole_halves {cols / lanes}; // halves wholly inside C
463 const std::size_t cut_width {cols % lanes}; // the cut half's elements inside C
464 const auto each_half = [&](auto&& at) {
465 for (std::uint64_t ya {0}; ya < bands; ++ya) {
466 for (std::uint64_t j {0}; j < lanes; ++j) {
467 const std::size_t row {(ya * lanes) + j};
468 if (row >= rows) {
469 continue;
470 }
471 for (std::uint64_t xb {0}; xb < 2; ++xb) {
472 const std::uint64_t z {(j * 2 * bands) + (ya * 2) + xb};
473 if (xb < whole_halves) {
474 at(c + (row * ldc) + (xb * lanes), z);
475 }
476 else if (xb == whole_halves && cut_width != 0) {
477 at(cut.data() + (row * lanes), z);
478 }
479 }
480 }
481 }
482 };
483 if (cut_width != 0) { // each row's elements by copies of fixed sizes (copy_few), where a loop was a call to memmove
484 for (std::size_t row {0}; row < rows; ++row) {
485 copy_few(c + (row * ldc) + (whole_halves * lanes), cut_width, cut.data() + (row * lanes));
486 }
487 }
488 each_half([](T* half, std::uint64_t z) { amx<amx_op::ldz>(amx_address(half, z)); });
489 const std::uint64_t used_bands {(rows + lanes - 1) / lanes}; // the accumulators C's rows fill
490 const std::uint64_t used_halves {std::min<std::uint64_t>(2, (cols + lanes - 1) / lanes)};
491 if (used_bands == bands && used_halves == 2) {
492 amx_depth(kc, ap, bp, ldb);
493 }
494 else {
496 }
497 each_half([](T* half, std::uint64_t z) { amx<amx_op::stz>(amx_address(half, z)); });
498 if (cut_width != 0) {
499 for (std::size_t row {0}; row < rows; ++row) {
500 copy_few(cut.data() + (row * lanes), cut_width, c + (row * ldc) + (whole_halves * lanes));
501 }
502 }
503 }
504
507 inline void amx_full_tile_f32(std::size_t kc,
508 const float* ap,
509 const float* bp,
510 std::size_t ldb,
511 float* tile,
512 std::size_t ld) noexcept
513 {
514 for (std::uint64_t ya {0}; ya < 2; ++ya) {
515 for (std::uint64_t xb {0}; xb < 2; ++xb) {
516 for (std::uint64_t j {0}; j < 16; ++j) {
517 amx<amx_op::ldz>(amx_address(tile + (((ya * 16) + j) * ld) + (xb * 16), (j * 4) + (ya * 2) + xb));
518 }
519 }
520 }
522 for (std::uint64_t ya {0}; ya < 2; ++ya) {
523 for (std::uint64_t xb {0}; xb < 2; ++xb) {
524 for (std::uint64_t j {0}; j < 16; ++j) {
525 amx<amx_op::stz>(amx_address(tile + (((ya * 16) + j) * ld) + (xb * 16), (j * 4) + (ya * 2) + xb));
526 }
527 }
528 }
529 }
530
537 inline void amx_tile_f32(std::size_t kc,
538 const float* ap,
539 const float* bp,
540 std::size_t ldb,
541 float* c,
542 std::size_t ldc,
543 std::size_t rows,
544 std::size_t cols) noexcept
545 {
546 if (rows == amx_f32_side && cols == amx_f32_side) {
547 amx_full_tile_f32(kc, ap, bp, ldb, c, ldc);
548 return;
549 }
550 amx_partial_tile(kc, ap, bp, ldb, c, ldc, rows, cols);
551 }
552
565 inline void amx_tile_f64(std::size_t kc,
566 const double* ap,
567 const double* bp,
568 std::size_t ldb,
569 double* c,
570 std::size_t ldc,
571 std::size_t rows,
572 std::size_t cols) noexcept
573 {
574 if ((rows < amx_mr || cols < amx_nr) && (cols < 8 || cols % 8 == 0)) {
575 amx_partial_tile(kc, ap, bp, ldb, c, ldc, rows, cols);
576 return;
577 }
578 through_edge<amx_mr, amx_nr>(c, ldc, rows, cols, [&](double* tile, std::size_t ld) noexcept { amx_full_tile_f64(kc, ap, bp, ldb, tile, ld); });
579 }
580
584 inline constexpr std::size_t amx_stream_least {2048};
585
594 inline void amx_pack_a(const double* a,
595 std::size_t lda,
596 std::size_t rows,
597 std::size_t kc,
598 double* out) noexcept
599 {
600 using pair = double __attribute__((vector_size(16)));
601 std::size_t ir {0};
602 for (; rows * kc >= amx_stream_least && ir + amx_mr <= rows; ir += amx_mr) {
603 double* const sliver {out + (ir * kc)};
604 std::size_t p {0};
605 for (; p + 2 <= kc; p += 2) { // two columns of four rows a step: the rows' pairs read whole, their lanes swapped
606 for (std::size_t i {0}; i < amx_mr; i += 4) {
607 std::array<pair, 4> row {};
608 for (std::size_t r {0}; r < 4; ++r) {
609 std::memcpy(&row.at(r), a + ((ir + i + r) * lda) + p, sizeof(pair));
610 }
611 const pair here_low {__builtin_shufflevector(row[0], row[1], 0, 2)};
612 const pair here_high {__builtin_shufflevector(row[2], row[3], 0, 2)};
613 const pair next_low {__builtin_shufflevector(row[0], row[1], 1, 3)};
614 const pair next_high {__builtin_shufflevector(row[2], row[3], 1, 3)};
615 __asm__ volatile ("stnp %q0, %q1, [%2]" // NOLINT(hicpp-no-assembler): as below, four elements at once
616 :
617 : "w" (here_low), "w" (here_high), "r" (sliver + (p * amx_mr) + i)
618 : "memory");
619 __asm__ volatile ("stnp %q0, %q1, [%2]" // NOLINT(hicpp-no-assembler): as below, four elements at once
620 :
621 : "w" (next_low), "w" (next_high), "r" (sliver + ((p + 1) * amx_mr) + i)
622 : "memory");
623 }
624 }
625 for (; p < kc; ++p) {
626 for (std::size_t i {0}; i < amx_mr; i += 2) {
627 const double first {a[((ir + i) * lda) + p]};
628 const double second {a[((ir + i + 1) * lda) + p]};
629 __asm__ volatile ("stnp %d0, %d1, [%2]" // NOLINT(hicpp-no-assembler): no builtin emits a pair past the L1 under both compilers
630 :
631 : "w" (first), "w" (second), "r" (sliver + (p * amx_mr) + i)
632 : "memory");
633 }
634 }
635 }
636 for (; ir + amx_mr <= rows; ir += amx_mr) { // whole slivers below the threshold, the loop free of the padding's test
637 double* const sliver {out + (ir * kc)};
638 std::size_t p {0};
639 for (; p + 2 <= kc; p += 2) { // as above, by ordinary stores
640 for (std::size_t i {0}; i < amx_mr; i += 2) {
641 pair first;
642 pair second;
643 std::memcpy(&first, a + ((ir + i) * lda) + p, sizeof(pair));
644 std::memcpy(&second, a + ((ir + i + 1) * lda) + p, sizeof(pair));
645 const pair here {__builtin_shufflevector(first, second, 0, 2)};
646 const pair next {__builtin_shufflevector(first, second, 1, 3)};
647 std::memcpy(sliver + (p * amx_mr) + i, &here, sizeof(pair));
648 std::memcpy(sliver + ((p + 1) * amx_mr) + i, &next, sizeof(pair));
649 }
650 }
651 for (; p < kc; ++p) {
652 for (std::size_t i {0}; i < amx_mr; ++i) {
653 sliver[(p * amx_mr) + i] = a[((ir + i) * lda) + p];
654 }
655 }
656 }
657 // The partial sliver as the whole ones, past the L1 from the same size, the rows past A zeros: element by element
658 // down the columns, it took a fifth of a thin product (M1 Pro, one core, 21 × 639 by 639 × 105: 28.4 → 23.1 µs).
659 if (ir < rows) {
660 double* const sliver {out + (ir * kc)};
661 const bool stream {rows * kc >= amx_stream_least};
662 std::size_t p {0};
663 for (; p + 2 <= kc; p += 2) {
664 for (std::size_t i {0}; i < amx_mr; i += 4) {
665 std::array<pair, 4> row {};
666 for (std::size_t r {0}; r < 4 && ir + i + r < rows; ++r) {
667 std::memcpy(&row.at(r), a + ((ir + i + r) * lda) + p, sizeof(pair));
668 }
669 const pair here_low {__builtin_shufflevector(row[0], row[1], 0, 2)};
670 const pair here_high {__builtin_shufflevector(row[2], row[3], 0, 2)};
671 const pair next_low {__builtin_shufflevector(row[0], row[1], 1, 3)};
672 const pair next_high {__builtin_shufflevector(row[2], row[3], 1, 3)};
673 if (stream) {
674 __asm__ volatile ("stnp %q0, %q1, [%2]" // NOLINT(hicpp-no-assembler): as above, four elements at once
675 :
676 : "w" (here_low), "w" (here_high), "r" (sliver + (p * amx_mr) + i)
677 : "memory");
678 __asm__ volatile ("stnp %q0, %q1, [%2]" // NOLINT(hicpp-no-assembler): as above, four elements at once
679 :
680 : "w" (next_low), "w" (next_high), "r" (sliver + ((p + 1) * amx_mr) + i)
681 : "memory");
682 }
683 else {
684 std::memcpy(sliver + (p * amx_mr) + i, &here_low, sizeof(pair));
685 std::memcpy(sliver + (p * amx_mr) + i + 2, &here_high, sizeof(pair));
686 std::memcpy(sliver + ((p + 1) * amx_mr) + i, &next_low, sizeof(pair));
687 std::memcpy(sliver + ((p + 1) * amx_mr) + i + 2, &next_high, sizeof(pair));
688 }
689 }
690 }
691 for (; p < kc; ++p) {
692 for (std::size_t i {0}; i < amx_mr; ++i) {
693 sliver[(p * amx_mr) + i] = ir + i < rows ? a[((ir + i) * lda) + p] : 0.0;
694 }
695 }
696 }
697 }
698
705 inline void amx_pack_a_f32(const float* a,
706 std::size_t lda,
707 std::size_t rows,
708 std::size_t kc,
709 float* out) noexcept
710 {
711 using quad = float __attribute__((vector_size(16)));
712 constexpr std::size_t stream_least {amx_stream_least * (sizeof(double) / sizeof(float))};
713 // Four rows' four elements as four columns.
714 const auto transposed = [](const quad& r0, const quad& r1, const quad& r2, const quad& r3) noexcept {
715 const quad low01 {__builtin_shufflevector(r0, r1, 0, 4, 1, 5)};
716 const quad high01 {__builtin_shufflevector(r0, r1, 2, 6, 3, 7)};
717 const quad low23 {__builtin_shufflevector(r2, r3, 0, 4, 1, 5)};
718 const quad high23 {__builtin_shufflevector(r2, r3, 2, 6, 3, 7)};
719 return std::array<quad, 4> {__builtin_shufflevector(low01, low23, 0, 1, 4, 5), __builtin_shufflevector(low01, low23, 2, 3, 6, 7),
721 };
722 // A column's eight rows, past the L1.
723 const auto store = [](const quad& upper, const quad& lower, float* at) noexcept {
724 __asm__ volatile ("stnp %q0, %q1, [%2]" // NOLINT(hicpp-no-assembler): as amx_pack_a, eight elements at once
725 :
726 : "w" (upper), "w" (lower), "r" (at)
727 : "memory");
728 };
729 std::size_t ir {0};
730 for (; rows * kc >= stream_least && ir + amx_f32_side <= rows; ir += amx_f32_side) {
731 float* const sliver {out + (ir * kc)};
732 std::size_t p {0};
733 for (; p + 4 <= kc; p += 4) { // four columns of eight rows a step: two blocks of 4 × 4 transposed
734 for (std::size_t i {0}; i < amx_f32_side; i += 8) {
735 std::array<quad, 8> row {};
736 for (std::size_t r {0}; r < 8; ++r) {
737 std::memcpy(&row.at(r), a + ((ir + i + r) * lda) + p, sizeof(quad));
738 }
739 const std::array<quad, 4> upper {transposed(row[0], row[1], row[2], row[3])};
740 const std::array<quad, 4> lower {transposed(row[4], row[5], row[6], row[7])};
741 float* const at {sliver + (p * amx_f32_side) + i};
742 store(upper[0], lower[0], at);
743 store(upper[1], lower[1], at + amx_f32_side);
744 store(upper[2], lower[2], at + (2 * amx_f32_side));
745 store(upper[3], lower[3], at + (3 * amx_f32_side));
746 }
747 }
748 for (; p < kc; ++p) {
749 for (std::size_t i {0}; i < amx_f32_side; i += 2) {
750 const float first {a[((ir + i) * lda) + p]};
751 const float second {a[((ir + i + 1) * lda) + p]};
752 __asm__ volatile ("stnp %s0, %s1, [%2]" // NOLINT(hicpp-no-assembler): as amx_pack_a, a pair past the L1
753 :
754 : "w" (first), "w" (second), "r" (sliver + (p * amx_f32_side) + i)
755 : "memory");
756 }
757 }
758 }
759 // The rest four columns of eight rows a step as above, the rows past A zeros, a partial sliver past the L1 from the
760 // same size: a row of A read along its length, where element by element down the columns took half a thin product
761 // (M1 Pro, one core, 21 × 639 by 639 × 105: 28.7 → 19.6 µs).
762 for (; ir < rows; ir += amx_f32_side) {
763 float* const sliver {out + (ir * kc)};
764 const std::size_t height {std::min(amx_f32_side, rows - ir)};
765 const bool stream {rows * kc >= stream_least}; // only a partial sliver gets here from that size
766 std::size_t p {0};
767 for (; p + 4 <= kc; p += 4) {
768 for (std::size_t i {0}; i < amx_f32_side; i += 8) {
769 std::array<quad, 8> row {};
770 for (std::size_t r {0}; r < 8 && i + r < height; ++r) {
771 std::memcpy(&row.at(r), a + ((ir + i + r) * lda) + p, sizeof(quad));
772 }
773 const std::array<quad, 4> upper {transposed(row[0], row[1], row[2], row[3])};
774 const std::array<quad, 4> lower {transposed(row[4], row[5], row[6], row[7])};
775 float* const at {sliver + (p * amx_f32_side) + i};
776 for (std::size_t c {0}; c < 4; ++c) {
777 if (stream) {
778 store(upper.at(c), lower.at(c), at + (c * amx_f32_side));
779 }
780 else {
781 std::memcpy(at + (c * amx_f32_side), &upper.at(c), sizeof(quad));
782 std::memcpy(at + (c * amx_f32_side) + 4, &lower.at(c), sizeof(quad));
783 }
784 }
785 }
786 }
787 for (; p < kc; ++p) {
788 for (std::size_t i {0}; i < amx_f32_side; ++i) {
789 sliver[(p * amx_f32_side) + i] = i < height ? a[((ir + i) * lda) + p] : 0.0F;
790 }
791 }
792 }
793 }
794
796 [[nodiscard]] constexpr bool amx_family_known(std::uint32_t family) noexcept
797 {
798 switch (family) {
799 case 0x1b588bb3U: // FIRESTORM_ICESTORM: M1, M1 Pro, Max, Ultra
800 case 0xda33d83dU: // BLIZZARD_AVALANCHE: M2 family
801 case 0xfa33415eU: // IBIZA: M3
802 case 0x5f4dea93U: // LOBOS: M3 Pro
803 case 0x72015832U: // PALMA: M3 Max
804 case 0x6f5129acU: // DONAN: M4
805 case 0x17d5b93aU: // BRAVA: M4 Pro, Max
806 return true;
807 default:
808 return false;
809 }
810 }
811
813 [[nodiscard]] inline std::uint32_t cpu_family() noexcept
814 {
815 std::uint32_t family {0};
816 std::size_t size {sizeof family};
817 (void)sysctlbyname("hw.cpufamily", &family, &size, nullptr, 0); // on failure family stays 0
818 return family;
819 }
820
825 [[nodiscard]] inline bool amx_dot_self_test() noexcept
826 {
827 constexpr std::size_t rows {11};
828 constexpr std::size_t count {21};
829 std::array<double, rows * count> m {};
830 std::array<double, count> x {};
831 std::array<double, rows> got {};
832 std::array<double, rows> want {};
833 for (std::size_t i {0}; i < m.size(); ++i) {
834 m[i] = std::ldexp(static_cast<double>((i * 37) % 101) - 50.0, -static_cast<int>(i % 9));
835 }
836 for (std::size_t j {0}; j < count; ++j) {
837 x[j] = 1.0 / static_cast<double>((j % 13) + 3);
838 }
839 for (std::size_t r {0}; r < rows; ++r) {
840 std::array<double, 8> lane {};
841 for (std::size_t j {0}; j < count; ++j) {
842 lane[j % 8] = std::fma(m[(r * count) + j], x[j], lane[j % 8]);
843 }
844 double sum {lane[0]};
845 for (std::size_t l {1}; l < 8; ++l) {
846 sum += lane[l];
847 }
848 want[r] = sum;
849 }
850 {
851 const amx_session on {};
852 amx_dot_rows_f64(m.data(), count, rows, x.data(), count, got.data());
853 }
854 return got == want;
855 }
856
859 [[nodiscard]] inline bool amx_f32_self_test() noexcept
860 {
861 constexpr std::size_t depth {7};
862 constexpr std::size_t side {amx_f32_side};
863 std::array<float, depth * side> a {};
864 std::array<float, depth * side> b {};
865 std::array<float, side * side> c {};
866 std::array<float, side * side> want {};
867 for (std::size_t i {0}; i < depth * side; ++i) {
868 a[i] = std::ldexp(static_cast<float>((i * 37) % 101) - 50.0F, -static_cast<int>(i % 9));
869 b[i] = 1.0F / static_cast<float>((i % 13) + 3);
870 }
871 for (std::size_t i {0}; i < side * side; ++i) {
872 c[i] = static_cast<float>(i % 17) * 0.1F;
873 want[i] = c[i];
874 }
875 for (const std::size_t rows : {side, std::size_t {19}}) {
876 const std::size_t cols {rows == side ? side : std::size_t {11}};
877 for (std::size_t i {0}; i < rows; ++i) {
878 for (std::size_t j {0}; j < cols; ++j) {
879 for (std::size_t p {0}; p < depth; ++p) {
880 want[(i * side) + j] = std::fma(a[(p * side) + i], b[(p * side) + j], want[(i * side) + j]);
881 }
882 }
883 }
884 {
885 const amx_session on {};
886 amx_tile_f32(depth, a.data(), b.data(), side, c.data(), side, rows, cols);
887 }
888 }
889 return c == want;
890 }
891
898 [[nodiscard]] inline bool amx_self_test() noexcept
899 {
900 constexpr std::size_t depth {7};
901 std::array<double, depth * amx_mr> a {};
902 std::array<double, depth * amx_nr> b {};
903 std::array<double, amx_mr * amx_nr> c {};
904 std::array<double, amx_mr * amx_nr> want {};
905 for (std::size_t i {0}; i < depth * amx_mr; ++i) {
906 a[i] = std::ldexp(static_cast<double>((i * 37) % 101) - 50.0, -static_cast<int>(i % 9));
907 }
908 for (std::size_t i {0}; i < depth * amx_nr; ++i) {
909 b[i] = 1.0 / static_cast<double>((i % 13) + 3);
910 }
911 for (std::size_t i {0}; i < amx_mr * amx_nr; ++i) {
912 c[i] = static_cast<double>(i % 17) * 0.1;
913 want[i] = c[i];
914 }
915 for (const std::size_t rows : {amx_mr, std::size_t {19}}) {
916 const std::size_t cols {rows == amx_mr ? amx_nr : std::size_t {11}};
917 for (std::size_t i {0}; i < rows; ++i) {
918 for (std::size_t j {0}; j < cols; ++j) {
919 for (std::size_t p {0}; p < depth; ++p) {
920 want[(i * amx_nr) + j] = std::fma(a[(p * amx_mr) + i], b[(p * amx_nr) + j], want[(i * amx_nr) + j]);
921 }
922 }
923 }
924 {
925 const amx_session on {};
926 amx_tile_f64(depth, a.data(), b.data(), amx_nr, c.data(), amx_nr, rows, cols);
927 }
928 }
929 const std::array<bool, 3> passed {c == want, amx_f32_self_test(), amx_dot_self_test()};
930 return std::count(passed.begin(), passed.end(), true) == 3; // no branch on a check that only a fault fails
931 }
932
934 [[nodiscard]] inline bool running_virtualised() noexcept
935 {
936 int present {0};
937 std::size_t size {sizeof present};
938 (void)sysctlbyname("kern.hv_vmm_present", &present, &size, nullptr, 0); // on failure present stays 0
939 return present != 0;
940 }
941
943 [[nodiscard]] inline std::atomic<bool>& amx_probed() noexcept
944 {
945 static std::atomic<bool> probed {false};
946 return probed;
947 }
948
951 [[nodiscard]] inline bool amx_usable_on(std::uint32_t family,
952 bool virtualised) noexcept
953 {
955 return false;
956 }
957 amx_probed().store(true);
958 return amx_self_test();
959 }
960
961#endif
962
964 [[nodiscard]] inline bool amx_usable() noexcept
965 {
966#ifdef NUMERICAL_AMX_COMPILED
967 static const bool usable {amx_usable_on(cpu_family(), running_virtualised())};
968 return usable;
969#else
970 return false;
971#endif
972 }
973} // namespace numerical::detail
974
975#endif // NUMERICAL_AMX_HPP
void copy_few(const T *from, std::size_t count, T *to) noexcept
copy_few_bytes of count elements of T, one at least and at most 128 bytes.
Definition simd.hpp:77
bool amx_usable() noexcept
Whether this chip's AMX can be used: known family and a passing self-test, decided once.
Definition amx.hpp:964
@ zero
C = A·B: C's elements before the product are not read.
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
The native vector of float and double the kernels hold their lanes in (GNU vector extensions,...