NumericAL
A header-only C++20 lexer built on REAL
Loading...
Searching...
No Matches
gemm.hpp
Go to the documentation of this file.
1
18#ifndef NUMERICAL_GEMM_HPP
19#define NUMERICAL_GEMM_HPP
20
21#include <algorithm>
22#include <array>
23#include <bit>
24#include <cmath>
25#include <atomic>
26#include <complex>
27#include <concepts>
28#include <cstddef>
29#include <cstdint>
30#include <cstring>
31#include <memory>
32#include <type_traits>
33#include <utility>
34#include <vector>
35
36#if defined(__GNUC__) && defined(__aarch64__)
37# include <arm_neon.h>
38# define NUMERICAL_NEON_COMPILED // the arm64 fused tile is compiled in (tested with #ifdef only)
39#endif
40
41#include "parallel.hpp"
42#include "amx.hpp"
43#include "buffer.hpp"
44#include "simd.hpp"
45#include "x86.hpp"
46
47// A function kept out of its callers: inlined, it spent GCC's growth budget for the whole translation unit, and the
48// products, the Cholesky factorisation and the LU solve lost their own inlining (one arm64 thread, GCC 14: a product of
49// order 8 0.29 → 0.33 µs, a Cholesky factorisation of order 17 0.77 → 0.87 µs, a solve of order 512 0.47 → 0.54 ms).
50#if !defined(NUMERICAL_OUT_OF_LINE)
51# if defined(_MSC_VER) && !defined(__clang__)
52# define NUMERICAL_OUT_OF_LINE __declspec(noinline)
53# else
54# define NUMERICAL_OUT_OF_LINE [[gnu::noinline]] // NOLINT(cppcoreguidelines-macro-usage): an attribute, which no function can stand for
55# endif
56#endif
57
58#if defined(__clang__)
59// A block's depth loop kept scalar: vectorised, Clang sums its terms in order but splits each multiply-add (which
60// -ffp-contract=on lets it fuse, not makes it) into a product and a sum, the last terms fused: one element rounded
61// two ways, where the packed kernels fuse every term.
62# define NUMERICAL_SCALAR_DEPTH _Pragma("clang loop vectorize(disable) interleave(disable)")
63#else
64# define NUMERICAL_SCALAR_DEPTH
65#endif
66
67#if defined(__GNUC__)
68// A block's loops unrolled whole, so that each of its sums stays in a register: GCC at -O2 does not unroll them,
69// and kept the sums in memory (a product of order 17 on one arm64 core: 2.0 µs, against 0.31 under Clang).
70# define NUMERICAL_UNROLL_BLOCK _Pragma("GCC unroll 32")
71#else
72# define NUMERICAL_UNROLL_BLOCK
73#endif
74
75namespace numerical {
76
82 enum class gemm_path : std::uint8_t
83 {
84 portable = 0,
85 neon = 1,
86 avx2 = 2,
87 avx512 = 3,
88 amx = 4,
89 };
90} // namespace numerical
91
92namespace numerical::detail {
93
95#ifdef NUMERICAL_NEON_COMPILED
96 inline constexpr bool neon_compiled {true};
97#else
98 inline constexpr bool neon_compiled {false};
99#endif
100
102 [[nodiscard]] inline bool gemm_path_usable(gemm_path path) noexcept
103 {
104 switch (path) {
105#ifdef NUMERICAL_X86_COMPILED
106 case gemm_path::avx2: return cpu_features().avx2_fma;
107 case gemm_path::avx512: return cpu_features().avx512f;
108#endif
109 case gemm_path::neon: return neon_compiled;
110 case gemm_path::amx: return amx_usable();
111 default: return path == gemm_path::portable;
112 }
113 }
114
117 {
118 for (auto p {static_cast<int>(most)}; p > 0; --p) {
119 if (gemm_path_usable(static_cast<gemm_path>(p))) {
120 return static_cast<gemm_path>(p);
121 }
122 }
123 return gemm_path::portable;
124 }
125
128 [[nodiscard]] inline std::atomic<gemm_path>& gemm_path_cap() noexcept
129 {
130 static std::atomic<gemm_path> cap {gemm_path::amx};
131 return cap;
132 }
133
136 [[nodiscard]] inline std::atomic<std::uint8_t>& gemm_path_found() noexcept
137 {
138 static std::atomic<std::uint8_t> found {0xFF};
139 return found;
140 }
141
149 {
150 const unsigned cap {static_cast<unsigned>(gemm_path_cap().load(std::memory_order_relaxed))};
151 const unsigned found {gemm_path_found().load(std::memory_order_relaxed)};
152 if (found >> 3U == cap) {
153 return static_cast<gemm_path>(found & 7U);
154 }
155 const gemm_path path {best_gemm_path(static_cast<gemm_path>(cap))};
156 gemm_path_found().store(static_cast<std::uint8_t>((cap << 3U) | static_cast<unsigned>(path)), std::memory_order_relaxed);
157 return path;
158 }
159
168 [[nodiscard]] inline bool rows_fuse() noexcept
169 {
170#ifdef NUMERICAL_NEON_COMPILED
171 return gemm_path_cap().load(std::memory_order_relaxed) != gemm_path::portable;
172#else
173 return false;
174#endif
175 }
176
177#ifdef NUMERICAL_NEON_COMPILED
181 inline void neon_scaled_add(double* y,
182 const double* x,
183 double f,
184 std::size_t count) noexcept
185 {
186 const float64x2_t splat {vdupq_n_f64(f)};
187 std::size_t i {0};
188 for (; i + 8 <= count; i += 8) { // four independent vectors a step: one chain each would wait on its own load
189 vst1q_f64(y + i, vfmaq_f64(vld1q_f64(y + i), splat, vld1q_f64(x + i)));
190 vst1q_f64(y + i + 2, vfmaq_f64(vld1q_f64(y + i + 2), splat, vld1q_f64(x + i + 2)));
191 vst1q_f64(y + i + 4, vfmaq_f64(vld1q_f64(y + i + 4), splat, vld1q_f64(x + i + 4)));
192 vst1q_f64(y + i + 6, vfmaq_f64(vld1q_f64(y + i + 6), splat, vld1q_f64(x + i + 6)));
193 }
194 for (; i + 2 <= count; i += 2) {
195 vst1q_f64(y + i, vfmaq_f64(vld1q_f64(y + i), splat, vld1q_f64(x + i)));
196 }
197 for (; i < count; ++i) {
198 y[i] = std::fma(f, x[i], y[i]);
199 }
200 }
201
205 inline void neon_scaled_add(float* y,
206 const float* x,
207 float f,
208 std::size_t count) noexcept
209 {
210 std::size_t i {0};
211# if !defined(__clang__)
212 const float32x4_t splat {vdupq_n_f32(f)};
213 for (; i + 16 <= count; i += 16) { // four independent vectors a step
214 vst1q_f32(y + i, vfmaq_f32(vld1q_f32(y + i), splat, vld1q_f32(x + i)));
215 vst1q_f32(y + i + 4, vfmaq_f32(vld1q_f32(y + i + 4), splat, vld1q_f32(x + i + 4)));
216 vst1q_f32(y + i + 8, vfmaq_f32(vld1q_f32(y + i + 8), splat, vld1q_f32(x + i + 8)));
217 vst1q_f32(y + i + 12, vfmaq_f32(vld1q_f32(y + i + 12), splat, vld1q_f32(x + i + 12)));
218 }
219 for (; i + 4 <= count; i += 4) {
220 vst1q_f32(y + i, vfmaq_f32(vld1q_f32(y + i), splat, vld1q_f32(x + i)));
221 }
222# endif
223 for (; i < count; ++i) {
224 y[i] = std::fma(f, x[i], y[i]);
225 }
226 }
227
231 inline void neon_subtract_products(double* y,
232 const double* f,
233 const double* x,
234 std::size_t ldx,
235 std::size_t terms,
236 std::size_t count) noexcept
237 {
238 // Named registers, not an array: GCC kept an array of vectors in memory, three to five times slower.
239 std::size_t i {0};
240 for (; i + 32 <= count; i += 32) {
241 float64x2_t z0 {vld1q_f64(y + i + 0)};
242 float64x2_t z1 {vld1q_f64(y + i + 2)};
243 float64x2_t z2 {vld1q_f64(y + i + 4)};
244 float64x2_t z3 {vld1q_f64(y + i + 6)};
245 float64x2_t z4 {vld1q_f64(y + i + 8)};
246 float64x2_t z5 {vld1q_f64(y + i + 10)};
247 float64x2_t z6 {vld1q_f64(y + i + 12)};
248 float64x2_t z7 {vld1q_f64(y + i + 14)};
249 float64x2_t z8 {vld1q_f64(y + i + 16)};
250 float64x2_t z9 {vld1q_f64(y + i + 18)};
251 float64x2_t z10 {vld1q_f64(y + i + 20)};
252 float64x2_t z11 {vld1q_f64(y + i + 22)};
253 float64x2_t z12 {vld1q_f64(y + i + 24)};
254 float64x2_t z13 {vld1q_f64(y + i + 26)};
255 float64x2_t z14 {vld1q_f64(y + i + 28)};
256 float64x2_t z15 {vld1q_f64(y + i + 30)};
257 for (std::size_t t {0}; t < terms; ++t) {
258 const float64x2_t minus {vdupq_n_f64(-f[t])};
259 const double* const row {x + (t * ldx) + i};
260 z0 = vfmaq_f64(z0, minus, vld1q_f64(row + 0));
261 z1 = vfmaq_f64(z1, minus, vld1q_f64(row + 2));
262 z2 = vfmaq_f64(z2, minus, vld1q_f64(row + 4));
263 z3 = vfmaq_f64(z3, minus, vld1q_f64(row + 6));
264 z4 = vfmaq_f64(z4, minus, vld1q_f64(row + 8));
265 z5 = vfmaq_f64(z5, minus, vld1q_f64(row + 10));
266 z6 = vfmaq_f64(z6, minus, vld1q_f64(row + 12));
267 z7 = vfmaq_f64(z7, minus, vld1q_f64(row + 14));
268 z8 = vfmaq_f64(z8, minus, vld1q_f64(row + 16));
269 z9 = vfmaq_f64(z9, minus, vld1q_f64(row + 18));
270 z10 = vfmaq_f64(z10, minus, vld1q_f64(row + 20));
271 z11 = vfmaq_f64(z11, minus, vld1q_f64(row + 22));
272 z12 = vfmaq_f64(z12, minus, vld1q_f64(row + 24));
273 z13 = vfmaq_f64(z13, minus, vld1q_f64(row + 26));
274 z14 = vfmaq_f64(z14, minus, vld1q_f64(row + 28));
275 z15 = vfmaq_f64(z15, minus, vld1q_f64(row + 30));
276 }
277 vst1q_f64(y + i + 0, z0);
278 vst1q_f64(y + i + 2, z1);
279 vst1q_f64(y + i + 4, z2);
280 vst1q_f64(y + i + 6, z3);
281 vst1q_f64(y + i + 8, z4);
282 vst1q_f64(y + i + 10, z5);
283 vst1q_f64(y + i + 12, z6);
284 vst1q_f64(y + i + 14, z7);
285 vst1q_f64(y + i + 16, z8);
286 vst1q_f64(y + i + 18, z9);
287 vst1q_f64(y + i + 20, z10);
288 vst1q_f64(y + i + 22, z11);
289 vst1q_f64(y + i + 24, z12);
290 vst1q_f64(y + i + 26, z13);
291 vst1q_f64(y + i + 28, z14);
292 vst1q_f64(y + i + 30, z15);
293 }
294 for (; i + 16 <= count; i += 16) {
295 float64x2_t y0 {vld1q_f64(y + i + 0)};
296 float64x2_t y1 {vld1q_f64(y + i + 2)};
297 float64x2_t y2 {vld1q_f64(y + i + 4)};
298 float64x2_t y3 {vld1q_f64(y + i + 6)};
299 float64x2_t y4 {vld1q_f64(y + i + 8)};
300 float64x2_t y5 {vld1q_f64(y + i + 10)};
301 float64x2_t y6 {vld1q_f64(y + i + 12)};
302 float64x2_t y7 {vld1q_f64(y + i + 14)};
303 for (std::size_t t {0}; t < terms; ++t) {
304 const float64x2_t minus {vdupq_n_f64(-f[t])};
305 const double* const row {x + (t * ldx) + i};
306 y0 = vfmaq_f64(y0, minus, vld1q_f64(row + 0));
307 y1 = vfmaq_f64(y1, minus, vld1q_f64(row + 2));
308 y2 = vfmaq_f64(y2, minus, vld1q_f64(row + 4));
309 y3 = vfmaq_f64(y3, minus, vld1q_f64(row + 6));
310 y4 = vfmaq_f64(y4, minus, vld1q_f64(row + 8));
311 y5 = vfmaq_f64(y5, minus, vld1q_f64(row + 10));
312 y6 = vfmaq_f64(y6, minus, vld1q_f64(row + 12));
313 y7 = vfmaq_f64(y7, minus, vld1q_f64(row + 14));
314 }
315 vst1q_f64(y + i + 0, y0);
316 vst1q_f64(y + i + 2, y1);
317 vst1q_f64(y + i + 4, y2);
318 vst1q_f64(y + i + 6, y3);
319 vst1q_f64(y + i + 8, y4);
320 vst1q_f64(y + i + 10, y5);
321 vst1q_f64(y + i + 12, y6);
322 vst1q_f64(y + i + 14, y7);
323 }
324 for (; i + 8 <= count; i += 8) {
325 float64x2_t y0 {vld1q_f64(y + i)};
326 float64x2_t y1 {vld1q_f64(y + i + 2)};
327 float64x2_t y2 {vld1q_f64(y + i + 4)};
328 float64x2_t y3 {vld1q_f64(y + i + 6)};
329 for (std::size_t t {0}; t < terms; ++t) {
330 const float64x2_t minus {vdupq_n_f64(-f[t])};
331 const double* const row {x + (t * ldx) + i};
332 y0 = vfmaq_f64(y0, minus, vld1q_f64(row));
333 y1 = vfmaq_f64(y1, minus, vld1q_f64(row + 2));
334 y2 = vfmaq_f64(y2, minus, vld1q_f64(row + 4));
335 y3 = vfmaq_f64(y3, minus, vld1q_f64(row + 6));
336 }
337 vst1q_f64(y + i, y0);
338 vst1q_f64(y + i + 2, y1);
339 vst1q_f64(y + i + 4, y2);
340 vst1q_f64(y + i + 6, y3);
341 }
342 for (; i + 4 <= count; i += 4) { // two chains interleaved, each waiting on its own last multiply-add
343 float64x2_t y0 {vld1q_f64(y + i)};
344 float64x2_t y1 {vld1q_f64(y + i + 2)};
345 for (std::size_t t {0}; t < terms; ++t) {
346 const float64x2_t minus {vdupq_n_f64(-f[t])};
347 const double* const row {x + (t * ldx) + i};
348 y0 = vfmaq_f64(y0, minus, vld1q_f64(row));
349 y1 = vfmaq_f64(y1, minus, vld1q_f64(row + 2));
350 }
351 vst1q_f64(y + i, y0);
352 vst1q_f64(y + i + 2, y1);
353 }
354 for (; i + 2 <= count; i += 2) {
355 float64x2_t y0 {vld1q_f64(y + i)};
356 for (std::size_t t {0}; t < terms; ++t) {
357 y0 = vfmaq_f64(y0, vdupq_n_f64(-f[t]), vld1q_f64(x + (t * ldx) + i));
358 }
359 vst1q_f64(y + i, y0);
360 }
361 for (; i < count; ++i) {
362 double sum {y[i]};
363 for (std::size_t t {0}; t < terms; ++t) {
364 sum = std::fma(-f[t], x[(t * ldx) + i], sum);
365 }
366 y[i] = sum;
367 }
368 }
369
373 inline void neon_subtract_rows(double* y,
374 std::size_t ldy,
375 const double* f,
376 std::size_t ldf,
377 const double* x,
378 std::size_t ldx,
379 std::size_t terms,
380 std::size_t count) noexcept
381 {
382 static_assert(product_rows == 8, "eight rows, sixteen named registers");
383 std::size_t i {0};
384 for (; i + 4 <= count; i += 4) {
385 float64x2_t r0a {vld1q_f64(y + (0 * ldy) + i)};
386 float64x2_t r0b {vld1q_f64(y + (0 * ldy) + i + 2)};
387 float64x2_t r1a {vld1q_f64(y + (1 * ldy) + i)};
388 float64x2_t r1b {vld1q_f64(y + (1 * ldy) + i + 2)};
389 float64x2_t r2a {vld1q_f64(y + (2 * ldy) + i)};
390 float64x2_t r2b {vld1q_f64(y + (2 * ldy) + i + 2)};
391 float64x2_t r3a {vld1q_f64(y + (3 * ldy) + i)};
392 float64x2_t r3b {vld1q_f64(y + (3 * ldy) + i + 2)};
393 float64x2_t r4a {vld1q_f64(y + (4 * ldy) + i)};
394 float64x2_t r4b {vld1q_f64(y + (4 * ldy) + i + 2)};
395 float64x2_t r5a {vld1q_f64(y + (5 * ldy) + i)};
396 float64x2_t r5b {vld1q_f64(y + (5 * ldy) + i + 2)};
397 float64x2_t r6a {vld1q_f64(y + (6 * ldy) + i)};
398 float64x2_t r6b {vld1q_f64(y + (6 * ldy) + i + 2)};
399 float64x2_t r7a {vld1q_f64(y + (7 * ldy) + i)};
400 float64x2_t r7b {vld1q_f64(y + (7 * ldy) + i + 2)};
401 for (std::size_t t {0}; t < terms; ++t) {
402 const float64x2_t low {vld1q_f64(x + (t * ldx) + i)};
403 const float64x2_t high {vld1q_f64(x + (t * ldx) + i + 2)};
404 const float64x2_t m0 {vdupq_n_f64(-f[(0 * ldf) + t])};
405 r0a = vfmaq_f64(r0a, m0, low);
406 r0b = vfmaq_f64(r0b, m0, high);
407 const float64x2_t m1 {vdupq_n_f64(-f[(1 * ldf) + t])};
408 r1a = vfmaq_f64(r1a, m1, low);
409 r1b = vfmaq_f64(r1b, m1, high);
410 const float64x2_t m2 {vdupq_n_f64(-f[(2 * ldf) + t])};
411 r2a = vfmaq_f64(r2a, m2, low);
412 r2b = vfmaq_f64(r2b, m2, high);
413 const float64x2_t m3 {vdupq_n_f64(-f[(3 * ldf) + t])};
414 r3a = vfmaq_f64(r3a, m3, low);
415 r3b = vfmaq_f64(r3b, m3, high);
416 const float64x2_t m4 {vdupq_n_f64(-f[(4 * ldf) + t])};
417 r4a = vfmaq_f64(r4a, m4, low);
418 r4b = vfmaq_f64(r4b, m4, high);
419 const float64x2_t m5 {vdupq_n_f64(-f[(5 * ldf) + t])};
420 r5a = vfmaq_f64(r5a, m5, low);
421 r5b = vfmaq_f64(r5b, m5, high);
422 const float64x2_t m6 {vdupq_n_f64(-f[(6 * ldf) + t])};
423 r6a = vfmaq_f64(r6a, m6, low);
424 r6b = vfmaq_f64(r6b, m6, high);
425 const float64x2_t m7 {vdupq_n_f64(-f[(7 * ldf) + t])};
426 r7a = vfmaq_f64(r7a, m7, low);
427 r7b = vfmaq_f64(r7b, m7, high);
428 }
429 vst1q_f64(y + (0 * ldy) + i, r0a);
430 vst1q_f64(y + (0 * ldy) + i + 2, r0b);
431 vst1q_f64(y + (1 * ldy) + i, r1a);
432 vst1q_f64(y + (1 * ldy) + i + 2, r1b);
433 vst1q_f64(y + (2 * ldy) + i, r2a);
434 vst1q_f64(y + (2 * ldy) + i + 2, r2b);
435 vst1q_f64(y + (3 * ldy) + i, r3a);
436 vst1q_f64(y + (3 * ldy) + i + 2, r3b);
437 vst1q_f64(y + (4 * ldy) + i, r4a);
438 vst1q_f64(y + (4 * ldy) + i + 2, r4b);
439 vst1q_f64(y + (5 * ldy) + i, r5a);
440 vst1q_f64(y + (5 * ldy) + i + 2, r5b);
441 vst1q_f64(y + (6 * ldy) + i, r6a);
442 vst1q_f64(y + (6 * ldy) + i + 2, r6b);
443 vst1q_f64(y + (7 * ldy) + i, r7a);
444 vst1q_f64(y + (7 * ldy) + i + 2, r7b);
445 }
446 for (; i < count; ++i) { // the last columns one at a time, the eight rows' chains interleaved
447 double s0 {y[(0 * ldy) + i]};
448 double s1 {y[(1 * ldy) + i]};
449 double s2 {y[(2 * ldy) + i]};
450 double s3 {y[(3 * ldy) + i]};
451 double s4 {y[(4 * ldy) + i]};
452 double s5 {y[(5 * ldy) + i]};
453 double s6 {y[(6 * ldy) + i]};
454 double s7 {y[(7 * ldy) + i]};
455 for (std::size_t t {0}; t < terms; ++t) {
456 const double v {x[(t * ldx) + i]};
457 s0 = std::fma(-f[(0 * ldf) + t], v, s0);
458 s1 = std::fma(-f[(1 * ldf) + t], v, s1);
459 s2 = std::fma(-f[(2 * ldf) + t], v, s2);
460 s3 = std::fma(-f[(3 * ldf) + t], v, s3);
461 s4 = std::fma(-f[(4 * ldf) + t], v, s4);
462 s5 = std::fma(-f[(5 * ldf) + t], v, s5);
463 s6 = std::fma(-f[(6 * ldf) + t], v, s6);
464 s7 = std::fma(-f[(7 * ldf) + t], v, s7);
465 }
466 y[(0 * ldy) + i] = s0;
467 y[(1 * ldy) + i] = s1;
468 y[(2 * ldy) + i] = s2;
469 y[(3 * ldy) + i] = s3;
470 y[(4 * ldy) + i] = s4;
471 y[(5 * ldy) + i] = s5;
472 y[(6 * ldy) + i] = s6;
473 y[(7 * ldy) + i] = s7;
474 }
475 }
476
477#endif
478
485 inline void subtract_product(double* y,
486 const double& f,
487 const double* x,
488 std::size_t count)
489 {
490 const double factor {f}; // read once: it may be an element of y's array
491#ifdef NUMERICAL_X86_COMPILED
493 if (path == gemm_path::avx512) {
494 avx512_scaled_add(y, x, -factor, count);
495 return;
496 }
497 if (path == gemm_path::avx2) {
498 avx2_scaled_add(y, x, -factor, count);
499 return;
500 }
501#endif
502#ifdef NUMERICAL_NEON_COMPILED
503 if (rows_fuse()) { // the NEON or the AMX path: the cap alone answers, one load
504 neon_scaled_add(y, x, -factor, count);
505 return;
506 }
507#endif
508 subtract_product<double>(y, factor, x, count);
509 }
510
512 inline void subtract_product(double* y,
513 const double* x,
514 const double& f,
515 std::size_t count)
516 {
517 subtract_product(y, f, x, count);
518 }
519
522 inline void subtract_products(double* y,
523 const double* f,
524 const double* x,
525 std::size_t ldx,
526 std::size_t terms,
527 std::size_t count)
528 {
529#ifdef NUMERICAL_X86_COMPILED
531 if (path == gemm_path::avx512) {
532 avx512_subtract_products(y, f, x, ldx, terms, count);
533 return;
534 }
535 if (path == gemm_path::avx2) {
536 avx2_subtract_products(y, f, x, ldx, terms, count);
537 return;
538 }
539#endif
540#ifdef NUMERICAL_NEON_COMPILED
541 if (rows_fuse()) { // the NEON or the AMX path: the cap alone answers, one load
542 neon_subtract_products(y, f, x, ldx, terms, count);
543 return;
544 }
545#endif
546 subtract_products<double>(y, f, x, ldx, terms, count);
547 }
548
551 inline void subtract_rows(double* y,
552 std::size_t ldy,
553 const double* f,
554 std::size_t ldf,
555 const double* x,
556 std::size_t ldx,
557 std::size_t terms,
558 std::size_t count)
559 {
560#ifdef NUMERICAL_X86_COMPILED
562 if (path == gemm_path::avx512) {
563 avx512_subtract_rows(y, ldy, f, ldf, x, ldx, terms, count);
564 return;
565 }
566 if (path == gemm_path::avx2) {
567 avx2_subtract_rows(y, ldy, f, ldf, x, ldx, terms, count);
568 return;
569 }
570#endif
571#ifdef NUMERICAL_NEON_COMPILED
572 if (rows_fuse()) { // the NEON or the AMX path: the cap alone answers, one load
573 neon_subtract_rows(y, ldy, f, ldf, x, ldx, terms, count);
574 return;
575 }
576#endif
577 subtract_rows<double>(y, ldy, f, ldf, x, ldx, terms, count);
578 }
579
586 inline void subtract_products(std::complex<double>* y,
587 const std::complex<double>* f,
588 const std::complex<double>* x,
589 std::size_t ldx,
590 std::size_t terms,
591 std::size_t count)
592 {
593#ifdef NUMERICAL_X86_COMPILED
594 // A complex number's parts are an array of two (the standard's layout guarantee): addressed as doubles.
597 auto* const into {reinterpret_cast<double*>(y)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
598 const auto* const scales {reinterpret_cast<const double*>(f)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
599 const auto* const rows {reinterpret_cast<const double*>(x)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
600 if (path == gemm_path::avx512) {
601 avx512_subtract_complex_products(into, scales, rows, ldx, terms, count);
602 }
603 else {
604 avx2_subtract_complex_products(into, scales, rows, ldx, terms, count);
605 }
606 return;
607 }
608#endif
609 subtract_products<std::complex<double>>(y, f, x, ldx, terms, count);
610 }
611
614 inline void subtract_product(std::complex<double>* y,
615 const std::complex<double>& f,
616 const std::complex<double>* x,
617 std::size_t count)
618 {
619 const std::complex<double> factor {f}; // read once: it may be an element of y's array
620#ifdef NUMERICAL_X86_COMPILED
623 subtract_products(y, &factor, x, 0, 1, count);
624 return;
625 }
626#endif
627 subtract_product<std::complex<double>>(y, factor, x, count);
628 }
629
631 inline void subtract_product(std::complex<double>* y,
632 const std::complex<double>* x,
633 const std::complex<double>& f,
634 std::size_t count)
635 {
636 const std::complex<double> factor {f}; // read once: it may be an element of y's array
637#ifdef NUMERICAL_X86_COMPILED
640 subtract_products(y, &factor, x, 0, 1, count);
641 return;
642 }
643#endif
644 subtract_product<std::complex<double>>(y, x, factor, count);
645 }
646
648 inline void add_product(double* y,
649 const double& f,
650 const double* x,
651 std::size_t count)
652 {
653 const double factor {f}; // read once: it may be an element of y's array
654#ifdef NUMERICAL_X86_COMPILED
656 if (path == gemm_path::avx512) {
657 avx512_scaled_add(y, x, factor, count);
658 return;
659 }
660 if (path == gemm_path::avx2) {
661 avx2_scaled_add(y, x, factor, count);
662 return;
663 }
664#endif
665#ifdef NUMERICAL_NEON_COMPILED
666 if (rows_fuse()) { // the NEON or the AMX path: the cap alone answers, one load
667 neon_scaled_add(y, x, factor, count);
668 return;
669 }
670#endif
671 add_product<double>(y, factor, x, count);
672 }
673
676 inline void add_product(std::complex<float>* y,
677 const std::complex<float>& f,
678 const std::complex<float>* x,
679 std::size_t count)
680 {
681 const std::complex<float> factor {f}; // read once: it may be an element of y's array
682#ifdef NUMERICAL_X86_COMPILED
685 for (std::size_t i {avx2_scaled_add_c32(y, x, factor, count)}; i < count; ++i) {
686 complex_scaled_add(y[i], factor, x[i]);
687 }
688 return;
689 }
690#endif
691 add_product<std::complex<float>>(y, factor, x, count);
692 }
693
695 inline void add_product(std::complex<double>* y,
696 const std::complex<double>& f,
697 const std::complex<double>* x,
698 std::size_t count)
699 {
700 const std::complex<double> factor {f}; // read once: it may be an element of y's array
701#ifdef NUMERICAL_X86_COMPILED
704 for (std::size_t i {avx2_scaled_add_c64(y, x, factor, count)}; i < count; ++i) {
705 complex_scaled_add(y[i], factor, x[i]);
706 }
707 return;
708 }
709#endif
710 add_product<std::complex<double>>(y, factor, x, count);
711 }
712
715 [[nodiscard]] inline double dot_product(const double* x,
716 const double* y,
717 std::size_t count)
718 {
719#ifdef NUMERICAL_X86_COMPILED
721 if (path == gemm_path::avx512) {
722 return avx512_dot(x, y, count);
723 }
724 if (path == gemm_path::avx2) {
725 return avx2_dot(x, y, count);
726 }
727#endif
728 return dot_product<double>(x, y, count);
729 }
730
733 [[nodiscard]] inline float dot_product(const float* x,
734 const float* y,
735 std::size_t count)
736 {
737#ifdef NUMERICAL_X86_COMPILED
739 if (path == gemm_path::avx512) {
740 return avx512_dot(x, y, count);
741 }
742 if (path == gemm_path::avx2) {
743 return avx2_dot(x, y, count);
744 }
745#endif
746 return dot_product<float>(x, y, count);
747 }
748
752 template <split_complex C>
754 const C* y,
755 std::size_t count)
756 {
757#ifdef NUMERICAL_X86_COMPILED
759 return avx2_complex_dot(x, y, count);
760 }
761#endif
762 return dot_product<C>(x, y, count); // the native vectors' (the overloads below come here)
763 }
764
766 [[nodiscard]] inline std::complex<double> dot_product(const std::complex<double>* x,
767 const std::complex<double>* y,
768 std::size_t count)
769 {
770 return complex_dot_product(x, y, count);
771 }
772
774 [[nodiscard]] inline std::complex<float> dot_product(const std::complex<float>* x,
775 const std::complex<float>* y,
776 std::size_t count)
777 {
778 return complex_dot_product(x, y, count);
779 }
780
782 template <split_complex C>
783 void complex_row_dots(const C* m,
784 std::size_t ld,
785 std::size_t rows,
786 const C* x,
787 std::size_t count,
788 C* out)
789 {
790#ifdef NUMERICAL_X86_COMPILED
792 avx2_complex_row_dots(m, ld, rows, x, count, out);
793 return;
794 }
795#endif
796 for (std::size_t r {0}; r < rows; ++r) {
797 out[r] = complex_dot_product(m + (r * ld), x, count);
798 }
799 }
800
802 inline void row_dots(const std::complex<double>* m,
803 std::size_t ld,
804 std::size_t rows,
805 const std::complex<double>* x,
806 std::size_t count,
807 std::complex<double>* out)
808 {
809 complex_row_dots(m, ld, rows, x, count, out);
810 }
811
813 inline void row_dots(const std::complex<float>* m,
814 std::size_t ld,
815 std::size_t rows,
816 const std::complex<float>* x,
817 std::size_t count,
818 std::complex<float>* out)
819 {
820 complex_row_dots(m, ld, rows, x, count, out);
821 }
822
825 inline void row_dots(const double* m,
826 std::size_t ld,
827 std::size_t rows,
828 const double* x,
829 std::size_t count,
830 double* out)
831 {
832#ifdef NUMERICAL_X86_COMPILED
834 if (path == gemm_path::avx512) {
835 interleaved_row_dots<double, 32, avx512_dot_blocks>(m, ld, rows, x, count, out);
836 return;
837 }
838 if (path == gemm_path::avx2) {
839 interleaved_row_dots<double, 32, avx2_dot_blocks>(m, ld, rows, x, count, out);
840 return;
841 }
842#endif
843 row_dots<double>(m, ld, rows, x, count, out);
844 }
845
848 inline void row_dots(const float* m,
849 std::size_t ld,
850 std::size_t rows,
851 const float* x,
852 std::size_t count,
853 float* out)
854 {
855#ifdef NUMERICAL_X86_COMPILED
857 if (path == gemm_path::avx512) {
858 interleaved_row_dots<float, 64, avx512_dot_blocks>(m, ld, rows, x, count, out);
859 return;
860 }
861 if (path == gemm_path::avx2) {
862 interleaved_row_dots<float, 64, avx2_dot_blocks>(m, ld, rows, x, count, out);
863 return;
864 }
865#endif
866 row_dots<float>(m, ld, rows, x, count, out);
867 }
868
870 inline constexpr std::size_t gemm_mr {4};
871
874 template <typename T>
875 inline constexpr std::size_t gemm_nr {sizeof(T) >= 16 ? std::size_t {6} : std::size_t {96} / sizeof(T)};
876
878 inline constexpr std::size_t gemm_kc {256};
879
881 inline constexpr std::size_t gemm_mc {64};
882
884 inline constexpr std::size_t gemm_slivers_per_task {4};
885
888 inline constexpr std::size_t tasks_per_thread {4};
889
893 inline constexpr std::size_t gemm_kernel_speedup {16};
894
896 template <typename T>
897 inline constexpr std::size_t gemm_nc {gemm_nr<T> * 128};
898
906#ifdef NUMERICAL_X86_COMPILED
907 inline constexpr std::size_t gemm_packed_threshold {std::size_t {1} << 11U};
908#else
909 inline constexpr std::size_t gemm_packed_threshold {std::size_t {1} << 13U};
910#endif
911
915 inline constexpr std::size_t gemm_min_depth {8};
916
917 template <typename T>
918 using gemm_tile = std::array<T, gemm_mr * gemm_nr<T>>;
919
928 template <typename T>
929 void gemm_kernel(std::size_t kc,
930 const T* ap,
931 const T* bp,
932 std::size_t ldb,
933 gemm_tile<T>& tile)
934 {
935 constexpr std::size_t nr {gemm_nr<T>};
936 for (std::size_t p {0}; p < kc; ++p) {
937 for (std::size_t i {0}; i < gemm_mr; ++i) {
938 const T ai {ap[(p * gemm_mr) + i]};
939 for (std::size_t j {0}; j < nr; ++j) {
940 tile[(i * nr) + j] += ai * bp[(p * ldb) + j];
941 }
942 }
943 }
944 }
945
946#if defined(__GNUC__)
953 template <typename T>
954 requires std::is_same_v<T, float> || std::is_same_v<T, double> || std::is_same_v<T, std::uint64_t>
955 void gemm_kernel(std::size_t kc,
956 const T* ap,
957 const T* bp,
958 std::size_t ldb,
959 gemm_tile<T>& tile)
960 {
961 using lane = typename native_vector<T>::type;
962 constexpr std::size_t nr {gemm_nr<T>};
963 constexpr std::size_t lanes {nr * sizeof(T) / native_vector_bytes};
964 // The tile moves in and out one vector at a time: a single copy of the whole array would leave it in
965 // memory under GCC, which then keeps no accumulator in a register (7.4 GFLOP/s instead of 11.8 on one
966 // x86-64 core).
967 std::array<lane, gemm_mr * lanes> acc {};
968# pragma GCC unroll 32
969 for (std::size_t q {0}; q < acc.size(); ++q) {
970 lane v;
971 std::memcpy(&v, tile.data() + (q * (native_vector_bytes / sizeof(T))), sizeof(v));
972 acc[q] = v;
973 }
974 // Unrolled, so that every accumulator has a fixed index and stays in a register: GCC at -O2 does not
975 // unroll on its own, and then reads and writes the tile through memory.
976 for (std::size_t p {0}; p < kc; ++p) {
977 std::array<lane, lanes> b {};
978 std::memcpy(b.data(), bp + (p * ldb), sizeof(b));
979# pragma GCC unroll 8
980 for (std::size_t i {0}; i < gemm_mr; ++i) {
981 const T ai {ap[(p * gemm_mr) + i]};
982# pragma GCC unroll 8
983 for (std::size_t j {0}; j < lanes; ++j) {
984 acc[(i * lanes) + j] += ai * b[j];
985 }
986 }
987 }
988# pragma GCC unroll 32
989 for (std::size_t q {0}; q < acc.size(); ++q) {
990 const lane v {acc[q]};
991 std::memcpy(tile.data() + (q * (native_vector_bytes / sizeof(T))), &v, sizeof(v));
992 }
993 }
994
995#endif
996
997#ifdef NUMERICAL_NEON_COMPILED
1005 inline void neon_kernel(std::size_t kc,
1006 const double* ap,
1007 const double* bp,
1008 std::size_t ldb,
1009 gemm_tile<double>& tile) noexcept
1010 {
1011 static_assert(gemm_mr == 4 && gemm_nr<double> == 12, "two rows of A a vector, twelve columns six vectors");
1012 constexpr std::size_t lanes {6};
1013 std::array<float64x2_t, 4 * lanes> acc {};
1015 for (std::size_t q {0}; q < acc.size(); ++q) {
1016 acc[q] = vld1q_f64(tile.data() + (2 * q));
1017 }
1018 for (std::size_t p {0}; p < kc; ++p) {
1019 const float64x2_t top {vld1q_f64(ap + (p * gemm_mr))};
1020 const float64x2_t bottom {vld1q_f64(ap + (p * gemm_mr) + 2)};
1022 for (std::size_t j {0}; j < lanes; ++j) {
1023 const float64x2_t b {vld1q_f64(bp + (p * ldb) + (2 * j))};
1024 acc[j] = vfmaq_laneq_f64(acc[j], b, top, 0);
1025 acc[lanes + j] = vfmaq_laneq_f64(acc[lanes + j], b, top, 1);
1026 acc[(2 * lanes) + j] = vfmaq_laneq_f64(acc[(2 * lanes) + j], b, bottom, 0);
1027 acc[(3 * lanes) + j] = vfmaq_laneq_f64(acc[(3 * lanes) + j], b, bottom, 1);
1028 }
1029 }
1031 for (std::size_t q {0}; q < acc.size(); ++q) {
1032 vst1q_f64(tile.data() + (2 * q), acc[q]);
1033 }
1034 }
1035
1038 inline void neon_kernel(std::size_t kc,
1039 const float* ap,
1040 const float* bp,
1041 std::size_t ldb,
1042 gemm_tile<float>& tile) noexcept
1043 {
1044 static_assert(gemm_mr == 4 && gemm_nr<float> == 24, "four rows of A a vector, 24 columns six vectors");
1045 constexpr std::size_t lanes {6};
1046 std::array<float32x4_t, 4 * lanes> acc {};
1048 for (std::size_t q {0}; q < acc.size(); ++q) {
1049 acc[q] = vld1q_f32(tile.data() + (4 * q));
1050 }
1051 for (std::size_t p {0}; p < kc; ++p) {
1052 const float32x4_t column {vld1q_f32(ap + (p * gemm_mr))};
1054 for (std::size_t j {0}; j < lanes; ++j) {
1055 const float32x4_t b {vld1q_f32(bp + (p * ldb) + (4 * j))};
1056 acc[j] = vfmaq_laneq_f32(acc[j], b, column, 0);
1057 acc[lanes + j] = vfmaq_laneq_f32(acc[lanes + j], b, column, 1);
1058 acc[(2 * lanes) + j] = vfmaq_laneq_f32(acc[(2 * lanes) + j], b, column, 2);
1059 acc[(3 * lanes) + j] = vfmaq_laneq_f32(acc[(3 * lanes) + j], b, column, 3);
1060 }
1061 }
1063 for (std::size_t q {0}; q < acc.size(); ++q) {
1064 vst1q_f32(tile.data() + (4 * q), acc[q]);
1065 }
1066 }
1067
1068#endif
1069
1074 template <std::size_t Mr,
1075 typename T>
1076 void gemm_pack_a(const T* a,
1077 std::size_t lda,
1078 std::size_t rows,
1079 std::size_t kc,
1080 T* out)
1081 {
1082 for (std::size_t ir {0}; ir < rows; ir += Mr) {
1083 const std::size_t mr {std::min(Mr, rows - ir)};
1084 T* sliver {out + (ir * kc)};
1085 if (mr == Mr) { // a whole sliver, its loop free of the padding's test (M1 Pro, order 64 for the AMX: 1.21 → 0.96 µs)
1086 for (std::size_t p {0}; p < kc; ++p) {
1087 for (std::size_t i {0}; i < Mr; ++i) {
1088 sliver[(p * Mr) + i] = a[((ir + i) * lda) + p];
1089 }
1090 }
1091 continue;
1092 }
1093 for (std::size_t p {0}; p < kc; ++p) {
1094 for (std::size_t i {0}; i < Mr; ++i) {
1095 sliver[(p * Mr) + i] = i < mr ? a[((ir + i) * lda) + p] : T {};
1096 }
1097 }
1098 }
1099 }
1100
1105 template <std::size_t Nr,
1106 typename T>
1107 void gemm_pack_b(const T* b,
1108 std::size_t ldb,
1109 std::size_t kc,
1110 std::size_t cols,
1111 T* out)
1112 {
1113 // A sliver's row copied whole, then its padding: element by element under a test of the width, the loop took 0.48 ns
1114 // an element (one M1 Pro core, 512 × 32 floats: 7.9 µs, most of a thin product on the AMX).
1115 for (std::size_t jr {0}; jr < cols; jr += Nr) {
1116 const std::size_t width {std::min(Nr, cols - jr)};
1117 T* sliver {out + (jr * kc)};
1118 for (std::size_t p {0}; p < kc; ++p) {
1119 T* const row {sliver + (p * Nr)};
1120 std::copy_n(b + (p * ldb) + jr, width, row);
1121 std::fill(row + width, row + Nr, T {});
1122 }
1123 }
1124 }
1125
1130 template <std::size_t Nr,
1131 typename T>
1132 void gemm_pack_bt(const T* bt,
1133 std::size_t ldbt,
1134 std::size_t kc,
1135 std::size_t cols,
1136 T* out)
1137 {
1138 static_assert(Nr % 2 == 0, "a sliver's columns go by pairs");
1139 for (std::size_t jr {0}; jr < cols; jr += Nr) {
1140 const std::size_t width {std::min(Nr, cols - jr)};
1141 T* sliver {out + (jr * kc)};
1142 if (width == Nr) {
1143 // Two rows of Bᵀ at once, each step two neighbours of a sliver's row: one column at a time wrote one element
1144 // per line of the sliver (order 112, twelve columns: 7.7 µs, by pairs of rows 4.8, by 2 × 2 blocks of doubles 3.9).
1145 for (std::size_t j {0}; j < Nr; j += 2) {
1146 const T* const first {bt + ((jr + j) * ldbt)};
1147 const T* const second {first + ldbt};
1148 std::size_t p {0};
1149 if constexpr (std::same_as<T, double>) {
1150#if defined(NUMERICAL_NEON_COMPILED)
1151 for (; p + 2 <= kc; p += 2) {
1152 const float64x2_t x {vld1q_f64(first + p)};
1153 const float64x2_t y {vld1q_f64(second + p)};
1154 vst1q_f64(sliver + (p * Nr) + j, vzip1q_f64(x, y));
1155 vst1q_f64(sliver + ((p + 1) * Nr) + j, vzip2q_f64(x, y));
1156 }
1157#elif defined(NUMERICAL_X86_COMPILED)
1158 for (; p + 2 <= kc; p += 2) {
1159 const __m128d x {_mm_loadu_pd(first + p)};
1160 const __m128d y {_mm_loadu_pd(second + p)};
1161 _mm_storeu_pd(sliver + (p * Nr) + j, _mm_unpacklo_pd(x, y));
1162 _mm_storeu_pd(sliver + ((p + 1) * Nr) + j, _mm_unpackhi_pd(x, y));
1163 }
1164#endif
1165 }
1166 for (; p < kc; ++p) {
1167 sliver[(p * Nr) + j] = first[p];
1168 sliver[(p * Nr) + j + 1] = second[p];
1169 }
1170 }
1171 continue;
1172 }
1173 for (std::size_t j {0}; j < Nr; ++j) {
1174 const T* const row {bt + ((jr + j) * ldbt)};
1175 for (std::size_t p {0}; p < kc; ++p) {
1176 sliver[(p * Nr) + j] = j < width ? row[p] : T {};
1177 }
1178 }
1179 }
1180 }
1181
1183 template <typename T>
1184 [[nodiscard]] const T* rows_in_place(const T* x,
1185 std::size_t ld,
1186 std::size_t& ld_out) noexcept
1187 {
1188 ld_out = ld;
1189 return x;
1190 }
1191
1194 template <typename T>
1195 [[nodiscard]] const T* rows_from_transpose(const T* x,
1196 std::size_t ld,
1197 std::size_t rows,
1198 std::size_t cols,
1200 std::size_t& ld_out)
1201 {
1202 scratch.resize(rows * cols);
1203 T* const out {scratch.data()};
1204 // By pieces (transpose_walk): along the rows of x, a power of two of bytes apart, the copy's rows evicted one
1205 // another (one arm64 core: 256 × 256 doubles 171 → 25 µs, 128 × 128 complex128 35 → 7.7; x86-64, GCC, 128 × 128
1206 // doubles 33 → 7.9, 64 × 64 complex128 9.1 → 3.2).
1207 transpose_walk(0, rows, 0, cols, transpose_piece(ld * sizeof(T), cols * sizeof(T)), [out, x, ld, cols](std::size_t i, std::size_t j) {
1208 out[(i * cols) + j] = x[(j * ld) + i];
1209 });
1210 ld_out = cols;
1211 return out;
1212 }
1213
1215 struct b_rows
1216 {
1218 template <std::size_t Nr,
1219 typename T>
1220 static void pack(const T* b,
1221 std::size_t ldb,
1222 std::size_t p0,
1223 std::size_t j0,
1224 std::size_t kc,
1225 std::size_t cols,
1226 T* out)
1227 {
1228 gemm_pack_b<Nr>(b + (p0 * ldb) + j0, ldb, kc, cols, out);
1229 }
1230
1232 template <typename Kernel>
1233 [[nodiscard]] static constexpr bool in_place(std::size_t ldb) noexcept
1234 {
1235 return Kernel::reads_b_in_place(ldb);
1236 }
1237
1239 template <typename T>
1240 [[nodiscard]] static const T* rows_of(const T* x,
1241 std::size_t ld,
1242 std::size_t /*rows*/,
1243 std::size_t /*cols*/,
1244 buffer<T>& /*scratch*/,
1245 std::size_t& ld_out)
1246 {
1247 return rows_in_place(x, ld, ld_out);
1248 }
1249
1251 template <typename T>
1252 [[nodiscard]] static const T& at(const T* b,
1253 std::size_t ldb,
1254 std::size_t p,
1255 std::size_t j) noexcept
1256 {
1257 return b[(p * ldb) + j];
1258 }
1259 };
1260
1263 {
1266 template <std::size_t Nr,
1267 typename T>
1268 static void pack(const T* bt,
1269 std::size_t ldbt,
1270 std::size_t p0,
1271 std::size_t j0,
1272 std::size_t kc,
1273 std::size_t cols,
1274 T* out)
1275 {
1276 gemm_pack_bt<Nr>(bt + (j0 * ldbt) + p0, ldbt, kc, cols, out);
1277 }
1278
1280 template <typename Kernel>
1281 [[nodiscard]] static constexpr bool in_place(std::size_t /*ldbt*/) noexcept
1282 {
1283 return false;
1284 }
1285
1288 template <typename T>
1289 [[nodiscard]] static const T* rows_of(const T* x,
1290 std::size_t ld,
1291 std::size_t rows,
1292 std::size_t cols,
1294 std::size_t& ld_out)
1295 {
1296 return rows_from_transpose(x, ld, rows, cols, scratch, ld_out);
1297 }
1298
1300 template <typename T>
1301 [[nodiscard]] static const T& at(const T* bt,
1302 std::size_t ldbt,
1303 std::size_t p,
1304 std::size_t j) noexcept
1305 {
1306 return bt[(j * ldbt) + p];
1307 }
1308 };
1309
1315 template <std::size_t Mr,
1316 typename T>
1317 void gemm_pack_at(const T* at,
1318 std::size_t ldat,
1319 std::size_t rows,
1320 std::size_t kc,
1321 T* out)
1322 {
1323 for (std::size_t ir {0}; ir < rows; ir += Mr) {
1324 const std::size_t mr {std::min(Mr, rows - ir)};
1325 T* sliver {out + (ir * kc)};
1326 for (std::size_t p {0}; p < kc; ++p) {
1327 const T* const column {at + (p * ldat) + ir};
1328 for (std::size_t i {0}; i < Mr; ++i) {
1329 sliver[(p * Mr) + i] = i < mr ? column[i] : T {};
1330 }
1331 }
1332 }
1333 }
1334
1336 struct a_rows
1337 {
1339 template <typename Kernel,
1340 typename T>
1341 static void pack(const T* a,
1342 std::size_t lda,
1343 std::size_t i0,
1344 std::size_t p0,
1345 std::size_t rows,
1346 std::size_t kc,
1347 T* out)
1348 {
1349 Kernel::pack_a(a + (i0 * lda) + p0, lda, rows, kc, out);
1350 }
1351
1352#ifdef NUMERICAL_X86_COMPILED // the complex tiles that read A in place
1355 template <typename T>
1356 [[nodiscard]] static const T* element(const T* a,
1357 std::size_t lda,
1358 std::size_t i,
1359 std::size_t p) noexcept
1360 {
1361 return a + (i * lda) + p;
1362 }
1363
1365 [[nodiscard]] static constexpr std::size_t step(std::size_t /*lda*/) noexcept
1366 {
1367 return 1;
1368 }
1369
1371 [[nodiscard]] static constexpr std::size_t across(std::size_t lda) noexcept
1372 {
1373 return lda;
1374 }
1375
1376#endif
1377
1379 template <typename T>
1380 [[nodiscard]] static const T* rows_of(const T* a,
1381 std::size_t lda,
1382 std::size_t /*rows*/,
1383 std::size_t /*cols*/,
1384 buffer<T>& /*scratch*/,
1385 std::size_t& ld_out)
1386 {
1387 return rows_in_place(a, lda, ld_out);
1388 }
1389
1391 template <typename T>
1392 [[nodiscard]] static const T& at(const T* a,
1393 std::size_t lda,
1394 std::size_t i,
1395 std::size_t p) noexcept
1396 {
1397 return a[(i * lda) + p];
1398 }
1399 };
1400
1403 {
1406 template <typename Kernel,
1407 typename T>
1408 static void pack(const T* at,
1409 std::size_t ldat,
1410 std::size_t i0,
1411 std::size_t p0,
1412 std::size_t rows,
1413 std::size_t kc,
1414 T* out)
1415 {
1416 gemm_pack_at<Kernel::mr>(at + (p0 * ldat) + i0, ldat, rows, kc, out);
1417 }
1418
1419#ifdef NUMERICAL_X86_COMPILED // the complex tiles that read A in place
1421 template <typename T>
1422 [[nodiscard]] static const T* element(const T* at,
1423 std::size_t ldat,
1424 std::size_t i,
1425 std::size_t p) noexcept
1426 {
1427 return at + (p * ldat) + i;
1428 }
1429
1431 [[nodiscard]] static constexpr std::size_t step(std::size_t ldat) noexcept
1432 {
1433 return ldat;
1434 }
1435
1437 [[nodiscard]] static constexpr std::size_t across(std::size_t /*ldat*/) noexcept
1438 {
1439 return 1;
1440 }
1441
1442#endif
1443
1445 template <typename T>
1446 [[nodiscard]] static const T* rows_of(const T* at,
1447 std::size_t ldat,
1448 std::size_t rows,
1449 std::size_t cols,
1451 std::size_t& ld_out)
1452 {
1453 return rows_from_transpose(at, ldat, rows, cols, scratch, ld_out);
1454 }
1455
1457 template <typename T>
1458 [[nodiscard]] static const T& at(const T* at_,
1459 std::size_t ldat,
1460 std::size_t i,
1461 std::size_t p) noexcept
1462 {
1463 return at_[(p * ldat) + i];
1464 }
1465 };
1466
1471 template <typename T>
1473 {
1474 static constexpr std::size_t mr {gemm_mr};
1475 static constexpr std::size_t nr {gemm_nr<T>};
1476 static constexpr std::size_t depth {gemm_kc};
1477 static constexpr std::size_t block_rows {gemm_mc};
1482 static constexpr std::size_t b_in_place_rows {mr};
1483 static constexpr std::size_t speedup {gemm_kernel_speedup};
1484 static constexpr std::size_t serial_below {1};
1485
1487 struct session
1488 {};
1489
1492 static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
1493 {
1494 return ldb <= nr;
1495 }
1496
1498 static void pack_a(const T* a,
1499 std::size_t lda,
1500 std::size_t rows,
1501 std::size_t kc,
1502 T* out)
1503 {
1504 gemm_pack_a<mr>(a, lda, rows, kc, out);
1505 }
1506
1508 static void tile(std::size_t kc,
1509 const T* ap,
1510 const T* bp,
1511 std::size_t ldb,
1512 T* c,
1513 std::size_t ldc,
1514 std::size_t rows,
1515 std::size_t cols)
1516 {
1518 accumulators.fill(T {});
1519 for (std::size_t i {0}; i < rows; ++i) {
1520 std::copy_n(c + (i * ldc), cols, accumulators.data() + (i * nr));
1521 }
1523 for (std::size_t i {0}; i < rows; ++i) {
1524 std::copy_n(accumulators.data() + (i * nr), cols, c + (i * ldc));
1525 }
1526 }
1527 };
1528
1529#ifdef NUMERICAL_NEON_COMPILED
1531 template <typename T>
1532 struct neon_tile_kernel : vector_tile_kernel<T>
1533 {
1537 NUMERICAL_ALWAYS_INLINE static void tile(std::size_t kc,
1538 const T* ap,
1539 const T* bp,
1540 std::size_t ldb,
1541 T* c,
1542 std::size_t ldc,
1543 std::size_t rows,
1544 std::size_t cols) noexcept
1545 {
1546 constexpr std::size_t nr {vector_tile_kernel<T>::nr};
1548 for (std::size_t i {0}; i < rows; ++i) {
1549 std::copy_n(c + (i * ldc), cols, accumulators.data() + (i * nr));
1550 }
1552 for (std::size_t i {0}; i < rows; ++i) {
1553 std::copy_n(accumulators.data() + (i * nr), cols, c + (i * ldc));
1554 }
1555 }
1556 };
1557#endif
1558
1559#ifdef NUMERICAL_AMX_COMPILED
1562 struct amx_tile_kernel
1563 {
1564 static constexpr std::size_t mr {amx_mr};
1565 static constexpr std::size_t nr {amx_nr};
1568 static constexpr std::size_t depth {512};
1569 static constexpr std::size_t block_rows {gemm_mc};
1572 static constexpr std::size_t b_in_place_rows {block_rows};
1573 static constexpr std::size_t speedup {gemm_kernel_speedup};
1578 static constexpr std::size_t serial_below {std::size_t {1} << 24};
1579 using session = amx_session;
1580
1582 static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
1583 {
1584 return ldb <= amx_in_place_stride;
1585 }
1586
1588 static constexpr bool partial_b_in_place {true};
1589
1591 static void pack_a(const double* a,
1592 std::size_t lda,
1593 std::size_t rows,
1594 std::size_t kc,
1595 double* out) noexcept
1596 {
1597 amx_pack_a(a, lda, rows, kc, out);
1598 }
1599
1601 static void tile(std::size_t kc,
1602 const double* ap,
1603 const double* bp,
1604 std::size_t ldb,
1605 double* c,
1606 std::size_t ldc,
1607 std::size_t rows,
1608 std::size_t cols) noexcept
1609 {
1610 amx_tile_f64(kc, ap, bp, ldb, c, ldc, rows, cols);
1611 }
1612 };
1613
1615 struct amx_tile_kernel_f32
1616 {
1617 static constexpr std::size_t mr {amx_f32_side};
1618 static constexpr std::size_t nr {amx_f32_side};
1619 static constexpr std::size_t depth {512};
1620 static constexpr std::size_t block_rows {gemm_mc};
1621 static constexpr std::size_t b_in_place_rows {block_rows};
1622 static constexpr std::size_t speedup {gemm_kernel_speedup};
1623 static constexpr std::size_t serial_below {std::size_t {1} << 24};
1624 using session = amx_session;
1625
1626 static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
1627 {
1628 return ldb <= 2 * amx_in_place_stride;
1629 }
1630
1632 static constexpr bool partial_b_in_place {true};
1633
1634 static void pack_a(const float* a,
1635 std::size_t lda,
1636 std::size_t rows,
1637 std::size_t kc,
1638 float* out) noexcept
1639 {
1640 amx_pack_a_f32(a, lda, rows, kc, out);
1641 }
1642
1643 static void tile(std::size_t kc,
1644 const float* ap,
1645 const float* bp,
1646 std::size_t ldb,
1647 float* c,
1648 std::size_t ldc,
1649 std::size_t rows,
1650 std::size_t cols) noexcept
1651 {
1652 amx_tile_f32(kc, ap, bp, ldb, c, ldc, rows, cols);
1653 }
1654 };
1655#endif
1656
1662#ifdef NUMERICAL_X86_COMPILED
1663 inline constexpr std::size_t narrow_least_depth {8};
1664#else
1665 inline constexpr std::size_t narrow_least_depth {2};
1666#endif
1667
1669 inline constexpr std::size_t narrow_columns {8};
1670
1680#if defined(NUMERICAL_X86_COMPILED) || !defined(__clang__)
1681 inline constexpr std::size_t gemm_rows_columns {24};
1682#else
1683 inline constexpr std::size_t gemm_rows_columns {8};
1684#endif
1685
1695 inline constexpr std::size_t gemm_rows_work {std::size_t {1} << 20U};
1696
1705#ifdef NUMERICAL_X86_COMPILED
1706 inline constexpr std::size_t gemm_rows_weight {1};
1707#else
1708 inline constexpr std::size_t gemm_rows_weight {2};
1709#endif
1710
1716 template <typename T>
1718 T a,
1719 T b) noexcept
1720 {
1721#if defined(__GNUC__) && !defined(__clang__)
1722 const T product {a * b};
1723 return sum + product;
1724#else
1725 return sum + (a * b);
1726#endif
1727 }
1728
1731 template <typename R>
1732 [[nodiscard]] inline std::complex<R> rounded_apart_add(std::complex<R> sum,
1733 std::complex<R> a,
1734 std::complex<R> b) noexcept
1735 {
1736 return sum + operator_times(a, b);
1737 }
1738
1742 inline constexpr std::size_t narrow_group {4};
1743
1749 inline constexpr std::size_t gemm_rows_share {std::size_t {1} << 19U};
1750
1758 [[nodiscard]] inline bool by_rows([[maybe_unused]] std::size_t m,
1759 std::size_t n,
1760 [[maybe_unused]] std::size_t k)
1761 {
1762#if defined(NUMERICAL_X86_COMPILED) || !defined(__clang__)
1763 if (n <= gemm_rows_columns || (m >= narrow_group && m * n * k <= gemm_rows_work)) {
1764 return true;
1765 }
1766 // The threads read only past the cheaper tests; a block of rows each at most.
1767 const std::size_t all {max_threads()};
1768 const std::size_t threads {std::min(all, m / narrow_group)};
1769 return m >= narrow_group && 2 * threads >= all && m * n * k <= gemm_rows_share * threads;
1770#else
1771 return n <= gemm_rows_columns; // Clang on arm64: the AMX's and NEON's packed tiles kept small products ahead
1772#endif
1773 }
1774
1778 enum class term_rounding : unsigned char
1779 {
1780 compiler,
1781 fused,
1782 x86_fused,
1783 };
1784
1786 template <term_rounding Mode>
1787 using rounding_is = std::integral_constant<term_rounding, Mode>;
1788
1790 template <typename T>
1792 T sum,
1793 T a,
1794 T b) noexcept
1795 {
1796 return rounded_apart_add(sum, a, b);
1797 }
1798
1800 template <typename T>
1802 T sum,
1803 T a,
1804 T b) noexcept
1805 {
1806 return std::fma(a, b, sum);
1807 }
1808
1813 enum class c_start : bool
1814 {
1815 kept,
1816 zero
1817 };
1818
1819#ifdef NUMERICAL_X86_COMPILED
1822 template <std::size_t R, std::size_t N, typename T>
1824 const T* a,
1825 std::size_t lda,
1826 const T* b,
1827 std::size_t ldb,
1828 T* c,
1829 std::size_t ldc,
1830 c_start start) noexcept
1831 {
1832 std::array<T, R * N> sums {}; // from zero where C holds nothing
1833 if (start == c_start::kept) {
1835 for (std::size_t e {0}; e < R * N; ++e) {
1836 sums[e] = c[((e / N) * ldc) + (e % N)];
1837 }
1838 }
1839 for (std::size_t p {0}; p < k; ++p) {
1841 for (std::size_t r {0}; r < R; ++r) {
1842 const T ap {a[(r * lda) + p]};
1844 for (std::size_t j {0}; j < N; ++j) {
1845 sums[(r * N) + j] = std::fma(ap, b[(p * ldb) + j], sums[(r * N) + j]);
1846 }
1847 }
1848 }
1850 for (std::size_t e {0}; e < R * N; ++e) {
1851 c[((e / N) * ldc) + (e % N)] = sums[e];
1852 }
1853 }
1854
1860 NUMERICAL_TARGET("avx2,fma") inline void plain_row_x86(std::size_t n,
1861 std::size_t k,
1862 const double* a,
1863 const double* b,
1864 std::size_t ldb,
1865 double* c,
1866 c_start start) noexcept
1867 {
1868 constexpr std::size_t width {4};
1869 const bool kept {start == c_start::kept};
1870 // A zero depth leaves C as it is: none of its elements read or written.
1871 if (k == 0 && kept) {
1872 return;
1873 }
1874 std::size_t j {0};
1875 for (; j + (4 * width) <= n; j += 4 * width) { // four vectors of columns at a time
1877 __m256d s1 {kept ? _mm256_loadu_pd(c + j + width) : _mm256_setzero_pd()};
1878 __m256d s2 {kept ? _mm256_loadu_pd(c + j + (2 * width)) : _mm256_setzero_pd()};
1879 __m256d s3 {kept ? _mm256_loadu_pd(c + j + (3 * width)) : _mm256_setzero_pd()};
1880 for (std::size_t p {0}; p < k; ++p) {
1881 const __m256d ap {_mm256_set1_pd(a[p])};
1882 const double * row {b + (p * ldb) + j};
1884 s1 = _mm256_fmadd_pd(ap, _mm256_loadu_pd(row + width), s1);
1885 s2 = _mm256_fmadd_pd(ap, _mm256_loadu_pd(row + (2 * width)), s2);
1886 s3 = _mm256_fmadd_pd(ap, _mm256_loadu_pd(row + (3 * width)), s3);
1887 }
1888 _mm256_storeu_pd(c + j, s0);
1889 _mm256_storeu_pd(c + j + width, s1);
1890 _mm256_storeu_pd(c + j + (2 * width), s2);
1891 _mm256_storeu_pd(c + j + (3 * width), s3);
1892 }
1893 for (; j + width <= n; j += width) {
1895 for (std::size_t p {0}; p < k; ++p) {
1896 sum = _mm256_fmadd_pd(_mm256_set1_pd(a[p]), _mm256_loadu_pd(b + (p * ldb) + j), sum);
1897 }
1898 _mm256_storeu_pd(c + j, sum);
1899 }
1900 for (; j < n; ++j) {
1901 double sum {kept ? c[j] : double {}};
1902 for (std::size_t p {0}; p < k; ++p) {
1903 sum = std::fma(a[p], b[(p * ldb) + j], sum);
1904 }
1905 c[j] = sum;
1906 }
1907 _mm256_zeroupper(); // the upper halves cleared: code past the row may be SSE alone
1908 }
1909
1911 NUMERICAL_TARGET("avx2,fma") inline void plain_row_x86(std::size_t n,
1912 std::size_t k,
1913 const float* a,
1914 const float* b,
1915 std::size_t ldb,
1916 float* c,
1917 c_start start) noexcept
1918 {
1919 constexpr std::size_t width {8};
1920 const bool kept {start == c_start::kept};
1921 // A zero depth leaves C as it is: none of its elements read or written.
1922 if (k == 0 && kept) {
1923 return;
1924 }
1925 std::size_t j {0};
1926 for (; j + (4 * width) <= n; j += 4 * width) { // four vectors of columns at a time
1928 __m256 s1 {kept ? _mm256_loadu_ps(c + j + width) : _mm256_setzero_ps()};
1929 __m256 s2 {kept ? _mm256_loadu_ps(c + j + (2 * width)) : _mm256_setzero_ps()};
1930 __m256 s3 {kept ? _mm256_loadu_ps(c + j + (3 * width)) : _mm256_setzero_ps()};
1931 for (std::size_t p {0}; p < k; ++p) {
1932 const __m256 ap {_mm256_set1_ps(a[p])};
1933 const float * row {b + (p * ldb) + j};
1935 s1 = _mm256_fmadd_ps(ap, _mm256_loadu_ps(row + width), s1);
1936 s2 = _mm256_fmadd_ps(ap, _mm256_loadu_ps(row + (2 * width)), s2);
1937 s3 = _mm256_fmadd_ps(ap, _mm256_loadu_ps(row + (3 * width)), s3);
1938 }
1939 _mm256_storeu_ps(c + j, s0);
1940 _mm256_storeu_ps(c + j + width, s1);
1941 _mm256_storeu_ps(c + j + (2 * width), s2);
1942 _mm256_storeu_ps(c + j + (3 * width), s3);
1943 }
1944 for (; j + width <= n; j += width) {
1946 for (std::size_t p {0}; p < k; ++p) {
1947 sum = _mm256_fmadd_ps(_mm256_set1_ps(a[p]), _mm256_loadu_ps(b + (p * ldb) + j), sum);
1948 }
1949 _mm256_storeu_ps(c + j, sum);
1950 }
1951 for (; j < n; ++j) {
1952 float sum {kept ? c[j] : float {}};
1953 for (std::size_t p {0}; p < k; ++p) {
1954 sum = std::fma(a[p], b[(p * ldb) + j], sum);
1955 }
1956 c[j] = sum;
1957 }
1958 _mm256_zeroupper(); // the upper halves cleared: code past the row may be SSE alone
1959 }
1960
1961#endif
1962
1963#ifdef NUMERICAL_X86_COMPILED
1971 template <std::size_t R, std::size_t V>
1972 NUMERICAL_TARGET("avx2") bool narrow_columns_complex_x86(std::size_t k,
1973 const std::complex<double>* a,
1974 std::size_t lda,
1975 const std::complex<double>* b,
1976 std::size_t ldb,
1977 std::complex<double>* c,
1978 std::size_t ldc,
1979 c_start start) noexcept
1980 {
1981 static_assert(V == 1 || V == 2, "two or four complex columns, two to a vector");
1982 std::array<ymm_register, R * V> sums {};
1984 auto* const cs {reinterpret_cast<double*>(c)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is the array of its two parts
1985 const auto* const as {reinterpret_cast<const double*>(a)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): as above
1986 const auto* const bs {reinterpret_cast<const double*>(b)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): as above
1988 for (std::size_t r {0}; r < R; ++r) {
1990 for (std::size_t v {0}; v < V; ++v) {
1991 sums[(r * V) + v].v = start == c_start::kept ? _mm256_loadu_pd(cs + (2 * ((r * ldc) + (2 * v)))) : _mm256_setzero_pd();
1992 }
1993 }
1994 for (std::size_t p {0}; p < k; ++p) {
1995 const double* const row {bs + (2 * p * ldb)};
1996 std::array<ymm_register, V> bw {};
1997 std::array<ymm_register, V> bx {};
1999 for (std::size_t v {0}; v < V; ++v) {
2000 bw[v].v = _mm256_loadu_pd(row + (4 * v));
2001 bx[v].v = _mm256_permute_pd(bw[v].v, 0b0101); // each number's parts swapped
2002 }
2004 for (std::size_t r {0}; r < R; ++r) {
2005 const double* const ap {as + (2 * ((r * lda) + p))};
2006 const __m256d re {_mm256_set1_pd(ap[0])};
2007 const __m256d im {_mm256_set1_pd(ap[1])};
2009 for (std::size_t v {0}; v < V; ++v) {
2010 const __m256d product {_mm256_addsub_pd(_mm256_mul_pd(re, bw[v].v), _mm256_mul_pd(im, bx[v].v))};
2012 sums[(r * V) + v].v = _mm256_add_pd(sums[(r * V) + v].v, product);
2013 }
2014 }
2015 }
2016 const bool declined {_mm256_movemask_pd(nan) != 0};
2017 if (!declined) {
2019 for (std::size_t r {0}; r < R; ++r) {
2021 for (std::size_t v {0}; v < V; ++v) {
2022 _mm256_storeu_pd(cs + (2 * ((r * ldc) + (2 * v))), sums[(r * V) + v].v);
2023 }
2024 }
2025 }
2026 _mm256_zeroupper(); // the upper halves cleared: code past the kernel may be SSE alone
2027 return !declined;
2028 }
2029
2031 template <std::size_t R>
2032 NUMERICAL_TARGET("avx2") bool narrow_column_complex_x86(std::size_t k,
2033 const std::complex<double>* a,
2034 std::size_t lda,
2035 const std::complex<double>* b,
2036 std::size_t ldb,
2037 std::complex<double>* c,
2038 std::size_t ldc,
2039 c_start start) noexcept
2040 {
2041 std::array<xmm_register, R> sums {};
2043 auto* const cs {reinterpret_cast<double*>(c)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is the array of its two parts
2044 const auto* const as {reinterpret_cast<const double*>(a)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): as above
2045 const auto* const bs {reinterpret_cast<const double*>(b)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): as above
2047 for (std::size_t r {0}; r < R; ++r) {
2048 sums[r].v = start == c_start::kept ? _mm_loadu_pd(cs + (2 * r * ldc)) : _mm_setzero_pd();
2049 }
2050 for (std::size_t p {0}; p < k; ++p) {
2051 const __m128d bh {_mm_loadu_pd(bs + (2 * p * ldb))};
2052 const __m128d bhx {_mm_permute_pd(bh, 0b01)}; // the parts swapped
2054 for (std::size_t r {0}; r < R; ++r) {
2055 const double* const ap {as + (2 * ((r * lda) + p))};
2058 sums[r].v = _mm_add_pd(sums[r].v, product);
2059 }
2060 }
2061 const bool declined {_mm_movemask_pd(nan) != 0};
2062 if (!declined) {
2064 for (std::size_t r {0}; r < R; ++r) {
2065 _mm_storeu_pd(cs + (2 * r * ldc), sums[r].v);
2066 }
2067 }
2068 return !declined;
2069 }
2070
2076 template <std::size_t R, std::size_t N>
2077 bool narrow_block_complex_x86(std::size_t k,
2078 const std::complex<double>* a,
2079 std::size_t lda,
2080 const std::complex<double>* b,
2081 std::size_t ldb,
2082 std::complex<double>* c,
2083 std::size_t ldc,
2084 c_start start) noexcept
2085 {
2086 if constexpr (N == 1) {
2087 return narrow_column_complex_x86<R>(k, a, lda, b, ldb, c, ldc, start);
2088 }
2089 else if constexpr (N == 2 || N == 4) {
2090 return narrow_columns_complex_x86<R, N / 2>(k, a, lda, b, ldb, c, ldc, start);
2091 }
2092 else {
2093 constexpr std::size_t w {N > 4 ? 4 : 2};
2094 std::array<std::complex<double>, R * w> first {};
2095 if (start == c_start::kept) {
2096 for (std::size_t e {0}; e < R * w; ++e) {
2097 first[e] = c[((e / w) * ldc) + (e % w)];
2098 }
2099 }
2100 if (!narrow_columns_complex_x86<R, w / 2>(k, a, lda, b, ldb, first.data(), w, start)
2101 || !narrow_block_complex_x86<R, N - w>(k, a, lda, b + w, ldb, c + w, ldc, start)) {
2102 return false;
2103 }
2104 for (std::size_t e {0}; e < R * w; ++e) {
2105 c[((e / w) * ldc) + (e % w)] = first[e];
2106 }
2107 return true;
2108 }
2109 }
2110
2111#endif
2112
2118 template <std::size_t R, std::size_t N, term_rounding Mode, typename T>
2120 const T* a,
2121 std::size_t lda,
2122 const T* b,
2123 std::size_t ldb,
2124 T* c,
2125 std::size_t ldc,
2126 c_start start) noexcept
2127 {
2128#ifdef NUMERICAL_X86_COMPILED
2129 if constexpr (Mode == term_rounding::x86_fused) {
2130 narrow_block_x86<R, N>(k, a, lda, b, ldb, c, ldc, start);
2131 }
2132 else
2133#endif
2134 {
2135#ifdef NUMERICAL_X86_COMPILED
2136 if constexpr (Mode == term_rounding::compiler && std::is_same_v<T, std::complex<double>>) {
2137 if (const gemm_path path {gemm_path_current()};
2138 (path == gemm_path::avx2 || path == gemm_path::avx512) && narrow_block_complex_x86<R, N>(k, a, lda, b, ldb, c, ldc, start)) {
2139 return;
2140 }
2141 }
2142#endif
2143 std::array<T, R * N> sums {}; // from zero where C holds nothing
2144 if (start == c_start::kept) {
2146 for (std::size_t e {0}; e < R * N; ++e) {
2147 sums[e] = c[((e / N) * ldc) + (e % N)];
2148 }
2149 }
2151 for (std::size_t p {0}; p < k; ++p) {
2153 for (std::size_t r {0}; r < R; ++r) {
2154 const T ap {a[(r * lda) + p]};
2156 for (std::size_t j {0}; j < N; ++j) {
2157 sums[(r * N) + j] = rounded_term(rounding_is<Mode> {}, sums[(r * N) + j], ap, b[(p * ldb) + j]);
2158 }
2159 }
2160 }
2162 for (std::size_t e {0}; e < R * N; ++e) {
2163 c[((e / N) * ldc) + (e % N)] = sums[e];
2164 }
2165 }
2166 }
2167
2170 template <std::size_t R, term_rounding Mode, typename T>
2171 void narrow_rows(std::size_t n,
2172 std::size_t k,
2173 const T* a,
2174 std::size_t lda,
2175 const T* b,
2176 std::size_t ldb,
2177 T* c,
2178 std::size_t ldc,
2179 c_start start) noexcept
2180 {
2181 static_assert(narrow_columns == 8, "the switch below names each narrower last block");
2182 std::size_t j {0};
2183 for (; j + narrow_columns <= n; j += narrow_columns) {
2184 narrow_block<R, narrow_columns, Mode>(k, a, lda, b + j, ldb, c + j, ldc, start);
2185 }
2186 switch (n - j) {
2187 case 1: narrow_block<R, 1, Mode>(k, a, lda, b + j, ldb, c + j, ldc, start); break;
2188 case 2: narrow_block<R, 2, Mode>(k, a, lda, b + j, ldb, c + j, ldc, start); break;
2189 case 3: narrow_block<R, 3, Mode>(k, a, lda, b + j, ldb, c + j, ldc, start); break;
2190 case 4: narrow_block<R, 4, Mode>(k, a, lda, b + j, ldb, c + j, ldc, start); break;
2191 case 5: narrow_block<R, 5, Mode>(k, a, lda, b + j, ldb, c + j, ldc, start); break;
2192 case 6: narrow_block<R, 6, Mode>(k, a, lda, b + j, ldb, c + j, ldc, start); break;
2193 case 7: narrow_block<R, 7, Mode>(k, a, lda, b + j, ldb, c + j, ldc, start); break;
2194 default: break; // a whole number of blocks
2195 }
2196 }
2197
2199 template <typename T>
2200 void zero_block(T* c,
2201 std::size_t ldc,
2202 std::size_t first,
2203 std::size_t last,
2204 std::size_t from,
2205 std::size_t to) noexcept
2206 {
2207 if (to - from == ldc) { // whole rows: one run
2208 std::fill(c + (first * ldc), c + (last * ldc), T {});
2209 return;
2210 }
2211 for (std::size_t i {first}; i < last; ++i) {
2212 std::fill(c + (i * ldc) + from, c + (i * ldc) + to, T {});
2213 }
2214 }
2215
2218 template <typename T>
2219 void zero_c(T* c,
2220 std::size_t ldc,
2221 std::size_t m,
2222 std::size_t n)
2223 {
2224 if (ldc == n) {
2225 parallel_zero(c, m * n);
2226 return;
2227 }
2228 zero_block(c, ldc, 0, m, 0, n);
2229 }
2230
2243 template <term_rounding Mode, typename T>
2244 void gemm_rows_rounded(std::size_t m,
2245 std::size_t n,
2246 std::size_t k,
2247 const T* a,
2248 std::size_t lda,
2249 const T* b,
2250 std::size_t ldb,
2251 T* c,
2252 std::size_t ldc,
2253 c_start start = c_start::kept)
2254 {
2255 if (k >= (n <= narrow_columns ? std::size_t {2} : narrow_least_depth)) {
2256 parallel_for((m + narrow_group - 1) / narrow_group, narrow_group * k * n / gemm_rows_weight, [&](std::size_t g) {
2257 const std::size_t first {g * narrow_group};
2258 if (first + narrow_group <= m) {
2259 narrow_rows<narrow_group, Mode>(n, k, a + (first * lda), lda, b, ldb, c + (first * ldc), ldc, start);
2260 return;
2261 }
2262 for (std::size_t i {first}; i < m; ++i) {
2263 narrow_rows<1, Mode>(n, k, a + (i * lda), lda, b, ldb, c + (i * ldc), ldc, start);
2264 }
2265 });
2266 return;
2267 }
2268 parallel_for(m, k * n / vector_speedup, [&](std::size_t i) {
2269#ifdef NUMERICAL_X86_COMPILED
2270 if constexpr (Mode == term_rounding::x86_fused) {
2271 plain_row_x86(n, k, a + (i * lda), b, ldb, c + (i * ldc), start); // C zeroed in its sums
2272 }
2273 else
2274#endif
2275 {
2276 if (start == c_start::zero) {
2277 zero_block(c, ldc, i, i + 1, 0, n);
2278 }
2279 for (std::size_t p {0}; p < k; ++p) {
2280 const T aip {a[(i * lda) + p]};
2281 for (std::size_t j {0}; j < n; ++j) {
2282 c[(i * ldc) + j] = rounded_term(rounding_is<Mode> {}, c[(i * ldc) + j], aip, b[(p * ldb) + j]);
2283 }
2284 }
2285 }
2286 });
2287 }
2288
2294 template <std::floating_point T>
2296 {
2297#ifdef NUMERICAL_X86_COMPILED
2298 if constexpr (std::is_same_v<T, double> || std::is_same_v<T, float>) {
2301 return true;
2302 }
2303 }
2304#endif
2305 return rows_fuse();
2306 }
2307
2310 enum class complex_term : std::uint8_t
2311 {
2312 compiler,
2313 tiles,
2314 split,
2315 };
2316
2319 {
2322 }
2323#ifdef NUMERICAL_X86_COMPILED
2324 return complex_term::tiles; // the AVX2 and AVX-512 paths, both on a complex tile
2325#else
2326 return complex_term::split;
2327#endif
2328 }
2329
2335 template <typename T>
2336 void gemm_rows(std::size_t m,
2337 std::size_t n,
2338 std::size_t k,
2339 const T* a,
2340 std::size_t lda,
2341 const T* b,
2342 std::size_t ldb,
2343 T* c,
2344 std::size_t ldc,
2345 c_start start = c_start::kept)
2346 {
2347#ifdef NUMERICAL_X86_COMPILED
2348 if constexpr (std::is_same_v<T, double> || std::is_same_v<T, float>) {
2351 gemm_rows_rounded<term_rounding::x86_fused>(m, n, k, a, lda, b, ldb, c, ldc, start);
2352 return;
2353 }
2354 }
2355#endif
2356 if constexpr (std::is_floating_point_v<T>) {
2357 if (rows_fuse()) {
2358 gemm_rows_rounded<term_rounding::fused>(m, n, k, a, lda, b, ldb, c, ldc, start);
2359 return;
2360 }
2361 }
2362 gemm_rows_rounded<term_rounding::compiler>(m, n, k, a, lda, b, ldb, c, ldc, start);
2363 }
2364
2366 inline constexpr std::size_t packed_line {64};
2367
2370 template <typename T>
2371 [[nodiscard]] T* line_aligned(T* at) noexcept
2372 {
2373 const std::size_t miss {static_cast<std::size_t>(std::bit_cast<std::uintptr_t>(at) % packed_line)};
2374 return at + (((packed_line - miss) % packed_line) / sizeof(T));
2375 }
2376
2387 template <typename T>
2389 {
2390 public:
2391
2393 static constexpr std::size_t kept_limit {(std::size_t {32} << 20U) / sizeof(T)};
2394
2397 std::size_t b_count)
2398 : storage_ {grown(std::exchange(kept(), buffer<T> {}), a_count + padding + b_count + padding)},
2399 a_ {aligned(storage_.data())},
2400 b_ {aligned(storage_.data() + a_count + padding)}
2401 {}
2402
2407
2409 {
2410 kept() = storage_.size() <= kept_limit ? std::move(storage_) : buffer<T> {};
2411 }
2412
2415 {
2416 return a_;
2417 }
2418
2421 {
2422 return b_;
2423 }
2424
2427 {
2428 thread_local buffer<T> storage;
2429 [[maybe_unused]] thread_local const bool registered {on_release([] { storage = buffer<T> {}; })};
2430 return storage;
2431 }
2432
2433 private:
2434
2435 static constexpr std::size_t padding {(packed_line / sizeof(T)) + 1};
2436
2439 std::size_t count)
2440 {
2441 storage.resize(std::max(storage.size(), count), T {});
2442 return storage;
2443 }
2444
2446 [[nodiscard]] static T* aligned(T* at) noexcept
2447 {
2448 return line_aligned(at);
2449 }
2450
2454 };
2455
2458 template <typename Kernel>
2459 [[nodiscard]] constexpr bool gemm_shares(std::size_t m,
2460 std::size_t n,
2461 std::size_t k) noexcept
2462 {
2463 return m * n * k >= Kernel::serial_below;
2464 }
2465
2469 template <typename Kernel>
2470 inline constexpr bool prefetches_next_row_of_c {requires(const double* c, std::size_t ldc) {
2471 Kernel::prefetch_rows(c, ldc);
2472 }};
2473
2475 template <typename Kernel,
2476 typename T>
2477 void prefetch_next_row(const T* /*c*/,
2478 std::size_t /*ldc*/) noexcept
2479 {
2480 }
2481
2483 template <typename Kernel,
2484 typename T>
2486 void prefetch_next_row(const T* c,
2487 std::size_t ldc) noexcept
2488 {
2489 Kernel::prefetch_rows(c, ldc);
2490 }
2491
2494 template <typename Kernel,
2495 typename T>
2496 inline constexpr bool writes_fresh {requires(const T * x, T * y, std::size_t z) {
2497 Kernel::tile(z, x, x, z, y, z, z, z, true);
2498 }};
2499
2501 template <typename Kernel,
2502 typename T>
2503 inline constexpr bool reads_a_in_place {requires(const T * x, T * y, std::size_t z) {
2504 Kernel::tile_a_in_place(z, x, z, z, x, z, y, z, z, z);
2505 }};
2506
2509 template <typename Kernel>
2510 inline constexpr bool reads_partial_b_in_place {requires { requires Kernel::partial_b_in_place; }};
2511
2513 template <typename Kernel,
2514 typename T>
2515 requires (!reads_a_in_place<Kernel, T>)
2516 [[nodiscard]] constexpr bool packs_a(const T* a_packed,
2517 std::size_t /*n*/) noexcept
2518 {
2519 return a_packed == nullptr;
2520 }
2521
2524 template <typename Kernel,
2525 typename T>
2527 [[nodiscard]] constexpr bool packs_a(const T* a_packed,
2528 std::size_t n) noexcept
2529 {
2530 return a_packed == nullptr && n > Kernel::nr;
2531 }
2532
2534 template <typename Kernel,
2535 typename T>
2536 requires (!writes_fresh<Kernel, T>)
2537 [[nodiscard]] constexpr bool zeroes_c(bool empty) noexcept
2538 {
2539 return empty;
2540 }
2541
2543 template <typename Kernel,
2544 typename T>
2546 [[nodiscard]] constexpr bool zeroes_c(bool /*empty*/) noexcept
2547 {
2548 return false;
2549 }
2550
2552 template <typename Kernel>
2553 [[nodiscard]] constexpr std::size_t task_rows_cap(std::size_t /*kc*/) noexcept
2554 {
2555 return Kernel::block_rows;
2556 }
2557
2559 template <typename Kernel>
2560 requires requires(std::size_t kc)
2561 {
2562 Kernel::task_rows_for(kc);
2563 }
2564
2565 [[nodiscard]] constexpr std::size_t task_rows_cap(std::size_t kc) noexcept
2566 {
2567 return Kernel::task_rows_for(kc);
2568 }
2569
2579 template <typename Kernel,
2580 typename T,
2581 typename B = b_rows,
2582 typename A = a_rows>
2583 void gemm_packed(std::size_t m,
2584 std::size_t n,
2585 std::size_t k,
2586 const T* a,
2587 std::size_t lda,
2588 const T* b,
2589 std::size_t ldb,
2590 T* c,
2591 std::size_t ldc,
2592 const T* a_packed = nullptr,
2593 c_start start = c_start::kept)
2594 {
2595 constexpr std::size_t mr {Kernel::mr};
2596 constexpr std::size_t nr {Kernel::nr};
2597 constexpr std::size_t nc_max {nr * 128};
2598 // A read where it lies by a product of one sliver of columns, which reads each of its elements once: packing it
2599 // cost more than the tiles (430 × 607 by 607 × 6 complex128, half the time).
2600 const bool a_own {packs_a<Kernel, T>(a_packed, n)}; // A packed here
2601#ifdef NUMERICAL_X86_COMPILED // the complex tiles, the only ones that read A in place
2602 const bool a_in_place {a_packed == nullptr && !a_own};
2603#endif
2604 const std::size_t a_rows {((m + mr) - 1) / mr * mr};
2605 const std::size_t b_cols {((std::min(nc_max, n) + nr) - 1) / nr * nr};
2606 const packing_storage<T> packed {a_own ? a_rows * std::min(Kernel::depth, k) : 0, b_cols * std::min(Kernel::depth, k)};
2607 const std::size_t a_blocks {a_own ? ((m + Kernel::block_rows) - 1) / Kernel::block_rows : 0};
2608 const T* const a_tiles {a_packed != nullptr ? a_packed : packed.a()};
2609 const std::size_t shared {gemm_shares<Kernel>(m, n, k) ? 1U : 0U}; // a weight of zero starts no thread
2610 // B's whole slivers unpacked: rows close enough, or few enough rows of A whatever B's stride
2611 // (Kernel::b_in_place_rows), where the kernel reads B in place at all.
2613 // B's last partial sliver in place too, where the kernel reads it so (\ref reads_partial_b_in_place): a row's load
2614 // of \p nr elements runs past the sliver's last column into B's next row (its rows \p nr apart at least), inside
2615 // B, but from B's last row, which alone is packed. Packing the sliver whole took most of a thin product on the AMX
2616 // (one M1 Pro core, 21 × 639 by 639 × 104 floats: 12.3 µs, 5.8 for 96 columns).
2617 const bool partial_in_place {in_place && reads_partial_b_in_place<Kernel> && ldb >= nr};
2618 for (std::size_t jc {0}; jc < n; jc += nc_max) {
2619 const std::size_t nc {std::min(nc_max, n - jc)};
2620 const std::size_t slivers {((nc + nr) - 1) / nr};
2621 for (std::size_t pc {0}; pc < k; pc += Kernel::depth) {
2622 const std::size_t kc {std::min(Kernel::depth, k - pc)};
2623 const std::size_t groups {((slivers + gemm_slivers_per_task) - 1) / gemm_slivers_per_task};
2624 // A task covers task_rows rows: a whole block, or fewer whole tiles when blocks would leave a thread fewer
2625 // than tasks_per_thread tasks (order 256 on twenty x86-64 threads: twelve tasks took 116 µs, 44 took 104).
2626 const std::size_t tiles {((m + mr) - 1) / mr};
2627 const std::size_t row_tasks {(((tasks_per_thread * max_threads()) + groups) - 1) / groups};
2628 const std::size_t task_rows {std::clamp(((tiles + row_tasks) - 1) / row_tasks, std::size_t {1}, task_rows_cap<Kernel>(kc) / mr) * mr};
2629 const std::size_t task_count {((m + task_rows) - 1) / task_rows};
2630 const std::size_t task_work {shared * task_rows * gemm_slivers_per_task * nr * kc / Kernel::speedup};
2631 // C holding nothing: zeroed by the tasks on threads, each its block before its first panel; else here, at
2632 // once and before the packing, which its lines would then evict (one AVX-512 thread, order 256: 434 → 428 µs).
2633 // A tile written fresh by the first panel needs C zeroed by no one: one pass over C fewer, and none read.
2634 const bool empty {start == c_start::zero && pc == 0}; // C holds nothing before this panel's tiles
2635 const bool zeroing {zeroes_c<Kernel, T>(empty)}; // C to zero before they add to it
2636#ifdef NUMERICAL_X86_COMPILED // the x86-64 tiles, the only ones that write C fresh
2637 const bool fresh {empty && !zeroing};
2638#endif
2640 if (zeroing && !zero_tasks) {
2641 zero_block(c, ldc, 0, m, jc, jc + nc);
2642 }
2643 // One pass packs both: B's groups of slivers first, then A's row blocks.
2644 // The packing weighed as the tile's work is (\p Kernel's speedup): a small product then packs in parallel too
2645 // (eight AVX2 cores: order 128 40 → 30 µs, 192 92 → 62).
2646 parallel_for(groups + a_blocks, shared * Kernel::block_rows * kc * gemm_kernel_speedup / Kernel::speedup, [&](std::size_t piece) {
2647 if (piece < groups) {
2648 const std::size_t jr {piece * gemm_slivers_per_task * nr};
2649 const std::size_t width {std::min(gemm_slivers_per_task * nr, nc - jr)};
2650 const std::size_t whole {in_place ? width - (width % nr) : 0}; // read in place
2651 if (whole < width && !partial_in_place) { // a partial sliver is packed even so: its outside zero
2652 B::template pack<nr>(b, ldb, pc, jc + jr + whole, kc, width - whole, packed.b() + ((jr + whole) * kc));
2653 }
2654 else if (whole < width && pc + kc == k) { // B's last row alone
2655 B::template pack<nr>(b, ldb, k - 1, jc + jr + whole, 1, width - whole, packed.b() + ((jr + whole) * kc));
2656 }
2657 }
2658 else {
2659 const std::size_t ic {(piece - groups) * Kernel::block_rows};
2660 A::template pack<Kernel>(a, lda, ic, pc, std::min(Kernel::block_rows, m - ic), kc, packed.a() + (ic * kc));
2661 }
2662 });
2663 parallel_for(task_count * groups, task_work, [&](std::size_t task) {
2664 const std::size_t ic {(task / groups) * task_rows};
2665 const std::size_t first {(task % groups) * gemm_slivers_per_task};
2666 const std::size_t last {std::min(first + gemm_slivers_per_task, slivers)};
2667 const std::size_t ic_end {std::min(ic + task_rows, m)};
2668 if (zero_tasks) {
2669 zero_block(c, ldc, ic, ic_end, jc + (first * nr), jc + std::min(last * nr, nc));
2670 }
2671 [[maybe_unused]] const typename Kernel::session on {};
2672 // A sliver (the operand each step broadcasts) stays in the L1 cache across the task's B slivers,
2673 // which stream from the L2, as BLIS orders its loops: B sliver outer, its 32 KiB filled the L1
2674 // (a product of order 2048 on one AVX-512 core: 72 GFLOP/s, 77 in this order; AVX2 and AMX gain too).
2675 for (std::size_t ir {ic}; ir < ic_end; ir += mr) {
2676 for (std::size_t s {first}; s < last; ++s) {
2677 const std::size_t jr {s * nr};
2678 const bool direct {in_place && jr + nr <= nc};
2679 if (ir + mr < ic_end) { // the next row's tile of C, while this one computes
2680 prefetch_next_row<Kernel>(c + ((ir + mr) * ldc) + jc + jr, ldc);
2681 }
2682 if constexpr (reads_partial_b_in_place<Kernel>) {
2683 if (partial_in_place && !direct) { // B's last row, if in this panel, after the others
2684 const std::size_t ahead {pc + kc == k ? kc - 1 : kc};
2685 T* const tile {c + (ir * ldc) + jc + jr};
2686 Kernel::tile(ahead, a_tiles + (ir * kc), b + (pc * ldb) + jc + jr, ldb, tile, ldc, std::min(mr, m - ir),
2687 nc - jr);
2688 if (ahead < kc) { // its accumulators stored and loaded again: each sum goes on from its value
2689 Kernel::tile(1, a_tiles + (ir * kc) + (ahead * mr), packed.b() + (jr * kc), nr, tile, ldc,
2690 std::min(mr, m - ir), nc - jr);
2691 }
2692 continue;
2693 }
2694 }
2695 const T* const b_tile {direct ? b + (pc * ldb) + jc + jr : packed.b() + (jr * kc)};
2696 const std::size_t b_ld {direct ? ldb : nr};
2697#ifdef NUMERICAL_X86_COMPILED
2698 if constexpr (reads_a_in_place<Kernel, T>) {
2699 if (a_in_place) {
2700 Kernel::tile_a_in_place(kc, A::element(a, lda, ir, pc), A::step(lda), A::across(lda), b_tile, b_ld,
2701 c + (ir * ldc) + jc + jr, ldc, std::min(mr, m - ir), std::min(nr, nc - jr), fresh);
2702 continue;
2703 }
2704 }
2705 if constexpr (writes_fresh<Kernel, T>) {
2706 Kernel::tile(kc, a_tiles + (ir * kc), b_tile, b_ld, c + (ir * ldc) + jc + jr, ldc, std::min(mr, m - ir),
2707 std::min(nr, nc - jr), fresh);
2708 continue;
2709 }
2710#endif
2711 Kernel::tile(kc, a_tiles + (ir * kc), b_tile, b_ld, c + (ir * ldc) + jc + jr, ldc, std::min(mr, m - ir),
2712 std::min(nr, nc - jr));
2713 }
2714 }
2715 });
2716 }
2717 }
2718 }
2719
2728 template <typename Kernel,
2729 typename T,
2730 typename B = b_rows,
2731 typename A = a_rows>
2732 void gemm_packed_triangle(std::size_t n,
2733 std::size_t k,
2734 const T* a,
2735 std::size_t lda,
2736 const T* b,
2737 std::size_t ldb,
2738 T* c,
2739 std::size_t ldc,
2740 bool lower)
2741 {
2742 constexpr std::size_t mr {Kernel::mr};
2743 constexpr std::size_t nr {Kernel::nr};
2744 constexpr std::size_t nc_max {nr * 128};
2745 const std::size_t a_rows {((n + mr) - 1) / mr * mr};
2746 const std::size_t b_cols {((std::min(nc_max, n) + nr) - 1) / nr * nr};
2747 const packing_storage<T> packed {a_rows * std::min(Kernel::depth, k), b_cols * std::min(Kernel::depth, k)};
2748 const std::size_t a_blocks {((n + Kernel::block_rows) - 1) / Kernel::block_rows};
2749 const std::size_t shared {gemm_shares<Kernel>(n, n / 2, k) ? 1U : 0U};
2750 const bool in_place {B::template in_place<Kernel>(ldb)};
2751 // Whether rows [r0, r1) and columns [j0, j1) of C meet the triangle, and whether they lie wholly in it.
2752 const auto meets = [lower](std::size_t r0, std::size_t r1, std::size_t j0, std::size_t j1) { return lower ? j0 < r1 : r0 < j1; };
2753 const auto inside = [lower](std::size_t r0, std::size_t r1, std::size_t j0, std::size_t j1) { return lower ? j1 <= r0 + 1 : r1 <= j0 + 1; };
2754 for (std::size_t jc {0}; jc < n; jc += nc_max) {
2755 const std::size_t nc {std::min(nc_max, n - jc)};
2756 const std::size_t slivers {((nc + nr) - 1) / nr};
2757 for (std::size_t pc {0}; pc < k; pc += Kernel::depth) {
2758 const std::size_t kc {std::min(Kernel::depth, k - pc)};
2759 const std::size_t groups {((slivers + gemm_slivers_per_task) - 1) / gemm_slivers_per_task};
2760 parallel_for(groups + a_blocks, shared * Kernel::block_rows * kc * gemm_kernel_speedup / Kernel::speedup, [&](std::size_t piece) {
2761 if (piece < groups) {
2762 const std::size_t jr {piece * gemm_slivers_per_task * nr};
2763 const std::size_t width {std::min(gemm_slivers_per_task * nr, nc - jr)};
2764 const std::size_t whole {in_place ? width - (width % nr) : 0}; // read in place
2765 if (whole < width) { // a partial sliver is packed even so: its outside zero
2766 B::template pack<nr>(b, ldb, pc, jc + jr + whole, kc, width - whole, packed.b() + ((jr + whole) * kc));
2767 }
2768 return;
2769 }
2770 const std::size_t ic {(piece - groups) * Kernel::block_rows};
2771 const std::size_t rows {std::min(Kernel::block_rows, n - ic)};
2772 if (meets(ic, ic + rows, jc, jc + nc)) {
2773 A::template pack<Kernel>(a, lda, ic, pc, rows, kc, packed.a() + (ic * kc));
2774 }
2775 });
2776 const std::size_t tiles {((n + mr) - 1) / mr};
2777 const std::size_t row_tasks {(((tasks_per_thread * max_threads()) + groups) - 1) / groups};
2778 const std::size_t task_rows {std::clamp(((tiles + row_tasks) - 1) / row_tasks, std::size_t {1}, task_rows_cap<Kernel>(kc) / mr) * mr};
2779 const std::size_t task_count {((n + task_rows) - 1) / task_rows};
2780 parallel_for(task_count * groups, shared * task_rows * gemm_slivers_per_task * nr * kc / (2 * Kernel::speedup), [&](std::size_t task) {
2781 const std::size_t ic {(task / groups) * task_rows};
2782 const std::size_t first {(task % groups) * gemm_slivers_per_task};
2783 const std::size_t last {std::min(first + gemm_slivers_per_task, slivers)};
2784 const std::size_t ic_end {std::min(ic + task_rows, n)};
2785 [[maybe_unused]] const typename Kernel::session on {};
2786 for (std::size_t ir {ic}; ir < ic_end; ir += mr) {
2787 const std::size_t rows {std::min(mr, n - ir)};
2788 for (std::size_t s {first}; s < last; ++s) {
2789 const std::size_t jr {s * nr};
2790 const std::size_t j0 {jc + jr};
2791 const std::size_t cols {std::min(nr, nc - jr)};
2792 if (!meets(ir, ir + rows, j0, j0 + cols)) {
2793 continue;
2794 }
2795 const bool direct {in_place && jr + nr <= nc};
2796 const T* const bp {direct ? b + (pc * ldb) + j0 : packed.b() + (jr * kc)};
2797 const std::size_t ldbp {direct ? ldb : nr};
2798 T* const block {c + (ir * ldc) + j0};
2799 if (inside(ir, ir + rows, j0, j0 + cols)) {
2800 Kernel::tile(kc, packed.a() + (ir * kc), bp, ldbp, block, ldc, rows, cols);
2801 continue;
2802 }
2803 std::array<T, mr * nr> copy {};
2804 for (std::size_t i {0}; i < rows; ++i) {
2805 std::copy_n(block + (i * ldc), cols, copy.data() + (i * nr));
2806 }
2807 Kernel::tile(kc, packed.a() + (ir * kc), bp, ldbp, copy.data(), nr, rows, cols);
2808 for (std::size_t i {0}; i < rows; ++i) {
2809 for (std::size_t j {0}; j < cols; ++j) {
2810 if (lower ? j0 + j <= ir + i : j0 + j >= ir + i) {
2811 block[(i * ldc) + j] = copy[(i * nr) + j];
2812 }
2813 }
2814 }
2815 }
2816 }
2817 });
2818 }
2819 }
2820 }
2821
2822#ifdef NUMERICAL_X86_COMPILED
2824 struct avx512_tile_kernel
2825 {
2826 static constexpr std::size_t mr {avx512_mr};
2827 static constexpr std::size_t nr {avx512_nr};
2828 static constexpr std::size_t depth {gemm_kc};
2829 static constexpr std::size_t block_rows {96};
2830 static constexpr std::size_t b_in_place_rows {mr};
2833 static constexpr std::size_t speedup {gemm_kernel_speedup / 4};
2834 static constexpr std::size_t serial_below {1};
2835
2837 struct session
2838 {};
2839
2842 static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
2843 {
2844 return ldb <= nr;
2845 }
2846
2848 static void pack_a(const double* a,
2849 std::size_t lda,
2850 std::size_t rows,
2851 std::size_t kc,
2852 double* out)
2853 {
2854 gemm_pack_a<mr>(a, lda, rows, kc, out);
2855 }
2856
2858 static void tile(std::size_t kc,
2859 const double* ap,
2860 const double* bp,
2861 std::size_t ldb,
2862 double* c,
2863 std::size_t ldc,
2864 std::size_t rows,
2865 std::size_t cols,
2866 bool fresh = false) noexcept
2867 {
2868 avx512_tile_f64(kc, ap, bp, ldb, c, ldc, rows, cols, fresh);
2869 }
2870 };
2871
2873 struct avx2_tile_kernel
2874 {
2875 static constexpr std::size_t mr {avx2_mr};
2876 static constexpr std::size_t nr {avx2_nr};
2877 static constexpr std::size_t depth {gemm_kc};
2878 static constexpr std::size_t block_rows {72};
2879 static constexpr std::size_t b_in_place_rows {mr};
2880
2886 [[nodiscard]] static constexpr std::size_t task_rows_for(std::size_t kc) noexcept
2887 {
2888 return kc <= 128 ? 36 : block_rows;
2889 }
2890
2896 static void prefetch_rows(const double* c,
2897 std::size_t ldc) noexcept
2898 {
2899 for (std::size_t r {0}; r < mr; ++r) {
2900 _mm_prefetch(reinterpret_cast<const char*>(c + (r * ldc)), _MM_HINT_T0); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the intrinsic takes a byte address
2901 }
2902 }
2903
2906 static constexpr std::size_t speedup {gemm_kernel_speedup / 4};
2907 static constexpr std::size_t serial_below {1};
2908
2910 struct session
2911 {};
2912
2915 static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
2916 {
2917 return ldb <= nr;
2918 }
2919
2921 static void pack_a(const double* a,
2922 std::size_t lda,
2923 std::size_t rows,
2924 std::size_t kc,
2925 double* out)
2926 {
2927 gemm_pack_a<mr>(a, lda, rows, kc, out);
2928 }
2929
2931 static void tile(std::size_t kc,
2932 const double* ap,
2933 const double* bp,
2934 std::size_t ldb,
2935 double* c,
2936 std::size_t ldc,
2937 std::size_t rows,
2938 std::size_t cols,
2939 bool fresh = false) noexcept
2940 {
2941 avx2_tile_f64(kc, ap, bp, ldb, c, ldc, rows, cols, fresh);
2942 }
2943 };
2946 {
2947 static constexpr std::size_t mr {avx512_mr};
2948 static constexpr std::size_t nr {avx512_nr_f32};
2949 static constexpr std::size_t depth {gemm_kc};
2950 static constexpr std::size_t block_rows {96};
2951 static constexpr std::size_t b_in_place_rows {mr};
2952 static constexpr std::size_t speedup {gemm_kernel_speedup / 4};
2953 static constexpr std::size_t serial_below {1};
2954
2955 struct session
2956 {};
2957
2958 static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
2959 {
2960 return ldb <= nr;
2961 }
2962
2963 static void pack_a(const float* a,
2964 std::size_t lda,
2965 std::size_t rows,
2966 std::size_t kc,
2967 float* out)
2968 {
2969 gemm_pack_a<mr>(a, lda, rows, kc, out);
2970 }
2971
2972 static void tile(std::size_t kc,
2973 const float* ap,
2974 const float* bp,
2975 std::size_t ldb,
2976 float* c,
2977 std::size_t ldc,
2978 std::size_t rows,
2979 std::size_t cols,
2980 bool fresh = false) noexcept
2981 {
2982 avx512_tile_f32(kc, ap, bp, ldb, c, ldc, rows, cols, fresh);
2983 }
2984 };
2985
2988 {
2989 static constexpr std::size_t mr {avx2_mr};
2990 static constexpr std::size_t nr {avx2_nr_f32};
2991 static constexpr std::size_t depth {gemm_kc};
2992 static constexpr std::size_t block_rows {72};
2993 static constexpr std::size_t b_in_place_rows {mr};
2994 static constexpr std::size_t speedup {gemm_kernel_speedup / 4};
2995 static constexpr std::size_t serial_below {1};
2996
2997 [[nodiscard]] static constexpr std::size_t task_rows_for(std::size_t kc) noexcept
2998 {
2999 return kc <= 128 ? 36 : block_rows;
3000 }
3001
3002 struct session
3003 {};
3004
3005 static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
3006 {
3007 return ldb <= nr;
3008 }
3009
3010 static void pack_a(const float* a,
3011 std::size_t lda,
3012 std::size_t rows,
3013 std::size_t kc,
3014 float* out)
3015 {
3016 gemm_pack_a<mr>(a, lda, rows, kc, out);
3017 }
3018
3019 static void tile(std::size_t kc,
3020 const float* ap,
3021 const float* bp,
3022 std::size_t ldb,
3023 float* c,
3024 std::size_t ldc,
3025 std::size_t rows,
3026 std::size_t cols,
3027 bool fresh = false) noexcept
3028 {
3029 avx2_tile_f32(kc, ap, bp, ldb, c, ldc, rows, cols, fresh);
3030 }
3031 };
3032
3035 {
3036 static constexpr std::size_t mr {avx512_complex_mr};
3037 static constexpr std::size_t nr {avx512_complex_nr};
3038 static constexpr std::size_t depth {gemm_kc / 2};
3039 static constexpr std::size_t block_rows {96};
3040 static constexpr std::size_t b_in_place_rows {mr};
3041 static constexpr std::size_t speedup {gemm_kernel_speedup / 16};
3045 static constexpr std::size_t serial_below {std::size_t {343000}};
3046
3048 struct session
3049 {};
3050
3053 static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
3054 {
3055 return ldb <= nr;
3056 }
3057
3059 static void pack_a(const std::complex<double>* a,
3060 std::size_t lda,
3061 std::size_t rows,
3062 std::size_t kc,
3063 std::complex<double>* out)
3064 {
3065 gemm_pack_a<mr>(a, lda, rows, kc, out);
3066 }
3067
3069 static void tile(std::size_t kc,
3070 const std::complex<double>* ap,
3071 const std::complex<double>* bp,
3072 std::size_t ldb,
3073 std::complex<double>* c,
3074 std::size_t ldc,
3075 std::size_t rows,
3076 std::size_t cols,
3077 bool fresh = false) noexcept
3078 {
3079 avx512_tile_c64(kc, ap, bp, ldb, c, ldc, rows, cols, fresh);
3080 }
3081
3084 static void tile_a_in_place(std::size_t kc,
3085 const std::complex<double>* a,
3086 std::size_t a_step,
3087 std::size_t a_row,
3088 const std::complex<double>* bp,
3089 std::size_t ldb,
3090 std::complex<double> * c,
3091 std::size_t ldc,
3092 std::size_t rows,
3093 std::size_t cols,
3094 bool fresh = false) noexcept
3095 {
3096 avx512_tile_c64_a_in_place(kc, a, a_step, a_row, bp, ldb, c, ldc, rows, cols, fresh);
3097 }
3098 };
3099
3102 {
3103 static constexpr std::size_t mr {avx2_complex_mr};
3104 static constexpr std::size_t nr {avx2_complex_nr};
3105 static constexpr std::size_t depth {gemm_kc / 2};
3106 static constexpr std::size_t block_rows {72};
3107 static constexpr std::size_t b_in_place_rows {mr};
3108 static constexpr std::size_t speedup {gemm_kernel_speedup / 16};
3111 static constexpr std::size_t serial_below {std::size_t {140608}};
3112
3114 struct session
3115 {};
3116
3119 static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
3120 {
3121 return ldb <= nr;
3122 }
3123
3125 static void pack_a(const std::complex<double>* a,
3126 std::size_t lda,
3127 std::size_t rows,
3128 std::size_t kc,
3129 std::complex<double>* out)
3130 {
3131 gemm_pack_a<mr>(a, lda, rows, kc, out);
3132 }
3133
3135 static void tile(std::size_t kc,
3136 const std::complex<double>* ap,
3137 const std::complex<double>* bp,
3138 std::size_t ldb,
3139 std::complex<double>* c,
3140 std::size_t ldc,
3141 std::size_t rows,
3142 std::size_t cols,
3143 bool fresh = false) noexcept
3144 {
3145 avx2_tile_c64(kc, ap, bp, ldb, c, ldc, rows, cols, fresh);
3146 }
3147
3150 static void tile_a_in_place(std::size_t kc,
3151 const std::complex<double>* a,
3152 std::size_t a_step,
3153 std::size_t a_row,
3154 const std::complex<double>* bp,
3155 std::size_t ldb,
3156 std::complex<double> * c,
3157 std::size_t ldc,
3158 std::size_t rows,
3159 std::size_t cols,
3160 bool fresh = false) noexcept
3161 {
3162 avx2_tile_c64_a_in_place(kc, a, a_step, a_row, bp, ldb, c, ldc, rows, cols, fresh);
3163 }
3164 };
3167 {
3168 static constexpr std::size_t mr {avx512_complex_mr};
3169 static constexpr std::size_t nr {avx512_complex_nr_f32};
3170 static constexpr std::size_t depth {gemm_kc}; // an element is two `float`, the size of a `double`
3171 static constexpr std::size_t block_rows {96};
3172 static constexpr std::size_t b_in_place_rows {mr};
3173 static constexpr std::size_t speedup {gemm_kernel_speedup / 16};
3176 static constexpr std::size_t serial_below {std::size_t {592704}};
3177
3178 struct session
3179 {};
3180
3181 static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
3182 {
3183 return ldb <= nr;
3184 }
3185
3186 static void pack_a(const std::complex<float>* a,
3187 std::size_t lda,
3188 std::size_t rows,
3189 std::size_t kc,
3190 std::complex<float>* out)
3191 {
3192 gemm_pack_a<mr>(a, lda, rows, kc, out);
3193 }
3194
3195 static void tile(std::size_t kc,
3196 const std::complex<float>* ap,
3197 const std::complex<float>* bp,
3198 std::size_t ldb,
3199 std::complex<float>* c,
3200 std::size_t ldc,
3201 std::size_t rows,
3202 std::size_t cols,
3203 bool fresh = false) noexcept
3204 {
3205 avx512_tile_c32(kc, ap, bp, ldb, c, ldc, rows, cols, fresh);
3206 }
3207
3210 static void tile_a_in_place(std::size_t kc,
3211 const std::complex<float>* a,
3212 std::size_t a_step,
3213 std::size_t a_row,
3214 const std::complex<float>* bp,
3215 std::size_t ldb,
3216 std::complex<float> * c,
3217 std::size_t ldc,
3218 std::size_t rows,
3219 std::size_t cols,
3220 bool fresh = false) noexcept
3221 {
3222 avx512_tile_c32_a_in_place(kc, a, a_step, a_row, bp, ldb, c, ldc, rows, cols, fresh);
3223 }
3224 };
3225
3228 {
3229 static constexpr std::size_t mr {avx2_complex_mr};
3230 static constexpr std::size_t nr {avx2_complex_nr_f32};
3231 static constexpr std::size_t depth {gemm_kc}; // an element is two `float`, the size of a `double`
3232 static constexpr std::size_t block_rows {72};
3233 static constexpr std::size_t b_in_place_rows {mr};
3234 static constexpr std::size_t speedup {gemm_kernel_speedup / 16};
3237 static constexpr std::size_t serial_below {std::size_t {140608}};
3238
3239 struct session
3240 {};
3241
3242 static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
3243 {
3244 return ldb <= nr;
3245 }
3246
3247 static void pack_a(const std::complex<float>* a,
3248 std::size_t lda,
3249 std::size_t rows,
3250 std::size_t kc,
3251 std::complex<float>* out)
3252 {
3253 gemm_pack_a<mr>(a, lda, rows, kc, out);
3254 }
3255
3256 static void tile(std::size_t kc,
3257 const std::complex<float>* ap,
3258 const std::complex<float>* bp,
3259 std::size_t ldb,
3260 std::complex<float>* c,
3261 std::size_t ldc,
3262 std::size_t rows,
3263 std::size_t cols,
3264 bool fresh = false) noexcept
3265 {
3266 avx2_tile_c32(kc, ap, bp, ldb, c, ldc, rows, cols, fresh);
3267 }
3268
3271 static void tile_a_in_place(std::size_t kc,
3272 const std::complex<float>* a,
3273 std::size_t a_step,
3274 std::size_t a_row,
3275 const std::complex<float>* bp,
3276 std::size_t ldb,
3277 std::complex<float> * c,
3278 std::size_t ldc,
3279 std::size_t rows,
3280 std::size_t cols,
3281 bool fresh = false) noexcept
3282 {
3283 avx2_tile_c32_a_in_place(kc, a, a_step, a_row, bp, ldb, c, ldc, rows, cols, fresh);
3284 }
3285 };
3286
3287#endif
3288
3290 template <typename T>
3291 [[nodiscard]] bool gemm_elsewhere(std::size_t m,
3292 std::size_t n,
3293 std::size_t k,
3294 const T* a,
3295 std::size_t lda,
3296 const T* b,
3297 std::size_t ldb,
3298 T* c,
3299 std::size_t ldc,
3300 c_start start = c_start::kept)
3301 {
3302 (void)m, (void)n, (void)k, (void)a, (void)lda, (void)b, (void)ldb, (void)c, (void)ldc, (void)start;
3303 return false;
3304 }
3305
3309 template <typename B,
3310 typename A = a_rows>
3311 [[nodiscard]] bool gemm_double_elsewhere(std::size_t m,
3312 std::size_t n,
3313 std::size_t k,
3314 const double* a,
3315 std::size_t lda,
3316 const double* b,
3317 std::size_t ldb,
3318 double* c,
3319 std::size_t ldc,
3320 c_start start = c_start::kept)
3321 {
3323#ifdef NUMERICAL_AMX_COMPILED
3324 if (path == gemm_path::amx && amx_suits(m, n, k)) { // else the NEON tile
3325 gemm_packed<amx_tile_kernel, double, B, A>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
3326 return true;
3327 }
3328#endif
3329#ifdef NUMERICAL_NEON_COMPILED
3330 if (path == gemm_path::neon || path == gemm_path::amx) {
3331 gemm_packed<neon_tile_kernel<double>, double, B, A>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
3332 return true;
3333 }
3334#endif
3335#ifdef NUMERICAL_X86_COMPILED
3336 if (path == gemm_path::avx512) {
3337 gemm_packed<avx512_tile_kernel, double, B, A>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
3338 return true;
3339 }
3340 if (path == gemm_path::avx2) {
3341 gemm_packed<avx2_tile_kernel, double, B, A>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
3342 return true;
3343 }
3344#endif
3345 (void)path;
3346 (void)m, (void)n, (void)k, (void)a, (void)lda, (void)b, (void)ldb, (void)c, (void)ldc, (void)start;
3347 return false;
3348 }
3349
3354 template <typename A,
3355 typename B>
3356 void gemm_triangle(std::size_t n,
3357 std::size_t k,
3358 const double* a,
3359 std::size_t lda,
3360 const double* b,
3361 std::size_t ldb,
3362 double* c,
3363 std::size_t ldc,
3364 bool lower)
3365 {
3367#ifdef NUMERICAL_AMX_COMPILED
3368 if (path == gemm_path::amx && amx_suits(n, n, k)) {
3369 gemm_packed_triangle<amx_tile_kernel, double, B, A>(n, k, a, lda, b, ldb, c, ldc, lower);
3370 return;
3371 }
3372#endif
3373#ifdef NUMERICAL_NEON_COMPILED
3374 if (path == gemm_path::neon || path == gemm_path::amx) {
3375 gemm_packed_triangle<neon_tile_kernel<double>, double, B, A>(n, k, a, lda, b, ldb, c, ldc, lower);
3376 return;
3377 }
3378#endif
3379#ifdef NUMERICAL_X86_COMPILED
3380 if (path == gemm_path::avx512) {
3381 gemm_packed_triangle<avx512_tile_kernel, double, B, A>(n, k, a, lda, b, ldb, c, ldc, lower);
3382 return;
3383 }
3384 if (path == gemm_path::avx2) {
3385 gemm_packed_triangle<avx2_tile_kernel, double, B, A>(n, k, a, lda, b, ldb, c, ldc, lower);
3386 return;
3387 }
3388#endif
3389 (void)path;
3390 gemm_packed_triangle<vector_tile_kernel<double>, double, B, A>(n, k, a, lda, b, ldb, c, ldc, lower);
3391 }
3392
3397 template <typename A,
3398 typename B>
3399 void gemm_triangle(std::size_t n,
3400 std::size_t k,
3401 const float* a,
3402 std::size_t lda,
3403 const float* b,
3404 std::size_t ldb,
3405 float* c,
3406 std::size_t ldc,
3407 bool lower)
3408 {
3409#ifdef NUMERICAL_X86_COMPILED
3411 if (path == gemm_path::avx512) {
3412 gemm_packed_triangle<avx512_tile_kernel_f32, float, B, A>(n, k, a, lda, b, ldb, c, ldc, lower);
3413 return;
3414 }
3415 if (path == gemm_path::avx2) {
3416 gemm_packed_triangle<avx2_tile_kernel_f32, float, B, A>(n, k, a, lda, b, ldb, c, ldc, lower);
3417 return;
3418 }
3419#endif
3420 gemm_packed_triangle<vector_tile_kernel<float>, float, B, A>(n, k, a, lda, b, ldb, c, ldc, lower);
3421 }
3422
3424 [[nodiscard]] inline bool gemm_elsewhere(std::size_t m,
3425 std::size_t n,
3426 std::size_t k,
3427 const double* a,
3428 std::size_t lda,
3429 const double* b,
3430 std::size_t ldb,
3431 double* c,
3432 std::size_t ldc,
3433 c_start start = c_start::kept)
3434 {
3435 return gemm_double_elsewhere<b_rows>(m, n, k, a, lda, b, ldb, c, ldc, start);
3436 }
3437
3441 template <typename B,
3442 typename A = a_rows>
3443 [[nodiscard]] bool gemm_float_elsewhere(std::size_t m,
3444 std::size_t n,
3445 std::size_t k,
3446 const float* a,
3447 std::size_t lda,
3448 const float* b,
3449 std::size_t ldb,
3450 float* c,
3451 std::size_t ldc,
3452 c_start start = c_start::kept)
3453 {
3454#ifdef NUMERICAL_NEON_COMPILED
3456# ifdef NUMERICAL_AMX_COMPILED
3457 if (path == gemm_path::amx && amx_suits_f32(m, n, k)) {
3458 gemm_packed<amx_tile_kernel_f32, float, B, A>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
3459 return true;
3460 }
3461# endif
3462 if (path == gemm_path::neon || path == gemm_path::amx) {
3463 gemm_packed<neon_tile_kernel<float>, float, B, A>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
3464 return true;
3465 }
3466#endif
3467#ifdef NUMERICAL_X86_COMPILED
3469 if (x86_path == gemm_path::avx512) {
3470 gemm_packed<avx512_tile_kernel_f32, float, B, A>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
3471 return true;
3472 }
3473 if (x86_path == gemm_path::avx2) {
3474 gemm_packed<avx2_tile_kernel_f32, float, B, A>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
3475 return true;
3476 }
3477#endif
3478 (void)m, (void)n, (void)k, (void)a, (void)lda, (void)b, (void)ldb, (void)c, (void)ldc, (void)start;
3479 return false;
3480 }
3481
3483 [[nodiscard]] inline bool gemm_elsewhere(std::size_t m,
3484 std::size_t n,
3485 std::size_t k,
3486 const float* a,
3487 std::size_t lda,
3488 const float* b,
3489 std::size_t ldb,
3490 float* c,
3491 std::size_t ldc,
3492 c_start start = c_start::kept)
3493 {
3494 return gemm_float_elsewhere<b_rows>(m, n, k, a, lda, b, ldb, c, ldc, start);
3495 }
3496
3497 template <typename T>
3498 void gemm(std::size_t m,
3499 std::size_t n,
3500 std::size_t k,
3501 const T* a,
3502 std::size_t lda,
3503 const T* b,
3504 std::size_t ldb,
3505 T* c,
3506 std::size_t ldc,
3507 c_start start = c_start::kept);
3508
3515 template <std::size_t Width, term_rounding Mode, typename T>
3516 void few_rows(std::size_t m,
3517 std::size_t n,
3518 std::size_t k,
3519 const T* a,
3520 std::size_t lda,
3521 const T* b,
3522 std::size_t ldb,
3523 T* c,
3524 std::size_t ldc,
3525 c_start start)
3526 {
3527 static_assert(narrow_group == 4, "the switches below name each count of rows up to narrow_group");
3528 const std::size_t groups {(m + narrow_group - 1) / narrow_group};
3529 const std::size_t blocks {n / Width};
3530 const auto block_of {[&](std::size_t first, std::size_t j) {
3531 const T* const rows {a + (first * lda)};
3532 T* const out {c + (first * ldc) + j};
3533 switch (std::min(narrow_group, m - first)) {
3534 case 1: narrow_block<1, Width, Mode>(k, rows, lda, b + j, ldb, out, ldc, start); break;
3535 case 2: narrow_block<2, Width, Mode>(k, rows, lda, b + j, ldb, out, ldc, start); break;
3536 case 3: narrow_block<3, Width, Mode>(k, rows, lda, b + j, ldb, out, ldc, start); break;
3537 default: narrow_block<4, Width, Mode>(k, rows, lda, b + j, ldb, out, ldc, start); break;
3538 }
3539 }};
3540 parallel_for(groups * blocks, narrow_group * Width * k, [&](std::size_t task) {
3541 block_of((task / blocks) * narrow_group, (task % blocks) * Width);
3542 });
3543 if (n % Width != 0) {
3544 const std::size_t j {blocks * Width};
3545 for (std::size_t first {0}; first < m; first += narrow_group) {
3546 const T* const rows {a + (first * lda)};
3547 T* const out {c + (first * ldc) + j};
3548 switch (std::min(narrow_group, m - first)) {
3549 case 1: narrow_rows<1, Mode>(n % Width, k, rows, lda, b + j, ldb, out, ldc, start); break;
3550 case 2: narrow_rows<2, Mode>(n % Width, k, rows, lda, b + j, ldb, out, ldc, start); break;
3551 case 3: narrow_rows<3, Mode>(n % Width, k, rows, lda, b + j, ldb, out, ldc, start); break;
3552 default: narrow_rows<4, Mode>(n % Width, k, rows, lda, b + j, ldb, out, ldc, start); break;
3553 }
3554 }
3555 }
3556 }
3557
3558#ifdef NUMERICAL_X86_COMPILED
3563 template <std::size_t R>
3564 inline constexpr std::size_t few_rows_columns {R == 1 ? 32 : (R == 2 ? 16 : (R == 3 ? 12 : 8))};
3565
3572 template <std::size_t R, typename T>
3573 void few_rows_x86(std::size_t n,
3574 std::size_t k,
3575 const T* a,
3576 std::size_t lda,
3577 const T* b,
3578 std::size_t ldb,
3579 T* c,
3580 std::size_t ldc,
3581 c_start start)
3582 {
3583 few_rows<few_rows_columns<R>, term_rounding::x86_fused>(R, n, k, a, lda, b, ldb, c, ldc, start);
3584 }
3585
3592 inline constexpr std::size_t few_rows_complex_least {std::size_t {1} << 16};
3593
3595 template <std::size_t R>
3596 inline constexpr std::size_t few_rows_complex_columns {R <= 2 ? 256 : 128};
3597
3605 template <typename Isa,
3606 std::size_t R,
3607 typename Z>
3608 NUMERICAL_ALWAYS_INLINE void few_rows_complex_chunk(std::size_t depth,
3609 std::size_t w,
3610 std::size_t k,
3611 const Z* a,
3612 std::size_t lda,
3613 const Z* b,
3614 std::size_t ldb,
3615 Z* c,
3616 std::size_t ldc,
3617 c_start start,
3618 typename Z::value_type* sums) noexcept
3619 {
3620 using P = typename Z::value_type;
3621 using vec = fft_reg<Isa, P>;
3622 constexpr Isa isa {};
3623 constexpr std::size_t width {sizeof(vec) / sizeof(P)}; // parts in a register
3624 const std::size_t parts {2 * w};
3625 const std::size_t whole {parts - (parts % width)};
3626 const auto* const bs {reinterpret_cast<const P*>(b)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is the array of its two parts ([complex.numbers])
3627 const auto* const as {reinterpret_cast<const P*>(a)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
3628 auto* const cs {reinterpret_cast<P*>(c)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
3629 const auto real_sums {[&](std::size_t r) noexcept { return sums + (2 * r * parts); }};
3630 const auto imaginary_sums {[&](std::size_t r) noexcept { return sums + (((2 * r) + 1) * parts); }};
3631 for (std::size_t pc {0}; pc < k; pc += depth) {
3632 const std::size_t kc {std::min(depth, k - pc)};
3633 std::fill(sums, sums + (2 * R * parts), P {});
3634 std::size_t p {0};
3635 for (; p + 2 <= kc; p += 2) { // two steps of the depth to each pass over the sums
3636 const P* const row0 {bs + (2 * (pc + p) * ldb)};
3637 const P* const row1 {row0 + (2 * ldb)};
3638 for (std::size_t r {0}; r < R; ++r) {
3639 const P* const ar {as + (2 * ((r * lda) + pc + p))};
3640 const vec re0 {fft_splat(isa, ar[0])};
3641 const vec im0 {fft_splat(isa, ar[1])};
3642 const vec re1 {fft_splat(isa, ar[2])};
3643 const vec im1 {fft_splat(isa, ar[3])};
3644 P* const rs {real_sums(r)};
3645 P* const is {imaginary_sums(r)};
3646 for (std::size_t j {0}; j < whole; j += width) {
3647 const vec v0 {fft_load(isa, row0 + j)};
3648 const vec v1 {fft_load(isa, row1 + j)};
3649 fft_store(rs + j, fft_fused(re1, v1, fft_fused(re0, v0, fft_load(isa, rs + j))));
3650 fft_store(is + j, fft_fused(im1, v1, fft_fused(im0, v0, fft_load(isa, is + j))));
3651 }
3652 for (std::size_t j {whole}; j < parts; ++j) {
3653 rs[j] = std::fma(ar[2], row1[j], std::fma(ar[0], row0[j], rs[j]));
3654 is[j] = std::fma(ar[3], row1[j], std::fma(ar[1], row0[j], is[j]));
3655 }
3656 }
3657 }
3658 if (p < kc) { // the panel's last step
3659 const P* const row0 {bs + (2 * (pc + p) * ldb)};
3660 for (std::size_t r {0}; r < R; ++r) {
3661 const P* const ar {as + (2 * ((r * lda) + pc + p))};
3662 const vec re0 {fft_splat(isa, ar[0])};
3663 const vec im0 {fft_splat(isa, ar[1])};
3664 P* const rs {real_sums(r)};
3665 P* const is {imaginary_sums(r)};
3666 for (std::size_t j {0}; j < whole; j += width) {
3667 const vec v0 {fft_load(isa, row0 + j)};
3668 fft_store(rs + j, fft_fused(re0, v0, fft_load(isa, rs + j)));
3669 fft_store(is + j, fft_fused(im0, v0, fft_load(isa, is + j)));
3670 }
3671 for (std::size_t j {whole}; j < parts; ++j) {
3672 rs[j] = std::fma(ar[0], row0[j], rs[j]);
3673 is[j] = std::fma(ar[1], row0[j], is[j]);
3674 }
3675 }
3676 }
3677 const bool fresh {start == c_start::zero && pc == 0}; // C holds nothing yet: 0 + the sum, as the tile stores it
3678 const vec zero {fft_splat(isa, P {})};
3679 for (std::size_t r {0}; r < R; ++r) {
3680 const P* const rs {real_sums(r)};
3681 const P* const is {imaginary_sums(r)};
3682 P* const out {cs + (2 * r * ldc)};
3683 for (std::size_t j {0}; j < whole; j += width) {
3684 const vec sum {fft_add(fft_load(isa, rs + j), fft_times_i(fft_load(isa, is + j)))};
3685 fft_store(out + j, fft_add(fresh ? zero : fft_load(isa, out + j), sum));
3686 }
3687 for (std::size_t j {whole}; j < parts; j += 2) {
3688 const P real_part {rs[j] - is[j + 1]};
3689 const P imaginary_part {rs[j + 1] + is[j]};
3690 out[j] = (fresh ? P {} : out[j]) + real_part;
3691 out[j + 1] = (fresh ? P {} : out[j + 1]) + imaginary_part;
3692 }
3693 }
3694 }
3695 }
3696
3698 template <std::size_t R,
3699 typename Z>
3700 NUMERICAL_TARGET("avx2,fma") void few_rows_complex_avx2(std::size_t depth,
3701 std::size_t w,
3702 std::size_t k,
3703 const Z* a,
3704 std::size_t lda,
3705 const Z* b,
3706 std::size_t ldb,
3707 Z* c,
3708 std::size_t ldc,
3709 c_start start) noexcept
3710 {
3711 alignas(64) std::array<typename Z::value_type, 4 * R * few_rows_complex_columns<R>> sums; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): each panel zeroes the parts it sums
3712 few_rows_complex_chunk<fft_avx2, R>(depth, w, k, a, lda, b, ldb, c, ldc, start, sums.data());
3713 _mm256_zeroupper(); // the upper halves cleared: code past the chunk may be SSE alone
3714 }
3715
3717 template <std::size_t R,
3718 typename Z>
3719 NUMERICAL_TARGET("avx512f,avx2,fma") void few_rows_complex_avx512(std::size_t depth,
3720 std::size_t w,
3721 std::size_t k,
3722 const Z* a,
3723 std::size_t lda,
3724 const Z* b,
3725 std::size_t ldb,
3726 Z* c,
3727 std::size_t ldc,
3728 c_start start) noexcept
3729 {
3730 alignas(64) std::array<typename Z::value_type, 4 * R * few_rows_complex_columns<R>> sums; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): each panel zeroes the parts it sums
3731 few_rows_complex_chunk<fft_avx512, R>(depth, w, k, a, lda, b, ldb, c, ldc, start, sums.data());
3732 _mm256_zeroupper(); // the upper halves cleared: code past the chunk may be SSE alone
3733 }
3734
3742 template <std::size_t R,
3743 split_complex Z>
3744 void few_rows_complex_x86(std::size_t n,
3745 std::size_t k,
3746 const Z* a,
3747 std::size_t lda,
3748 const Z* b,
3749 std::size_t ldb,
3750 Z* c,
3751 std::size_t ldc,
3752 c_start start)
3753 {
3754 constexpr bool doubles {std::is_same_v<Z, std::complex<double>>};
3755 const bool wide {gemm_path_current() == gemm_path::avx512};
3756 const std::size_t depth {doubles ? avx512_complex_tile_kernel::depth : avx512_complex_tile_kernel_f32::depth};
3757 // A chunk a thread at least, of whole registers' columns, and at most what the stack holds of sums.
3758 const std::size_t share {(n + max_threads() - 1) / max_threads()};
3759 const std::size_t width {std::clamp((share + 15) / 16 * 16, std::size_t {64}, few_rows_complex_columns<R>)};
3760 const std::size_t chunks {(n + width - 1) / width};
3761 parallel_for(chunks, 4 * R * width * k, [&](std::size_t t) {
3762 const std::size_t j {t * width};
3763 const std::size_t w {std::min(width, n - j)};
3764 if (wide) {
3765 few_rows_complex_avx512<R>(depth, w, k, a, lda, b + j, ldb, c + j, ldc, start);
3766 }
3767 else {
3768 few_rows_complex_avx2<R>(depth, w, k, a, lda, b + j, ldb, c + j, ldc, start);
3769 }
3770 });
3771 }
3772
3773#endif
3774
3775#ifdef NUMERICAL_NEON_COMPILED
3785 template <typename T>
3786 requires std::same_as<T, float>
3787 [[nodiscard]] constexpr bool few_rows_on_arm64(std::size_t m) noexcept
3788 {
3789 return m == 1 || (m > narrow_group && m < 16);
3790 }
3791
3793 template <typename T>
3794 requires std::same_as<T, double>
3795 [[nodiscard]] constexpr bool few_rows_on_arm64(std::size_t m) noexcept
3796 {
3797 return m <= narrow_group;
3798 }
3799
3801 template <typename T>
3802 concept few_rows_real = std::same_as<T, float> || std::same_as<T, double>;
3803
3805 template <typename T>
3806 inline constexpr std::size_t few_rows_neon_columns {std::is_same_v<T, float> ? 16 : 8};
3807
3815 template <typename T>
3816 requires std::same_as<T, float>
3817 [[nodiscard]] constexpr bool few_rows_left_to_amx(std::size_t m,
3818 std::size_t n,
3819 std::size_t k) noexcept
3820 {
3821 return n >= 32 && m * n * k >= (std::size_t {1} << 16U);
3822 }
3823
3826 template <typename T>
3827 requires std::same_as<T, double>
3828 [[nodiscard]] constexpr bool few_rows_left_to_amx(std::size_t /*m*/,
3829 std::size_t /*n*/,
3830 std::size_t /*k*/) noexcept
3831 {
3832 return false;
3833 }
3834
3836 template <few_rows_real T>
3837 [[nodiscard]] bool gemm_few_rows_arm64(std::size_t m,
3838 std::size_t n,
3839 std::size_t k,
3840 const T* a,
3841 std::size_t lda,
3842 const T* b,
3843 std::size_t ldb,
3844 T* c,
3845 std::size_t ldc,
3846 c_start start)
3847 {
3850 || (path == gemm_path::amx && few_rows_left_to_amx<T>(m, n, k))) {
3851 return false;
3852 }
3853 few_rows<few_rows_neon_columns<T>, term_rounding::fused>(m, n, k, a, lda, b, ldb, c, ldc, start);
3854 return true;
3855 }
3856
3858 template <typename T>
3859 [[nodiscard]] constexpr bool gemm_few_rows_arm64(std::size_t /*m*/,
3860 std::size_t /*n*/,
3861 std::size_t /*k*/,
3862 const T* /*a*/,
3863 std::size_t /*lda*/,
3864 const T* /*b*/,
3865 std::size_t /*ldb*/,
3866 T* /*c*/,
3867 std::size_t /*ldc*/,
3868 c_start /*start*/) noexcept
3869 {
3870 return false;
3871 }
3872
3873#endif
3874
3877 inline constexpr std::size_t split_columns {512};
3878
3880 inline constexpr std::size_t split_depth {256};
3881
3882#ifndef NUMERICAL_X86_COMPILED
3884 inline constexpr std::size_t split_row_width {32};
3885
3894 template <split_complex C>
3895 void split_row_columns(const C* a,
3896 const C* b,
3897 std::size_t row_step,
3898 std::size_t column_step,
3899 std::size_t inner,
3900 std::size_t first,
3901 std::size_t last,
3902 C* y,
3903 bool kept = false)
3904 {
3905 using R = typename C::value_type;
3906 const std::size_t step {2 * column_step}; // between the parts of a row's columns
3907 for (std::size_t j {first}; j < last; j += split_row_width) {
3908 const std::size_t count {std::min(split_row_width, last - j)};
3909 std::array<R, 2 * split_row_width> sums {}; // the columns' parts, interleaved
3910 for (std::size_t c {0}; kept && c < count; ++c) {
3911 sums[2 * c] = y[j + c].real();
3912 sums[(2 * c) + 1] = y[j + c].imag();
3913 }
3914 for (std::size_t pc {0}; pc < inner; pc += split_depth) {
3915 const std::size_t end {std::min(pc + split_depth, inner)};
3916 for (std::size_t k {pc}; k < end; ++k) {
3917 const R f {a[k].real()};
3918 const R* const row {reinterpret_cast<const R*>(b + (k * row_step) + (j * column_step))}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is its two parts ([complex.numbers])
3919 for (std::size_t c {0}; c < count; ++c) {
3920 sums[2 * c] = std::fma(f, row[c * step], sums[2 * c]);
3921 sums[(2 * c) + 1] = std::fma(f, row[(c * step) + 1], sums[(2 * c) + 1]);
3922 }
3923 }
3924 for (std::size_t k {pc}; k < end; ++k) {
3925 const R g {a[k].imag()};
3926 const R* const row {reinterpret_cast<const R*>(b + (k * row_step) + (j * column_step))}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
3927 for (std::size_t c {0}; c < count; ++c) {
3928 sums[2 * c] = std::fma(-g, row[(c * step) + 1], sums[2 * c]);
3929 sums[(2 * c) + 1] = std::fma(g, row[c * step], sums[(2 * c) + 1]);
3930 }
3931 }
3932 }
3933 for (std::size_t c {0}; c < count; ++c) {
3934 y[j + c] = {sums[2 * c], sums[(2 * c) + 1]};
3935 }
3936 }
3937 }
3938
3939#endif
3940
3941#ifdef NUMERICAL_NEON_COMPILED
3943 inline float64x2_t neon_splat(double v) noexcept
3944 {
3945 return vdupq_n_f64(v);
3946 }
3947
3949 inline float32x4_t neon_splat(float v) noexcept
3950 {
3951 return vdupq_n_f32(v);
3952 }
3953
3955 inline float64x2_t neon_crossed(double g) noexcept
3956 {
3957 return vsetq_lane_f64(-g, vdupq_n_f64(g), 0);
3958 }
3959
3961 inline float32x4_t neon_crossed(float g) noexcept
3962 {
3963 return vtrn1q_f32(vdupq_n_f32(-g), vdupq_n_f32(g));
3964 }
3965
3967 inline float64x2_t neon_exchanged(float64x2_t v) noexcept
3968 {
3969 return vextq_f64(v, v, 1);
3970 }
3971
3973 inline float32x4_t neon_exchanged(float32x4_t v) noexcept
3974 {
3975 return vrev64q_f32(v);
3976 }
3977
3979 inline float64x2_t neon_load(const double* p) noexcept
3980 {
3981 return vld1q_f64(p);
3982 }
3983
3985 inline float32x4_t neon_load(const float* p) noexcept
3986 {
3987 return vld1q_f32(p);
3988 }
3989
3991 inline void neon_store(double* p,
3992 float64x2_t v) noexcept
3993 {
3994 vst1q_f64(p, v);
3995 }
3996
3998 inline void neon_store(float* p,
3999 float32x4_t v) noexcept
4000 {
4001 vst1q_f32(p, v);
4002 }
4003
4006 float64x2_t f,
4007 float64x2_t v) noexcept
4008 {
4009 return vfmaq_f64(s, f, v);
4010 }
4011
4014 float32x4_t f,
4015 float32x4_t v) noexcept
4016 {
4017 return vfmaq_f32(s, f, v);
4018 }
4019
4026 template <std::size_t Registers,
4027 typename R>
4028 void neon_split_row_parts(const std::complex<R>* a,
4029 const R* b,
4030 std::size_t row_step,
4031 std::size_t pc,
4032 std::size_t end,
4033 bool first,
4034 std::size_t p,
4035 R* y) noexcept
4036 {
4037 using vec = decltype(neon_load(b));
4038 constexpr std::size_t lanes {16 / sizeof(R)};
4039 std::array<vec, Registers> sums {};
4040 if (!first) {
4041# pragma GCC unroll 16
4042 for (std::size_t t {0}; t < Registers; ++t) {
4043 sums[t] = neon_load(y + p + (t * lanes));
4044 }
4045 }
4046 for (std::size_t k {pc}; k < end; ++k) {
4047 const vec f {neon_splat(a[k].real())};
4048 const R* const row {b + (k * row_step) + p};
4049# pragma GCC unroll 16
4050 for (std::size_t t {0}; t < Registers; ++t) {
4051 sums[t] = neon_fused(sums[t], f, neon_load(row + (t * lanes)));
4052 }
4053 }
4054 for (std::size_t k {pc}; k < end; ++k) {
4055 const vec g {neon_crossed(a[k].imag())};
4056 const R* const row {b + (k * row_step) + p};
4057# pragma GCC unroll 16
4058 for (std::size_t t {0}; t < Registers; ++t) {
4059 sums[t] = neon_fused(sums[t], g, neon_exchanged(neon_load(row + (t * lanes))));
4060 }
4061 }
4062# pragma GCC unroll 16
4063 for (std::size_t t {0}; t < Registers; ++t) {
4064 neon_store(y + p + (t * lanes), sums[t]);
4065 }
4066 }
4067
4071 template <split_complex C>
4072 void neon_split_row_columns(const C* a,
4073 const C* b,
4074 std::size_t row_step,
4075 std::size_t inner,
4076 std::size_t first,
4077 std::size_t last,
4078 C* y,
4079 bool kept = false) noexcept
4080 {
4081 using R = typename C::value_type;
4082 constexpr std::size_t lanes {16 / sizeof(R)};
4083 const R* const parts {reinterpret_cast<const R*>(b + first)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is its two parts ([complex.numbers])
4084 R* const out {reinterpret_cast<R*>(y + first)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the same
4085 const std::size_t count {2 * (last - first)};
4086 const std::size_t whole {(count / lanes) * lanes}; // the parts in whole registers
4087 for (std::size_t pc {0}; pc < inner; pc += split_depth) {
4088 const std::size_t end {std::min(pc + split_depth, inner)};
4089 const bool starting {pc == 0 && !kept};
4090 std::size_t p {0};
4091 for (; p + (16 * lanes) <= count; p += 16 * lanes) {
4092 neon_split_row_parts<16>(a, parts, 2 * row_step, pc, end, starting, p, out);
4093 }
4094 for (; p + (4 * lanes) <= count; p += 4 * lanes) {
4095 neon_split_row_parts<4>(a, parts, 2 * row_step, pc, end, starting, p, out);
4096 }
4097 switch ((whole - p) / lanes) { // the registers left, their chains side by side
4098 case 3: neon_split_row_parts<3>(a, parts, 2 * row_step, pc, end, starting, p, out); break;
4099 case 2: neon_split_row_parts<2>(a, parts, 2 * row_step, pc, end, starting, p, out); break;
4100 case 1: neon_split_row_parts<1>(a, parts, 2 * row_step, pc, end, starting, p, out); break;
4101 default: break; // none
4102 }
4103 }
4104 split_row_columns(a, b, row_step, 1, inner, first + (whole / 2), last, y, kept);
4105 }
4106
4108 inline constexpr std::size_t few_rows_complex_neon_columns {256};
4109
4118 template <split_complex Z>
4119 void few_rows_complex_neon(std::size_t m,
4120 std::size_t n,
4121 std::size_t k,
4122 const Z* a,
4123 std::size_t lda,
4124 const Z* b,
4125 std::size_t ldb,
4126 Z* c,
4127 std::size_t ldc,
4128 c_start start)
4129 {
4130 const std::size_t chunks {(n + few_rows_complex_neon_columns - 1) / few_rows_complex_neon_columns};
4131 parallel_for(chunks, 4 * m * k * few_rows_complex_neon_columns, [&](std::size_t chunk) {
4132 const std::size_t first {chunk * few_rows_complex_neon_columns};
4133 const std::size_t last {std::min(n, first + few_rows_complex_neon_columns)};
4134 for (std::size_t i {0}; i < m; ++i) {
4135 Z* const row {c + (i * ldc)};
4136 if (start == c_start::zero) {
4137 std::fill(row + first, row + last, Z {});
4138 }
4139 neon_split_row_columns(a + (i * lda), b, ldb, k, first, last, row, true);
4140 }
4141 });
4142 }
4143
4144#endif
4145
4148 inline constexpr std::size_t split_kept {std::size_t {1} << 18};
4149
4154 inline constexpr std::size_t split_parts_rows {64};
4155
4163 [[nodiscard]] inline bool split_by_parts(std::size_t m) noexcept
4164 {
4166#ifdef NUMERICAL_AMX_COMPILED
4167 if (path == gemm_path::amx) {
4168 return m <= split_parts_rows;
4169 }
4170#endif
4171 return m <= gemm_mr;
4172 }
4173
4179 inline constexpr std::size_t split_narrow_area {1024};
4180
4184 template <typename R>
4191
4196 template <typename R>
4198 {
4199 thread_local split_operands<R> kept;
4200 [[maybe_unused]] thread_local const bool registered {on_release([] { kept = split_operands<R> {}; })};
4201 return kept;
4202 }
4203
4211 template <split_complex Z,
4212 typename B = b_rows,
4213 typename A = a_rows>
4214 void split_parts_block(std::size_t m,
4215 std::size_t nb,
4216 std::size_t kb,
4217 std::size_t pc,
4218 std::size_t jc,
4219 const Z* a,
4220 std::size_t lda,
4221 const Z* b,
4222 std::size_t ldb,
4223 typename Z::value_type* ar,
4224 typename Z::value_type* br,
4225 typename Z::value_type* cr)
4226 {
4227 using R = typename Z::value_type;
4228 parallel_for(m, kb, [&](std::size_t i) {
4229 R* const row {ar + (i * 3 * kb)};
4230 for (std::size_t p {0}; p < kb; ++p) {
4231 const Z value {A::at(a, lda, i, pc + p)};
4232 row[p] = value.real();
4233 row[kb + p] = -value.imag();
4234 row[(2 * kb) + p] = value.imag();
4235 }
4236 });
4237 parallel_for(kb, nb, [&](std::size_t p) {
4238 R* const row {br + (p * 2 * nb)};
4239 for (std::size_t j {0}; j < nb; ++j) {
4240 const Z value {B::at(b, ldb, pc + p, jc + j)};
4241 row[j] = value.real();
4242 row[nb + j] = value.imag();
4243 }
4244 });
4245 gemm(m, nb, kb, ar, 3 * kb, br, 2 * nb, cr, 2 * nb); // Re C += Re A·Re B
4246 gemm(m, nb, kb, ar + kb, 3 * kb, br + nb, 2 * nb, cr, 2 * nb); // Re C += −Im A·Im B
4247 gemm(m, nb, kb, ar, 3 * kb, br + nb, 2 * nb, cr + nb, 2 * nb); // Im C += Re A·Im B
4248 gemm(m, nb, kb, ar + (2 * kb), 3 * kb, br, 2 * nb, cr + nb, 2 * nb); // Im C += Im A·Re B
4249 }
4250
4256 template <split_complex Z,
4257 typename B = b_rows,
4258 typename A = a_rows>
4259 void split_expanded_block(std::size_t m,
4260 std::size_t nb,
4261 std::size_t kb,
4262 std::size_t pc,
4263 std::size_t jc,
4264 const Z* a,
4265 std::size_t lda,
4266 const Z* b,
4267 std::size_t ldb,
4268 typename Z::value_type* ar,
4269 typename Z::value_type* br,
4270 typename Z::value_type* cr)
4271 {
4272 using R = typename Z::value_type;
4273 parallel_for(m, kb, [&](std::size_t i) {
4274 for (std::size_t p {0}; p < kb; ++p) {
4275 const Z value {A::at(a, lda, i, pc + p)};
4276 ar[(i * 2 * kb) + p] = value.real();
4277 ar[(i * 2 * kb) + kb + p] = -value.imag();
4278 }
4279 });
4280 parallel_for(kb, 2 * nb, [&](std::size_t p) {
4281 R* const top {br + (p * 2 * nb)};
4282 R* const bottom {br + ((kb + p) * 2 * nb)};
4283 for (std::size_t j {0}; j < nb; ++j) {
4284 const Z value {B::at(b, ldb, pc + p, jc + j)};
4285 top[j] = value.real();
4286 top[nb + j] = value.imag();
4287 bottom[j] = value.imag();
4288 bottom[nb + j] = -value.real();
4289 }
4290 });
4291 gemm(m, 2 * nb, 2 * kb, ar, 2 * kb, br, 2 * nb, cr, 2 * nb);
4292 }
4293
4302 template <split_complex Z,
4303 typename B = b_rows,
4304 typename A = a_rows>
4305 void gemm_split(std::size_t m,
4306 std::size_t n,
4307 std::size_t k,
4308 const Z* a,
4309 std::size_t lda,
4310 const Z* b,
4311 std::size_t ldb,
4312 Z* c,
4313 std::size_t ldc)
4314 {
4315 using R = typename Z::value_type;
4316 const bool parts {split_by_parts(m)};
4317 const std::size_t widest {std::min(n, split_columns)};
4318 const std::size_t deepest {std::min(k, split_depth)};
4320 kept.a.resize(std::max(kept.a.size(), m * (parts ? 3 : 2) * deepest)); // grown only, written before they are read
4321 kept.b.resize(std::max(kept.b.size(), (parts ? 2 : 4) * deepest * widest));
4322 kept.c.resize(std::max(kept.c.size(), m * 2 * widest));
4323 R* const ar {kept.a.data()};
4324 R* const br {kept.b.data()};
4325 R* const cr {kept.c.data()};
4326 for (std::size_t jc {0}; jc < n; jc += split_columns) {
4327 const std::size_t nb {std::min(split_columns, n - jc)};
4328 parallel_for(m, nb, [&](std::size_t i) {
4329 for (std::size_t j {0}; j < nb; ++j) {
4330 cr[(i * 2 * nb) + j] = c[(i * ldc) + jc + j].real();
4331 cr[(i * 2 * nb) + nb + j] = c[(i * ldc) + jc + j].imag();
4332 }
4333 });
4334 for (std::size_t pc {0}; pc < k; pc += split_depth) {
4335 const std::size_t kb {std::min(split_depth, k - pc)};
4336 if (parts) {
4337 split_parts_block<Z, B, A>(m, nb, kb, pc, jc, a, lda, b, ldb, ar, br, cr);
4338 }
4339 else {
4340 split_expanded_block<Z, B, A>(m, nb, kb, pc, jc, a, lda, b, ldb, ar, br, cr);
4341 }
4342 }
4343 parallel_for(m, nb, [&](std::size_t i) {
4344 for (std::size_t j {0}; j < nb; ++j) {
4345 c[(i * ldc) + jc + j] = Z {cr[(i * 2 * nb) + j], cr[(i * 2 * nb) + nb + j]};
4346 }
4347 });
4348 }
4349 for (buffer<R>* operand : {&kept.a, &kept.b, &kept.c}) {
4350 if (operand->size() > split_kept) {
4351 *operand = buffer<R> {};
4352 }
4353 }
4354 }
4355
4360 inline constexpr std::size_t plain_loop_least_width {64};
4361
4372 [[nodiscard]] inline bool plain_loop_pays(std::size_t m,
4373 std::size_t n,
4374 std::size_t k) noexcept
4375 {
4377#ifdef NUMERICAL_NEON_COMPILED
4378 bool neon_tile {path == gemm_path::neon};
4379# ifdef NUMERICAL_AMX_COMPILED
4380 neon_tile = neon_tile || (path == gemm_path::amx && !amx_outruns_plain_loop(m, n, k)); // the NEON tile's rule below
4381# endif
4382 if (neon_tile) {
4383 return k <= 16 || (m <= 64 && k <= 64);
4384 }
4385#endif
4386#ifdef NUMERICAL_X86_COMPILED
4388 return n >= plain_loop_least_width && ((m <= 16 && k <= 64) || (m <= 32 && k <= 32));
4389 }
4390#endif
4391 (void)m, (void)n, (void)k;
4392 return false;
4393 }
4394
4396 template <typename T>
4397 [[nodiscard]] bool plain_loop_suits(std::size_t /*m*/,
4398 std::size_t /*n*/,
4399 std::size_t /*k*/) noexcept
4400 {
4401 return false;
4402 }
4403
4405 template <>
4406 [[nodiscard]] inline bool plain_loop_suits<double>(std::size_t m,
4407 std::size_t n,
4408 std::size_t k) noexcept
4409 {
4410 return plain_loop_pays(m, n, k);
4411 }
4412
4419 template <>
4420 [[nodiscard]] inline bool plain_loop_suits<float>(std::size_t m,
4421 std::size_t n,
4422 std::size_t k) noexcept
4423 {
4424 return plain_loop_pays(m, n, k);
4425 }
4426
4429 template <typename T>
4430 void gemm_large(std::size_t m,
4431 std::size_t n,
4432 std::size_t k,
4433 const T* a,
4434 std::size_t lda,
4435 const T* b,
4436 std::size_t ldb,
4437 T* c,
4438 std::size_t ldc,
4439 c_start start = c_start::kept)
4440 {
4441 if (plain_loop_suits<T>(m, n, k)) { // the plain loop rounds as the tiles do: the same bits either way
4442 gemm_rows(m, n, k, a, lda, b, ldb, c, ldc, start);
4443 return;
4444 }
4445#ifdef NUMERICAL_NEON_COMPILED
4446 if (gemm_few_rows_arm64(m, n, k, a, lda, b, ldb, c, ldc, start)) { // the same bits
4447 return;
4448 }
4449#endif
4450 if (!gemm_elsewhere(m, n, k, a, lda, b, ldb, c, ldc, start)) {
4451 gemm_packed<vector_tile_kernel<T>, T>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
4452 }
4453 }
4454
4455#ifdef NUMERICAL_X86_COMPILED
4457 template <split_complex Z,
4458 typename Double,
4459 typename Float>
4460 using x86_complex_kernel = std::conditional_t<std::is_same_v<Z, std::complex<double>>, Double, Float>;
4461#endif
4462
4467 template <typename B,
4468 typename A,
4469 split_complex T>
4470 void gemm_large_complex(std::size_t m,
4471 std::size_t n,
4472 std::size_t k,
4473 const T* a,
4474 std::size_t lda,
4475 const T* b,
4476 std::size_t ldb,
4477 T* c,
4478 std::size_t ldc,
4479 c_start start = c_start::kept)
4480 {
4482 if (path == gemm_path::portable) {
4483 gemm_packed<vector_tile_kernel<T>, T, B, A>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
4484 return;
4485 }
4486#ifdef NUMERICAL_X86_COMPILED
4487 if (path == gemm_path::avx512) { // a complex tile of its own on the x86-64 units
4489 return;
4490 }
4491 if (path == gemm_path::avx2) {
4493 return;
4494 }
4495#endif
4496 if (start == c_start::zero) { // the split adds to C
4497 zero_c(c, ldc, m, n);
4498 }
4499 if (n * k <= split_narrow_area) {
4500 // A narrow product by blocks of rows on the threads, each block its own split: the real product of the
4501 // expanded parts, as narrow, weighed too little to take threads, and the copies of the parts took none (one
4502 // M1 Pro, eight threads: 4000 × 4 by 4 × 8 65 → 36 µs, 2000 × 16 by 16 × 16 72 → 33; GCC 14 115 → 61). Each
4503 // element the same: a product cut by its rows is (gemm's guarantee).
4504 const std::size_t row_work {((4 * n * k) / vector_speedup) + (memory_element_weight * 2 * (n + k))};
4505 const std::size_t threads {parallel_threads(m, row_work)};
4506 if (threads > 1) {
4507 const std::size_t rows {((((m + threads) - 1) / threads) + (narrow_group - 1)) / narrow_group * narrow_group};
4508 const std::size_t blocks {((m + rows) - 1) / rows};
4509 parallel_for(blocks, work_per_thread, [&](std::size_t block) {
4510 const std::size_t r0 {block * rows};
4511 gemm_split<T, B, A>(std::min(rows, m - r0), n, k, &A::at(a, lda, r0, 0), lda, b, ldb, c + (r0 * ldc), ldc);
4512 });
4513 return;
4514 }
4515 }
4516 gemm_split<T, B, A>(m, n, k, a, lda, b, ldb, c, ldc);
4517 }
4518
4520 template <split_complex T>
4521 void gemm_large(std::size_t m,
4522 std::size_t n,
4523 std::size_t k,
4524 const T* a,
4525 std::size_t lda,
4526 const T* b,
4527 std::size_t ldb,
4528 T* c,
4529 std::size_t ldc,
4530 c_start start = c_start::kept)
4531 {
4532 gemm_large_complex<b_rows, a_rows>(m, n, k, a, lda, b, ldb, c, ldc, start);
4533 }
4534
4539 [[nodiscard]] inline bool few_complex_rows([[maybe_unused]] std::size_t m,
4540 [[maybe_unused]] std::size_t n,
4541 [[maybe_unused]] std::size_t k) noexcept
4542 {
4543 if (m == 0 || n == 0 || k == 0) {
4544 return false;
4545 }
4546#if defined(NUMERICAL_X86_COMPILED)
4549#elif defined(NUMERICAL_NEON_COMPILED)
4550 return m <= 2;
4551#else
4552 return false;
4553#endif
4554 }
4555
4582 template <typename T>
4583 void gemm(std::size_t m,
4584 std::size_t n,
4585 std::size_t k,
4586 const T* a,
4587 std::size_t lda,
4588 const T* b,
4589 std::size_t ldb,
4590 T* c,
4591 std::size_t ldc,
4592 c_start start)
4593 {
4594 if constexpr (split_complex<T>) {
4596#ifdef NUMERICAL_X86_COMPILED
4597 if (few_complex_rows(m, n, k)) {
4598 static_assert(narrow_group == 4, "the switch below names each count of rows up to narrow_group");
4599 switch (m) { // a few rows, B's rows streamed by chunks of columns on the threads: the tile's bits
4600 case 1: few_rows_complex_x86<1>(n, k, a, lda, b, ldb, c, ldc, start); return;
4601 case 2: few_rows_complex_x86<2>(n, k, a, lda, b, ldb, c, ldc, start); return;
4602 case 3: few_rows_complex_x86<3>(n, k, a, lda, b, ldb, c, ldc, start); return;
4603 default: few_rows_complex_x86<4>(n, k, a, lda, b, ldb, c, ldc, start); return;
4604 }
4605 }
4606#endif
4607#ifdef NUMERICAL_NEON_COMPILED
4608 if (few_complex_rows(m, n, k)) { // one or two rows, each by gemm_split's chains: its bits
4609 few_rows_complex_neon(m, n, k, a, lda, b, ldb, c, ldc, start);
4610 return;
4611 }
4612#endif
4613 if (m != 0 && n != 0 && k != 0) {
4614 gemm_large(m, n, k, a, lda, b, ldb, c, ldc, start);
4615 }
4616 else if (start == c_start::zero) {
4617 zero_c(c, ldc, m, n);
4618 }
4619 return;
4620 }
4621 }
4622#ifdef NUMERICAL_X86_COMPILED
4623 if constexpr (std::is_same_v<T, double> || std::is_same_v<T, float>) {
4624 if (const gemm_path path {gemm_path_current()};
4626 static_assert(narrow_group == 4, "the switch below names each count of rows up to narrow_group");
4627 switch (m) { // a few rows, by blocks of columns on the threads: the same bits
4628 case 1: few_rows_x86<1>(n, k, a, lda, b, ldb, c, ldc, start); return;
4629 case 2: few_rows_x86<2>(n, k, a, lda, b, ldb, c, ldc, start); return;
4630 case 3: few_rows_x86<3>(n, k, a, lda, b, ldb, c, ldc, start); return;
4631 default: few_rows_x86<4>(n, k, a, lda, b, ldb, c, ldc, start); return;
4632 }
4633 }
4634 }
4635#endif
4636 if (by_rows(m, n, k)) { // narrow, or small: by rows, the same bits
4637 gemm_rows(m, n, k, a, lda, b, ldb, c, ldc, start);
4638 return;
4639 }
4640 if (k < gemm_min_depth || m * n < ((gemm_packed_threshold + k) - 1) / k) {
4641 gemm_rows(m, n, k, a, lda, b, ldb, c, ldc, start);
4642 return;
4643 }
4644 gemm_large(m, n, k, a, lda, b, ldb, c, ldc, start);
4645 }
4646
4648 template <typename B,
4649 typename A>
4650 void gemm_large_layouts(std::size_t m,
4651 std::size_t n,
4652 std::size_t k,
4653 const double* a,
4654 std::size_t lda,
4655 const double* b,
4656 std::size_t ldb,
4657 double* c,
4658 std::size_t ldc,
4659 c_start start)
4660 {
4661 if (!gemm_double_elsewhere<B, A>(m, n, k, a, lda, b, ldb, c, ldc, start)) {
4662 gemm_packed<vector_tile_kernel<double>, double, B, A>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
4663 }
4664 }
4665
4667 template <typename B,
4668 typename A>
4669 void gemm_large_layouts(std::size_t m,
4670 std::size_t n,
4671 std::size_t k,
4672 const float* a,
4673 std::size_t lda,
4674 const float* b,
4675 std::size_t ldb,
4676 float* c,
4677 std::size_t ldc,
4678 c_start start)
4679 {
4680 if (!gemm_float_elsewhere<B, A>(m, n, k, a, lda, b, ldb, c, ldc, start)) {
4681 gemm_packed<vector_tile_kernel<float>, float, B, A>(m, n, k, a, lda, b, ldb, c, ldc, nullptr, start);
4682 }
4683 }
4684
4686 template <typename B,
4687 typename A,
4688 split_complex T>
4689 void gemm_large_layouts(std::size_t m,
4690 std::size_t n,
4691 std::size_t k,
4692 const T* a,
4693 std::size_t lda,
4694 const T* b,
4695 std::size_t ldb,
4696 T* c,
4697 std::size_t ldc,
4698 c_start start)
4699 {
4700 gemm_large_complex<B, A>(m, n, k, a, lda, b, ldb, c, ldc, start);
4701 }
4702
4713 template <typename A,
4714 typename B,
4715 typename T>
4716 void gemm_layouts(std::size_t m,
4717 std::size_t n,
4718 std::size_t k,
4719 const T* a,
4720 std::size_t lda,
4721 const T* b,
4722 std::size_t ldb,
4723 T* c,
4724 std::size_t ldc,
4725 c_start start = c_start::kept)
4726 {
4727 if constexpr (split_complex<T>) {
4729#ifdef NUMERICAL_NEON_COMPILED
4730 if (few_complex_rows(m, n, k)) {
4731 // gemm's route, which reads B by rows, on copies: two rows of complex128 by a transpose of order 256 took
4732 // 196 µs read in place, 116 copied (one arm64 core). The x86-64 tiles read it in place faster than it is
4733 // copied (one AVX-512 core, the same product 84 against 153 µs).
4736 std::size_t a_ld {};
4737 std::size_t b_ld {};
4738 const T* const left {A::rows_of(a, lda, m, k, a_rows_copy, a_ld)};
4739 const T* const right {B::rows_of(b, ldb, k, n, b_rows_copy, b_ld)};
4740 gemm(m, n, k, left, a_ld, right, b_ld, c, ldc, start);
4741 return;
4742 }
4743#endif
4744 if (m != 0 && n != 0 && k != 0) {
4745 gemm_large_complex<B, A>(m, n, k, a, lda, b, ldb, c, ldc, start);
4746 }
4747 else if (start == c_start::zero) {
4748 zero_c(c, ldc, m, n);
4749 }
4750 return;
4751 }
4752 }
4753 // The plain loop where gemm runs it (by_rows too), the same route whatever the contraction: under GCC's default
4754 // -ffp-contract=fast the portable path's vector tile and the plain loop round apart, and four rows of complex128
4755 // by a transpose had other bits than by its copy.
4756 if (by_rows(m, n, k) || k < gemm_min_depth || m * n < ((gemm_packed_threshold + k) - 1) / k) {
4759 std::size_t a_ld {};
4760 std::size_t b_ld {};
4761 const T* const left {A::rows_of(a, lda, m, k, a_rows_copy, a_ld)};
4762 const T* const right {B::rows_of(b, ldb, k, n, b_rows_copy, b_ld)};
4763 gemm_rows(m, n, k, left, a_ld, right, b_ld, c, ldc, start);
4764 return;
4765 }
4766 gemm_large_layouts<B, A>(m, n, k, a, lda, b, ldb, c, ldc, start);
4767 }
4768
4770 template <typename T>
4771 void gemm_bt(std::size_t m,
4772 std::size_t n,
4773 std::size_t k,
4774 const T* a,
4775 std::size_t lda,
4776 const T* bt,
4777 std::size_t ldbt,
4778 T* c,
4779 std::size_t ldc,
4780 c_start start = c_start::kept)
4781 {
4782 gemm_layouts<a_rows, b_transposed>(m, n, k, a, lda, bt, ldbt, c, ldc, start);
4783 }
4784
4786 template <typename T>
4787 void gemm_at(std::size_t m,
4788 std::size_t n,
4789 std::size_t k,
4790 const T* at,
4791 std::size_t ldat,
4792 const T* b,
4793 std::size_t ldb,
4794 T* c,
4795 std::size_t ldc,
4796 c_start start = c_start::kept)
4797 {
4798 gemm_layouts<a_transposed, b_rows>(m, n, k, at, ldat, b, ldb, c, ldc, start);
4799 }
4800
4802 template <typename T>
4803 void gemm_atbt(std::size_t m,
4804 std::size_t n,
4805 std::size_t k,
4806 const T* at,
4807 std::size_t ldat,
4808 const T* bt,
4809 std::size_t ldbt,
4810 T* c,
4811 std::size_t ldc,
4812 c_start start = c_start::kept)
4813 {
4814 gemm_layouts<a_transposed, b_transposed>(m, n, k, at, ldat, bt, ldbt, c, ldc, start);
4815 }
4816
4818 enum class left_kernel : unsigned char
4819 {
4820 plain,
4821 vector,
4822 neon,
4823 amx,
4824 avx512,
4825 avx2
4826 };
4827
4830 template <typename T>
4832 {
4834 std::size_t m;
4835 std::size_t k;
4836 const T* a;
4837 std::size_t lda;
4838 const T* packed;
4839 std::size_t granule;
4840 };
4841
4843 template <typename T>
4845 std::size_t k,
4846 const T* a,
4847 std::size_t lda) noexcept
4848 {
4849 return packed_left<T> {.kernel = left_kernel::plain, .m = m, .k = k, .a = a, .lda = lda, .packed = nullptr, .granule = 1};
4850 }
4851
4856 template <typename T>
4858 std::size_t r0,
4859 std::size_t rows) noexcept
4860 {
4862 part.m = rows;
4863 part.a = left.a + (r0 * left.lda);
4864 part.packed = left.packed == nullptr ? nullptr : left.packed + (r0 * left.k);
4865 return part;
4866 }
4867
4870 template <typename T>
4872 std::size_t /*n*/,
4873 std::size_t /*k*/) noexcept
4874 {
4875 return left_kernel::plain;
4876 }
4877
4880 template <>
4882 std::size_t n,
4883 std::size_t k) noexcept
4884 {
4885 if (k < gemm_min_depth || m * n < ((gemm_packed_threshold + k) - 1) / k || k > gemm_kc) {
4886 return left_kernel::plain;
4887 }
4889#ifdef NUMERICAL_AMX_COMPILED
4890 if (path == gemm_path::amx && amx_suits(m, n, k)) {
4891 return left_kernel::amx;
4892 }
4893#endif
4894#ifdef NUMERICAL_NEON_COMPILED
4895 if (path == gemm_path::neon || path == gemm_path::amx) {
4896 return left_kernel::neon;
4897 }
4898#endif
4899#ifdef NUMERICAL_X86_COMPILED
4900 if (path == gemm_path::avx512) {
4901 return left_kernel::avx512;
4902 }
4903 if (path == gemm_path::avx2) {
4904 return left_kernel::avx2;
4905 }
4906#endif
4907 (void)path;
4908 return left_kernel::vector;
4909 }
4910
4912 template <typename Kernel,
4913 typename T>
4914 [[nodiscard]] const T* pack_left_for(std::size_t m,
4915 std::size_t k,
4916 const T* a,
4917 std::size_t lda,
4919 {
4920 const std::size_t a_rows {((m + Kernel::mr) - 1) / Kernel::mr * Kernel::mr};
4921 storage.resize(std::max(storage.size(), (a_rows * k) + (packed_line / sizeof(T)) + 1));
4922 T* const out {line_aligned(storage.data())};
4923 const std::size_t blocks {((m + Kernel::block_rows) - 1) / Kernel::block_rows};
4924 parallel_for(blocks, Kernel::block_rows * k * gemm_kernel_speedup / Kernel::speedup, [&](std::size_t block) {
4925 const std::size_t ic {block * Kernel::block_rows};
4926 Kernel::pack_a(a + (ic * lda), lda, std::min(Kernel::block_rows, m - ic), k, out + (ic * k));
4927 });
4928 return out;
4929 }
4930
4937 template <typename T>
4939 std::size_t /*n*/,
4940 std::size_t k,
4941 const T* a,
4942 std::size_t lda,
4943 buffer<T>& /*storage*/)
4944 {
4945 return read_in_place(m, k, a, lda);
4946 }
4947
4949 template <>
4950 [[nodiscard]] inline packed_left<double> pack_left<double>(std::size_t m,
4951 std::size_t n,
4952 std::size_t k,
4953 const double* a,
4954 std::size_t lda,
4956 {
4957 packed_left<double> left {.kernel = left_kernel_for<double>(m, n, k), .m = m, .k = k, .a = a, .lda = lda, .packed = nullptr, .granule = 1};
4958#ifdef NUMERICAL_AMX_COMPILED
4959 if (left.kernel == left_kernel::amx) {
4960 left.packed = pack_left_for<amx_tile_kernel>(m, k, a, lda, storage);
4961 left.granule = amx_tile_kernel::mr;
4962 }
4963#endif
4964#ifdef NUMERICAL_NEON_COMPILED
4965 if (left.kernel == left_kernel::neon) {
4966 left.packed = pack_left_for<neon_tile_kernel<double>>(m, k, a, lda, storage);
4967 left.granule = neon_tile_kernel<double>::mr;
4968 }
4969#endif
4970#ifdef NUMERICAL_X86_COMPILED
4971 if (left.kernel == left_kernel::avx512) {
4972 left.packed = pack_left_for<avx512_tile_kernel>(m, k, a, lda, storage);
4973 left.granule = avx512_tile_kernel::mr;
4974 }
4975 if (left.kernel == left_kernel::avx2) {
4976 left.packed = pack_left_for<avx2_tile_kernel>(m, k, a, lda, storage);
4977 left.granule = avx2_tile_kernel::mr;
4978 }
4979#endif
4980 if (left.kernel == left_kernel::vector) {
4981 left.packed = pack_left_for<vector_tile_kernel<double>>(m, k, a, lda, storage);
4983 }
4984 return left;
4985 }
4986
4991 template <typename T>
4993 std::size_t n,
4994 const T* b,
4995 std::size_t ldb,
4996 T* c,
4997 std::size_t ldc)
4998 {
4999 gemm(left.m, n, left.k, left.a, left.lda, b, ldb, c, ldc);
5000 }
5001
5007 template <>
5009 std::size_t n,
5010 const double* b,
5011 std::size_t ldb,
5012 double* c,
5013 std::size_t ldc)
5014 {
5015 if (left.kernel == left_kernel::plain || left_kernel_for<double>(left.m, n, left.k) != left.kernel) {
5016 gemm(left.m, n, left.k, left.a, left.lda, b, ldb, c, ldc);
5017 return;
5018 }
5019#ifdef NUMERICAL_AMX_COMPILED
5020 if (left.kernel == left_kernel::amx) {
5021 gemm_packed<amx_tile_kernel>(left.m, n, left.k, left.a, left.lda, b, ldb, c, ldc, left.packed);
5022 return;
5023 }
5024#endif
5025#ifdef NUMERICAL_NEON_COMPILED
5026 if (left.kernel == left_kernel::neon) {
5027 gemm_packed<neon_tile_kernel<double>>(left.m, n, left.k, left.a, left.lda, b, ldb, c, ldc, left.packed);
5028 return;
5029 }
5030#endif
5031#ifdef NUMERICAL_X86_COMPILED
5032 if (left.kernel == left_kernel::avx512) {
5033 gemm_packed<avx512_tile_kernel>(left.m, n, left.k, left.a, left.lda, b, ldb, c, ldc, left.packed);
5034 return;
5035 }
5036 if (left.kernel == left_kernel::avx2) {
5037 gemm_packed<avx2_tile_kernel>(left.m, n, left.k, left.a, left.lda, b, ldb, c, ldc, left.packed);
5038 return;
5039 }
5040#endif
5041 gemm_packed<vector_tile_kernel<double>>(left.m, n, left.k, left.a, left.lda, b, ldb, c, ldc, left.packed);
5042 }
5043
5045 inline constexpr std::size_t dot_chunk {std::size_t {1} << 14U};
5046
5053 [[nodiscard]] constexpr std::size_t dot_chunk_weight(std::size_t chunks) noexcept
5054 {
5055 return chunks > 2 ? 4 * dot_chunk : dot_chunk;
5056 }
5057
5064 template <typename T,
5065 typename Dot>
5066 [[nodiscard]] T chunked_dot(const T* a,
5067 const T* b,
5068 std::size_t count,
5069 const Dot& dot)
5070 {
5071 const std::size_t chunks {(count + dot_chunk - 1) / dot_chunk};
5072 if (chunks <= 1) {
5073 return dot(a, b, count);
5074 }
5075 buffer<T> partial(chunks);
5076 parallel_for(chunks, dot_chunk_weight(chunks), [&](std::size_t c) {
5077 const std::size_t first {c * dot_chunk};
5078 partial[c] = dot(a + first, b + first, std::min(dot_chunk, count - first));
5079 });
5080 T total {};
5081 for (const T& sum : partial) {
5082 total += sum;
5083 }
5084 return total;
5085 }
5086
5088 template <typename T>
5089 [[nodiscard]] T chunked_dot(const T* a,
5090 const T* b,
5091 std::size_t count)
5092 {
5093 return chunked_dot(a, b, count, [](const T* x, const T* y, std::size_t n) { return dot_product(x, y, n); });
5094 }
5095
5098 template <typename T>
5100 const T* b,
5101 std::size_t count)
5102 {
5103 return chunked_dot(a, b, count, [](const T* x, const T* y, std::size_t n) { return conjugate_dot_product(x, y, n); });
5104 }
5105
5107 template <typename T>
5108 inline constexpr std::size_t multiply_add_weight {1};
5109
5111 template <>
5112 inline constexpr std::size_t multiply_add_weight<std::complex<double>> {4};
5113
5115 template <>
5116 inline constexpr std::size_t multiply_add_weight<std::complex<float>> {4};
5117
5119 inline constexpr std::size_t row_group {64};
5120
5128#ifdef NUMERICAL_X86_COMPILED
5129 inline constexpr std::size_t matrix_vector_weight {4};
5130#else
5131 inline constexpr std::size_t matrix_vector_weight {1};
5132#endif
5133
5134#ifdef NUMERICAL_AMX_COMPILED
5137 inline constexpr std::size_t amx_dot_least {std::size_t {1} << 15U};
5138
5142 inline constexpr std::size_t amx_dot_most {std::size_t {1} << 18U};
5143#endif
5144
5152 template <typename T>
5153 void matrix_times_vector(const T* a,
5154 std::size_t lda,
5155 std::size_t rows,
5156 std::size_t cols,
5157 const T* x,
5158 T* y)
5159 {
5160#ifdef NUMERICAL_AMX_COMPILED
5161 if constexpr (std::is_same_v<T, double>) {
5162 const std::size_t elements {rows * cols};
5164 const amx_session on {}; // on the caller alone
5165 amx_dot_rows_f64(a, lda, rows, x, cols, y);
5166 return;
5167 }
5168 }
5169#endif
5170 if (cols > dot_chunk) { // rows longer than a chunk: chunked_dot's partial sums
5171 parallel_for(rows, cols, [&](std::size_t r) { y[r] = chunked_dot(a + (r * lda), x, cols); });
5172 return;
5173 }
5174 // A complex multiply-add weighs four real ones. Groups of fewer rows where the matrix has too few for each thread
5175 // to take some (189 rows of 64 were three groups: six AVX2 cores' L3 read by one, a complex matrix of 189 × 206
5176 // times a vector 24 µs); each row its own dot product whatever the grouping, so the same bits.
5177 constexpr std::size_t weight {multiply_add_weight<T> * matrix_vector_weight};
5178 const std::size_t group {std::clamp<std::size_t>(rows / (4 * max_threads()), 8, row_group)};
5179 parallel_for((rows + group - 1) / group, group * cols * weight, [&](std::size_t g) {
5180 const std::size_t first {g * group};
5181 row_dots(a + (first * lda), lda, std::min(group, rows - first), x, cols, y + first);
5182 });
5183 }
5184
5188 inline constexpr std::size_t column_rows {128};
5189
5196 inline void columns_amx_lanes(const double* a,
5197 std::size_t lda,
5198 std::size_t rows,
5199 std::size_t cols,
5200 const double* x,
5201 double* y)
5202 {
5203 constexpr std::size_t lanes {8}; // a Z row's
5204 buffer<double> sums(lanes * rows, 0.0);
5205 for (std::size_t j {0}; j < cols; ++j) {
5206 double* const lane {sums.data() + ((j % lanes) * rows)};
5207 const double* const column {a + (j * lda)};
5208 const double factor {x[j]};
5209 for (std::size_t r {0}; r < rows; ++r) {
5210 lane[r] = std::fma(column[r], factor, lane[r]);
5211 }
5212 }
5213 for (std::size_t r {0}; r < rows; ++r) {
5214 double sum {sums[r]};
5215 for (std::size_t l {1}; l < lanes; ++l) {
5216 sum += sums[(l * rows) + r];
5217 }
5218 y[r] = sum;
5219 }
5220 }
5221
5235 template <typename T>
5236 requires std::is_same_v<T, double> || std::is_same_v<T, float>
5238 std::size_t lda,
5239 std::size_t rows,
5240 std::size_t cols,
5241 const T* x,
5242 T* y)
5243 {
5244#ifdef NUMERICAL_X86_COMPILED
5246 if ((path != gemm_path::avx2 && path != gemm_path::avx512) || cols > dot_chunk) {
5247 return false;
5248 }
5249 constexpr std::size_t partials {256 / sizeof(T)}; // both kernels' partial sums: four AVX-512 registers' worth
5250 const std::size_t done {cols - (cols % partials)};
5251 parallel_for((rows + column_rows - 1) / column_rows, column_rows * cols * matrix_vector_weight, [&](std::size_t block) {
5252 const std::size_t first {block * column_rows};
5253 const std::size_t height {std::min(column_rows, rows - first)};
5254 buffer<T> sums(partials * height, T {});
5255 const T* const part {a + first};
5256 avx2_column_partials<partials>(part, lda, height, done, x, sums.data());
5257 columns_finish(part, lda, height, done, cols, x, sums.data(), y + first);
5258 });
5259 return true;
5260#elif defined(__GNUC__)
5261 // Elsewhere the portable vectors' row_dots (columns_row_dots, its partials), or the AMX's lanes where the row-major
5262 // product takes the AMX (amx_dot_rows_f64), except for rows longer than a chunk.
5263 if (cols > dot_chunk) {
5264 return false;
5265 }
5266# ifdef NUMERICAL_AMX_COMPILED
5267 if constexpr (std::is_same_v<T, double>) {
5268 const std::size_t elements {rows * cols};
5269 if (elements >= amx_dot_least && elements < amx_dot_most && gemm_path_current() == gemm_path::amx) {
5270 parallel_for((rows + column_rows - 1) / column_rows, column_rows * cols * matrix_vector_weight, [&](std::size_t block) {
5271 const std::size_t first {block * column_rows};
5272 columns_amx_lanes(a + first, lda, std::min(column_rows, rows - first), cols, x, y + first);
5273 });
5274 return true;
5275 }
5276 }
5277# endif
5278 parallel_for((rows + column_rows - 1) / column_rows, column_rows * cols * matrix_vector_weight, [&](std::size_t block) {
5279 const std::size_t first {block * column_rows};
5280 columns_row_dots(a + first, lda, std::min(column_rows, rows - first), cols, x, y + first);
5281 });
5282 return true;
5283#else
5284 (void)a, (void)lda, (void)rows, (void)cols, (void)x, (void)y;
5285 return false;
5286#endif
5287 }
5288
5296 template <split_complex C>
5297 [[nodiscard]] bool columns_times_vector(const C* a,
5298 std::size_t lda,
5299 std::size_t rows,
5300 std::size_t cols,
5301 const C* x,
5302 C* y)
5303 {
5304#if defined(__GNUC__)
5305 if (cols > dot_chunk) {
5306 return false;
5307 }
5308 constexpr std::size_t weight {multiply_add_weight<C> * matrix_vector_weight};
5309 // Blocks small enough that each thread takes some: each row's sums its own whatever the block, so the same bits
5310 // (ten AVX-512 cores, complex64 500 × 500: 85 µs in four blocks of 128 rows).
5311 const std::size_t height {std::clamp<std::size_t>((((rows / (2 * max_threads())) + 15) / 16) * 16, 16, column_rows)};
5312# ifdef NUMERICAL_X86_COMPILED
5313 if (gemm_path_current() != gemm_path::portable) { // avx2_complex_dot's slots, on the AVX-512 path too
5314 using R = typename C::value_type;
5315 constexpr std::size_t slots {std::is_same_v<R, double> ? 8 : 16};
5316 const std::size_t done {cols - (cols % slots)};
5317 const auto* const parts {reinterpret_cast<const R*>(a)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): a complex number is an array of its two parts
5318 const auto* const xs {reinterpret_cast<const R*>(x)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): as above
5319 parallel_for((rows + height - 1) / height, height * cols * weight, [&](std::size_t block) {
5320 const std::size_t first {block * height};
5321 const std::size_t count {std::min(height, rows - first)};
5322 buffer<R> sums(2 * slots * 2 * count, R {});
5323 avx2_complex_column_slots<slots>(parts + (2 * first), lda, count, done, xs, sums.data());
5324 avx2_complex_columns_finish<slots>(parts + (2 * first), lda, count, done, cols, xs, sums.data(), y + first);
5325 });
5326 return true;
5327 }
5328# endif
5329 parallel_for((rows + height - 1) / height, height * cols * weight, [&](std::size_t block) {
5330 const std::size_t first {block * height};
5331 columns_complex_row_dots(a + first, lda, std::min(height, rows - first), cols, x, y + first);
5332 });
5333 return true;
5334#else
5335 (void)a, (void)lda, (void)rows, (void)cols, (void)x, (void)y;
5336 return false;
5337#endif
5338 }
5339} // namespace numerical::detail
5340
5341namespace numerical {
5342
5350 {
5351 detail::gemm_path_cap().store(most, std::memory_order_relaxed);
5352 return detail::best_gemm_path(most); // probes only the paths at or below the cap
5353 }
5354
5356 [[nodiscard]] inline gemm_path gemm_path_in_use() noexcept
5357 {
5358 return detail::gemm_path_current();
5359 }
5360} // namespace numerical
5361
5362#endif // NUMERICAL_GEMM_HPP
The matrix coprocessor of Apple's M-series chips (AMX), for the double matrix product.
numerical::buffer, the storage of the library's vectors, matrices and tensors: a std::vector whose el...
#define NUMERICAL_OUT_OF_LINE
Definition buffer.hpp:30
The storage of a product's packed operands, kept by the calling thread from one product to the next.
Definition gemm.hpp:2389
packing_storage(std::size_t a_count, std::size_t b_count)
Room for a_count packed elements of A and b_count of B.
Definition gemm.hpp:2396
static buffer< T > grown(buffer< T > storage, std::size_t count)
storage with at least count elements (it only grows, so a reuse writes nothing).
Definition gemm.hpp:2438
static buffer< T > & kept() noexcept
The storage this thread keeps between products.
Definition gemm.hpp:2426
T * b() const noexcept
B's packed elements.
Definition gemm.hpp:2420
static T * aligned(T *at) noexcept
line_aligned.
Definition gemm.hpp:2446
static constexpr std::size_t padding
Elements enough to reach a line.
Definition gemm.hpp:2435
packing_storage(packing_storage &&)=delete
static constexpr std::size_t kept_limit
The most elements a thread keeps between products (32 MiB).
Definition gemm.hpp:2393
T * a() const noexcept
A's packed elements.
Definition gemm.hpp:2414
packing_storage & operator=(packing_storage &&)=delete
packing_storage(const packing_storage &)=delete
packing_storage & operator=(const packing_storage &)=delete
A complex number of float or double, which gemm_split multiplies through the real kernel and the kern...
Definition simd.hpp:188
#define NUMERICAL_SCALAR_DEPTH
Definition gemm.hpp:64
#define NUMERICAL_UNROLL_BLOCK
Definition gemm.hpp:72
constexpr std::size_t gemm_packed_threshold
Below this many multiply-adds the plain loop (gemm_rows, by blocks of rows held in registers) runs in...
Definition gemm.hpp:909
bool gemm_path_usable(gemm_path path) noexcept
Whether path can run here.
Definition gemm.hpp:102
C complex_dot_product(const C *x, const C *y, std::size_t count)
Σ x[i] · y[i] of complex numbers: on the x86-64 units with AVX2 (the AVX-512 path too) their fused ch...
Definition gemm.hpp:753
complex_term complex_terms() noexcept
The complex_term of the path in use.
Definition gemm.hpp:2318
constexpr std::size_t split_columns
The columns of B and C, and the depth, of one real product of gemm_split: its real operands stay a fe...
Definition gemm.hpp:3877
void gemm_pack_b(const T *b, std::size_t ldb, std::size_t kc, std::size_t cols, T *out)
Copies kc × cols of B (from row-major, leading dimension ldb) into slivers of Nr columns,...
Definition gemm.hpp:1107
bool on_release(void(*step)()) noexcept
Registers step among the calling thread's thread_releases, while a place is left (a workspace past th...
Definition buffer.hpp:65
bool plain_loop_pays(std::size_t m, std::size_t n, std::size_t k) noexcept
Whether a double product of m rows, n columns and depth k, past gemm_rows's threshold,...
Definition gemm.hpp:4372
constexpr std::size_t gemm_rows_share
The multiply-adds by rows a thread takes at most past gemm_rows_work, the blocks of rows shared out: ...
Definition gemm.hpp:1749
void parallel_zero(T *out, std::size_t count)
Sets the count elements at out to zero, split across the threads the work pays for: a fresh result's ...
Definition parallel.hpp:891
const T * pack_left_for(std::size_t m, std::size_t k, const T *a, std::size_t lda, buffer< T > &storage)
A packed for Kernel into storage, in parallel by its blocks of rows, as gemm_packed packs it.
Definition gemm.hpp:4914
void gemm_pack_a(const T *a, std::size_t lda, std::size_t rows, std::size_t kc, T *out)
Copies rows × kc of A (from row-major, leading dimension lda) into slivers of Mr rows,...
Definition gemm.hpp:1076
void gemm_triangle(std::size_t n, std::size_t k, const double *a, std::size_t lda, const double *b, std::size_t ldb, double *c, std::size_t ldc, bool lower)
The triangle of the double product C = A·B (gemm_packed_triangle) on the tile of the path in use,...
Definition gemm.hpp:3356
void narrow_rows(std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start) noexcept
C += A·B for R whole rows of C of n columns (C = A·B as start says): blocks of narrow_columns,...
Definition gemm.hpp:2171
void transpose_walk(std::size_t i0, std::size_t i1, std::size_t j0, std::size_t j1, std::size_t piece, const Copy &copy)
transpose_pieces of side piece (transpose_piece's 8 or the whole tile), each side its own instance: a...
Definition simd.hpp:142
constexpr bool zeroes_c(bool empty) noexcept
Whether gemm_packed zeroes a C that holds nothing (empty) for Kernel's tiles to add to.
Definition gemm.hpp:2537
bool split_by_parts(std::size_t m) noexcept
Whether gemm_split runs a product of m rows of A by four real products of each block's parts (split_p...
Definition gemm.hpp:4163
void gemm_large(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
A product past the plain loop: on the AMX or x86-64 tiles where gemm_elsewhere takes it,...
Definition gemm.hpp:4430
constexpr bool prefetches_next_row_of_c
Whether Kernel has the C tile of the next row of tiles prefetched while a tile computes (Kernel::pref...
Definition gemm.hpp:2470
void subtract_rows(double *y, std::size_t ldy, const double *f, std::size_t ldf, const double *x, std::size_t ldx, std::size_t terms, std::size_t count)
subtract_rows for double, the units chosen as subtract_products chooses them: each element the value ...
Definition gemm.hpp:551
left_kernel
The tile a product's left operand is packed for (numerical::detail::pack_left): plain where it is not...
Definition gemm.hpp:4819
@ vector
The portable path's vector tile.
@ plain
gemm_rows, or a scalar whose products are not shared: A is read in place.
void gemm_pack_at(const T *at, std::size_t ldat, std::size_t rows, std::size_t kc, T *out)
The slivers of Mr rows of A (rows × kc), column by column, from its transpose at (kc × rows row-major...
Definition gemm.hpp:1317
T conjugate_dot_product(const T *x, const T *y, std::size_t count)
Σ conj(x[i]) · y[i], in order, where no native vector holds the parts.
Definition simd.hpp:1089
constexpr std::size_t dot_chunk_weight(std::size_t chunks) noexcept
The work chunked_dot weighs one of its chunks chunks, in multiply-adds for parallel_for's split: four...
Definition gemm.hpp:5053
bool amx_usable() noexcept
Whether this chip's AMX can be used: known family and a passing self-test, decided once.
Definition amx.hpp:964
left_kernel left_kernel_for(std::size_t, std::size_t, std::size_t) noexcept
The tile gemm runs a product of m × n by depth k on, for a left operand packed once for many products...
Definition gemm.hpp:4871
constexpr std::size_t gemm_kernel_speedup
The multiply-adds the register tile does in the time of one in work_per_thread's dependent chain (abo...
Definition gemm.hpp:893
void gemm_large_complex(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
A complex product past the plain loop: on the vector tile on the portable path, whole complex product...
Definition gemm.hpp:4470
void exchange(buffer< T > &a, std::size_t n, std::size_t i, std::size_t j)
Exchanges rows i and j, then columns i and j, of the row-major n × n a: a similarity.
Definition eigen.hpp:109
std::size_t parallel_threads(std::size_t count, std::size_t work_per_index)
The threads parallel_for shares count indices of work_per_index out to (threads_for): the hardware's,...
Definition parallel.hpp:766
carried_t< T > real_part(const T &x)
The real part of a real (or general) scalar, in carried_t.
void gemm_at(std::size_t m, std::size_t n, std::size_t k, const T *at, std::size_t ldat, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm_layouts with A given as Aᵀ (at, k × m, ldat apart).
Definition gemm.hpp:4787
T operator_times(const T &a, const T &b)
a · b, as std::operator* computes it, in line. libc++'s complex product (the four products and their ...
Definition simd.hpp:1122
constexpr std::size_t vector_speedup
The multiply-adds a vectorised inner loop does in the time of one in work_per_thread's dependent chai...
Definition parallel.hpp:739
void zero_c(T *c, std::size_t ldc, std::size_t m, std::size_t n)
Zeroes C's m × n elements, its rows ldc apart: on the threads where they lie together and are many (p...
Definition gemm.hpp:2219
constexpr std::size_t gemm_mr
The rows of C one register tile covers.
Definition gemm.hpp:870
constexpr std::size_t narrow_group
The rows of C one block of gemm_rows takes together: each element a chain of multiply-adds that waits...
Definition gemm.hpp:1742
void columns_amx_lanes(const double *a, std::size_t lda, std::size_t rows, std::size_t cols, const double *x, double *y)
amx_dot_rows_f64's sums of the rows rows of the column-major a (column j at a + j · lda) without the ...
Definition gemm.hpp:5196
void add_product(double *y, const double &f, const double *x, std::size_t count)
y[i] += f · x[i] for double, as subtract_product.
Definition gemm.hpp:648
packed_left< double > pack_left< double >(std::size_t m, std::size_t n, std::size_t k, const double *a, std::size_t lda, buffer< double > &storage)
pack_left for double, on the tile left_kernel_for chooses.
Definition gemm.hpp:4950
gemm_path best_gemm_path(gemm_path most) noexcept
The most capable path that can run here, most at the highest.
Definition gemm.hpp:116
void row_dots(const std::complex< double > *m, std::size_t ld, std::size_t rows, const std::complex< double > *x, std::size_t count, std::complex< double > *out)
row_dots for std::complex<double>, through complex_row_dots.
Definition gemm.hpp:802
void subtract_products(double *y, const double *f, const double *x, std::size_t ldx, std::size_t terms, std::size_t count)
y[i] -= Σₜ f[t] · x[(t · ldx) + i] for double, the units chosen as subtract_product chooses them,...
Definition gemm.hpp:522
constexpr std::size_t work_per_thread
The work one thread must have to pay for starting it: about 60 µs of multiply-adds,...
Definition parallel.hpp:729
constexpr std::size_t dot_chunk
The elements of one partial sum of chunked_dot: fixed, so the sum does not depend on the threads.
Definition gemm.hpp:5045
T chunked_conjugate_dot(const T *a, const T *b, std::size_t count)
Σ conj(a[i]) · b[i] (zdotc): chunked_dot of each chunk by conjugate_dot_product, without a conjugated...
Definition gemm.hpp:5099
void gemm_with(const packed_left< T > &left, std::size_t n, const T *b, std::size_t ldb, T *c, std::size_t ldc)
C += A·B for the packed left (pack_left) and the k × n b, as gemm computes it, A read in place here,...
Definition gemm.hpp:4992
const T * rows_from_transpose(const T *x, std::size_t ld, std::size_t rows, std::size_t cols, buffer< T > &scratch, std::size_t &ld_out)
A row-major rows × cols operand for the plain loop, copied into scratch from its transpose x (cols × ...
Definition gemm.hpp:1195
void zero_block(T *c, std::size_t ldc, std::size_t first, std::size_t last, std::size_t from, std::size_t to) noexcept
Zeroes rows first to last (excluded) of C, columns from to to (excluded), its rows ldc apart.
Definition gemm.hpp:2200
void gemm_layouts(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm with either operand given as its transpose (A: a_rows or a_transposed, B: b_rows or b_transposed...
Definition gemm.hpp:4716
void gemm_split(std::size_t m, std::size_t n, std::size_t k, const Z *a, std::size_t lda, const Z *b, std::size_t ldb, Z *c, std::size_t ldc)
C += A·B for complex numbers through the real kernel, by blocks of split_columns columns and split_de...
Definition gemm.hpp:4305
complex_term
How a complex product rounds a term that sums nothing (0 + a·b) on the path in use,...
Definition gemm.hpp:2311
@ compiler
The vector tile's and the plain loop's: 0 + a·b as the compiler rounds it (rounded_apart_add).
@ tiles
The x86-64 complex tiles': each part's product from zero, re·re − im·im and re·im + im·re,...
@ split
arm64's real kernel's (gemm_split): Re a·Re b from zero, −Im a·Im b fused into it; Re a·Im b,...
void subtract_product(double *y, const double &f, const double *x, std::size_t count)
y[i] −= f · x[i] for double on the widest x86-64 units the matrix product's kernel may use (gemm_path...
Definition gemm.hpp:485
void gemm_large_layouts(std::size_t m, std::size_t n, std::size_t k, const double *a, std::size_t lda, const double *b, std::size_t ldb, double *c, std::size_t ldc, c_start start)
A double product past the plain loop, B and A laid out as B and A say (gemm_layouts).
Definition gemm.hpp:4650
void gemm_atbt(std::size_t m, std::size_t n, std::size_t k, const T *at, std::size_t ldat, const T *bt, std::size_t ldbt, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm_layouts with both operands given as their transposes.
Definition gemm.hpp:4803
constexpr std::size_t plain_loop_least_width
The fewest columns of a product plain_loop_pays sends to the plain loop on the x86-64 tiles: there an...
Definition gemm.hpp:4360
packed_left< T > pack_left(std::size_t m, std::size_t, std::size_t k, const T *a, std::size_t lda, buffer< T > &)
The m × k a packed once for products with it of about n columns each (gemm_with), in storage (grown a...
Definition gemm.hpp:4938
void gemm_bt(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *bt, std::size_t ldbt, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm_layouts with B given as Bᵀ (bt, n × k, ldbt apart).
Definition gemm.hpp:4771
constexpr std::size_t split_kept
The most elements an operand of gemm_split keeps from one product to the next on a thread: 2 MiB of d...
Definition gemm.hpp:4148
T rounded_apart_add(T sum, T a, T b) noexcept
sum + a·b as the vector tile rounds it (vector_tile_kernel), so that the plain loop's elements are it...
Definition gemm.hpp:1717
bool plain_loop_suits(std::size_t, std::size_t, std::size_t) noexcept
plain_loop_pays for a product of T: double's and float's, those it was measured for.
Definition gemm.hpp:4397
constexpr std::size_t tasks_per_thread
The tasks a product's compute pass hands each thread at least, while its rows allow: a slower core th...
Definition gemm.hpp:888
constexpr std::size_t gemm_rows_work
The most multiply-adds of a product gemm runs by rows (gemm_rows) whatever its width: up to there its...
Definition gemm.hpp:1695
constexpr std::size_t split_depth
The depth of one real product of gemm_split (split_columns).
Definition gemm.hpp:3880
T * line_aligned(T *at) noexcept
The first element of at's run that starts a cache line (or at's nearest element towards one,...
Definition gemm.hpp:2371
std::atomic< std::uint8_t > & gemm_path_found() noexcept
The path gemm_path_current found last, packed with the cap it was found under (cap · 8 + path); 0xFF ...
Definition gemm.hpp:136
constexpr bool gemm_shares(std::size_t m, std::size_t n, std::size_t k) noexcept
Whether the product's threads pay: below Kernel's serial_below multiply-adds, the product runs on the...
Definition gemm.hpp:2459
constexpr std::size_t split_row_width
The columns split_row_columns holds at once: a block of B's rows read twice stays in the caches.
Definition gemm.hpp:3884
constexpr bool neon_compiled
Whether the NEON path's tile is compiled in: on every arm64 target GCC or Clang builds.
Definition gemm.hpp:98
void parallel_for(std::size_t count, std::size_t work_per_index, const Body &body)
Runs body(i) for every i in [0, count).
Definition parallel.hpp:790
constexpr std::size_t gemm_nr
The columns of C one register tile covers: 96 bytes of elements (12 double, 24 float),...
Definition gemm.hpp:875
void matrix_times_vector(const T *a, std::size_t lda, std::size_t rows, std::size_t cols, const T *x, T *y)
y = A x for the row-major rows × cols a, its rows lda apart: each element the dot product of its row ...
Definition gemm.hpp:5153
constexpr std::size_t split_narrow_area
The largest n·k of a complex product gemm_split shares out by blocks of rows: the real product of its...
Definition gemm.hpp:4179
constexpr std::size_t gemm_mc
The rows of one task: an A block (gemm_mc × gemm_kc) stays in the L2 cache.
Definition gemm.hpp:881
std::size_t transpose_piece(std::size_t from_bytes, std::size_t to_bytes) noexcept
The side of the square pieces a transposing copy walks a tile by, from the bytes between rows of its ...
Definition simd.hpp:100
void split_expanded_block(std::size_t m, std::size_t nb, std::size_t kb, std::size_t pc, std::size_t jc, const Z *a, std::size_t lda, const Z *b, std::size_t ldb, typename Z::value_type *ar, typename Z::value_type *br, typename Z::value_type *cr)
One block of gemm_split (split_parts_block's) by one real product of its expanded operands,...
Definition gemm.hpp:4259
gemm_path gemm_path_current() noexcept
The path the next double product takes: the most capable one under the cap that runs here (each path'...
Definition gemm.hpp:148
bool columns_times_vector(const T *a, std::size_t lda, std::size_t rows, std::size_t cols, const T *x, T *y)
y = A x for the column-major rows × cols a (column j at a + j · lda; NumPy's a.T, a transposed operan...
Definition gemm.hpp:5237
bool gemm_float_elsewhere(std::size_t m, std::size_t n, std::size_t k, const float *a, std::size_t lda, const float *b, std::size_t ldb, float *c, std::size_t ldc, c_start start=c_start::kept)
A float product on the AMX where it suits the product, on the NEON tile on arm64's fused paths,...
Definition gemm.hpp:3443
void narrow_block(std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start) noexcept
C += A·B for R rows and N columns of C (C = A·B as start says), each sum in ascending order of the de...
Definition gemm.hpp:2119
std::integral_constant< term_rounding, Mode > rounding_is
The rounding mode Mode as a type, which picks rounded_term's overload.
Definition gemm.hpp:1787
constexpr std::size_t packed_line
The cache line a packed operand starts.
Definition gemm.hpp:2366
constexpr std::size_t narrow_columns
The columns of C one block of gemm_rows keeps in registers over the whole depth.
Definition gemm.hpp:1669
const T * rows_in_place(const T *x, std::size_t ld, std::size_t &ld_out) noexcept
A row-major rows × cols operand for the plain loop, x itself (ld apart).
Definition gemm.hpp:1184
void gemm(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
C += A·B, every element of C summed in ascending order of the depth.
Definition gemm.hpp:4583
void few_rows(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start)
C += A·B (C = A·B as start says) for m rows of A, by groups of at most narrow_group rows and blocks o...
Definition gemm.hpp:3516
T rounded_term(rounding_is< term_rounding::compiler >, T sum, T a, T b) noexcept
sum + a·b as the compiler rounds it (rounded_apart_add).
Definition gemm.hpp:1791
bool few_complex_rows(std::size_t m, std::size_t n, std::size_t k) noexcept
Whether gemm runs a complex product of m rows, B k × n, a few rows at a time with B read by rows (few...
Definition gemm.hpp:4539
bool gemm_elsewhere(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
A product a faster path than the vector one takes, if any: none for this scalar.
Definition gemm.hpp:3291
constexpr std::size_t matrix_vector_weight
The weight of a matrix element in matrix_times_vector's share of the threads, beside its multiply-add...
Definition gemm.hpp:5131
std::array< T, gemm_mr *gemm_nr< T > > gemm_tile
Definition gemm.hpp:918
constexpr std::size_t narrow_least_depth
The least depth at which rows of C wider than one block (narrow_columns) run by blocks in gemm_rows; ...
Definition gemm.hpp:1665
packed_left< T > left_rows(const packed_left< T > &left, std::size_t r0, std::size_t rows) noexcept
The rows [r0, r0 + rows) of left, r0 a multiple of left.granule: their slivers in its packing,...
Definition gemm.hpp:4857
constexpr std::size_t gemm_kc
The depth of one panel: a B sliver (gemm_kc × gemm_nr) stays in the L1 cache.
Definition gemm.hpp:878
void gemm_pack_bt(const T *bt, std::size_t ldbt, std::size_t kc, std::size_t cols, T *out)
gemm_pack_b of B = Bᵀᵀ, from the row-major bt (cols × kc, leading dimension ldbt): each column of a s...
Definition gemm.hpp:1132
void gemm_packed(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, const T *a_packed=nullptr, c_start start=c_start::kept)
The packed product with the tile Kernel computes: for each block of B's columns and each panel of the...
Definition gemm.hpp:2583
left_kernel left_kernel_for< double >(std::size_t m, std::size_t n, std::size_t k) noexcept
left_kernel_for a double product: past gemm_rows, as gemm_elsewhere chooses, of a depth of one panel;...
Definition gemm.hpp:4881
double dot_product(const double *x, const double *y, std::size_t count)
Σ x[i] · y[i] for double, as subtract_product chooses its units (in another order of summation there,...
Definition gemm.hpp:715
void complex_scaled_add(C &y, const C &f, const C &x)
y += f · x of one complex number, its part products rounded apart as libc++'s operator computes them ...
Definition simd.hpp:1313
void split_parts_block(std::size_t m, std::size_t nb, std::size_t kb, std::size_t pc, std::size_t jc, const Z *a, std::size_t lda, const Z *b, std::size_t ldb, typename Z::value_type *ar, typename Z::value_type *br, typename Z::value_type *cr)
One block of gemm_split ([Re C | Im C] at cr, m × nb each, += A·B, A m × kb and B kb × nb) by four re...
Definition gemm.hpp:4214
void gemm_rows_rounded(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
C += A·B by the plain i-k-j loop, parallel over the rows of C. Past a depth of 1 (narrow_least_depth)...
Definition gemm.hpp:2244
bool rows_terms_fuse() noexcept
Whether gemm_rows rounds each term c + a·b as one fused multiply-add on the path in use,...
Definition gemm.hpp:2295
constexpr std::size_t row_group
The rows a thread takes of a matrix times a vector (matrix_times_vector).
Definition gemm.hpp:5119
constexpr bool reads_a_in_place
Whether Kernel reads A where it lies (Kernel::tile_a_in_place).
Definition gemm.hpp:2503
constexpr std::size_t split_parts_rows
The most rows of A for which gemm_split runs four real products of a block's parts on the AMX path (s...
Definition gemm.hpp:4154
constexpr std::size_t task_rows_cap(std::size_t) noexcept
The most rows of a task of Kernel over a panel of depth kc: its block of rows.
Definition gemm.hpp:2553
c_start
What a product finds in C: the sums it adds to (C += A·B), or nothing it reads (C = A·B)....
Definition gemm.hpp:1814
@ zero
C = A·B: C's elements before the product are not read.
constexpr std::size_t gemm_rows_weight
The divisor of a block of gemm_rows' rows' multiply-adds as its threads are weighed: on x86-64 its wo...
Definition gemm.hpp:1708
constexpr std::size_t product_rows
The rows of y subtract_rows takes at once.
Definition simd.hpp:731
packed_left< T > read_in_place(std::size_t m, std::size_t k, const T *a, std::size_t lda) noexcept
The m × k a read in place by gemm_with, unpacked: each product packs it as gemm does.
Definition gemm.hpp:4844
constexpr std::size_t gemm_min_depth
Below this depth the plain loop runs too: each panel reads and writes its tiles of C once,...
Definition gemm.hpp:915
bool plain_loop_suits< float >(std::size_t m, std::size_t n, std::size_t k) noexcept
plain_loop_pays for float: double's rule, where the plain loop outran the packed tiles as for double ...
Definition gemm.hpp:4420
bool gemm_double_elsewhere(std::size_t m, std::size_t n, std::size_t k, const double *a, std::size_t lda, const double *b, std::size_t ldb, double *c, std::size_t ldc, c_start start=c_start::kept)
A double product on the path in use (numerical::gemm_path_in_use), unless it is the portable one,...
Definition gemm.hpp:3311
constexpr std::size_t column_rows
The rows of a column-major matrix columns_times_vector takes at once: their partial sums (32 KB on an...
Definition gemm.hpp:5188
split_operands< R > & split_storage()
The calling thread's split_operands. A product uses them whole on one thread, and the real product it...
Definition gemm.hpp:4197
void gemm_kernel(std::size_t kc, const T *ap, const T *bp, std::size_t ldb, gemm_tile< T > &tile)
tile += Aₚ·Bₚ over one panel, for any scalar.
Definition gemm.hpp:929
bool by_rows(std::size_t m, std::size_t n, std::size_t k)
Whether gemm runs an m × k by k × n product by rows (gemm_rows): gemm_rows_columns columns or fewer; ...
Definition gemm.hpp:1758
void gemm_packed_triangle(std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, bool lower)
gemm_packed restricted to a triangle of the square C = A·B (n × n): its lower one (j ≤ i) where lower...
Definition gemm.hpp:2732
constexpr std::size_t gemm_nc
The columns of B packed at once, a whole number of slivers.
Definition gemm.hpp:897
std::atomic< gemm_path > & gemm_path_cap() noexcept
The most capable path products may take, as numerical::limit_gemm_path set it: none is probed until a...
Definition gemm.hpp:128
constexpr std::size_t gemm_rows_columns
The most columns of C for which gemm runs any product by gemm_rows, whose blocks keep C's rows in reg...
Definition gemm.hpp:1681
void gemm_with< double >(const packed_left< double > &left, std::size_t n, const double *b, std::size_t ldb, double *c, std::size_t ldc)
gemm_with for double: on the tile A was packed for, A read from its packing, where gemm would choose ...
Definition gemm.hpp:5008
bool plain_loop_suits< double >(std::size_t m, std::size_t n, std::size_t k) noexcept
plain_loop_pays for double.
Definition gemm.hpp:4406
bool rows_fuse() noexcept
Whether the tile a double or float product past the plain loop runs on fuses each term (gemm_rows rou...
Definition gemm.hpp:168
constexpr std::size_t gemm_slivers_per_task
The B slivers of one task.
Definition gemm.hpp:884
void prefetch_next_row(const T *, std::size_t) noexcept
Nothing, for a tile that prefetches no C (prefetches_next_row_of_c).
Definition gemm.hpp:2477
constexpr bool reads_partial_b_in_place
Whether Kernel reads B's last partial sliver in place (Kernel::partial_b_in_place): a tile that store...
Definition gemm.hpp:2510
constexpr bool writes_fresh
Whether Kernel's tiles write C as it holds nothing, their sums from zero (Kernel::tile's fresh): C th...
Definition gemm.hpp:2496
constexpr std::size_t multiply_add_weight
The real multiply-adds one multiply-add of T weighs: 1 for a real number.
Definition gemm.hpp:5108
T chunked_dot(const T *a, const T *b, std::size_t count, const Dot &dot)
Σ a[i] · b[i] for a plain dot product: chunks of dot_chunk elements in parallel, each by dot,...
Definition gemm.hpp:5066
constexpr bool packs_a(const T *a_packed, std::size_t) noexcept
Whether gemm_packed packs A for Kernel: unless given packed (a_packed).
Definition gemm.hpp:2516
void split_row_columns(const C *a, const C *b, std::size_t row_step, std::size_t column_step, std::size_t inner, std::size_t first, std::size_t last, C *y, bool kept=false)
Columns first to last of the complex vector a (inner) times B (B[k][j] at b[(k · row_step) + (j · col...
Definition gemm.hpp:3895
void gemm_rows(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
C += A·B by the plain i-k-j loop (gemm_rows_rounded), each element rounded as the tile of the path in...
Definition gemm.hpp:2336
void complex_row_dots(const C *m, std::size_t ld, std::size_t rows, const C *x, std::size_t count, C *out)
row_dots of complex numbers: each row by complex_dot_product, as chunked_dot sums a row.
Definition gemm.hpp:783
constexpr std::size_t memory_element_weight
The weight of an element a pass bound by the memory writes or moves, against a multiply-add,...
Definition parallel.hpp:760
term_rounding
How numerical::detail::gemm_rows rounds each term sum + a·b, as the tile of the path in use does: as ...
Definition gemm.hpp:1779
@ x86_fused
std::fma in functions built for AVX2 with FMA.
@ fused
std::fma, where every target has the instruction.
@ compiler
As the compiler rounds sum + a·b (rounded_apart_add).
The NumericAL public API.
Definition amx.hpp:44
std::vector< T, detail::default_init_allocator< T > > buffer
The storage of vector, matrix and tensor, namely a std::vector whose buffer<T>(n) and resize(n) leave...
Definition buffer.hpp:443
std::size_t max_threads()
The threads a kernel may use: the cap if one is set, else the hardware threads (starting none).
Definition parallel.hpp:995
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
T dot(const vector< T > &a, const vector< T > &b)
Dot product.
Definition vector.hpp:275
gemm_path gemm_path_in_use() noexcept
The path double matrix products take.
Definition gemm.hpp:5356
gemm_path limit_gemm_path(gemm_path most) noexcept
Keeps double matrix products on most or a more portable path: the most capable path at or below it th...
Definition gemm.hpp:5349
gemm_path
The kernels a double matrix product can run on (a float one too on arm64), from the most portable....
Definition gemm.hpp:83
@ avx512
AVX-512 with fused multiply-adds.
@ portable
The compiler's vectors (16 bytes on x86-64 wheels), as the compiler rounds c + a·b (see gemm).
@ avx2
AVX2 with fused multiply-adds.
@ amx
The Apple AMX coprocessor, fused multiply-adds; the products it does not suit on the NEON path's tile...
@ neon
arm64's vectors with fused multiply-adds, for double and float.
A minimal std::thread parallel-for for the hot kernels.
The native vector of float and double the kernels hold their lanes in (GNU vector extensions,...
#define NUMERICAL_ALWAYS_INLINE
Definition simd.hpp:25
A as gemm_packed reads it by default: row-major, m × k, its rows lda apart, packed by the tile.
Definition gemm.hpp:1337
static void pack(const T *a, std::size_t lda, std::size_t i0, std::size_t p0, std::size_t rows, std::size_t kc, T *out)
The rows from i0 of A, rows × kc from column p0, packed for Kernel.
Definition gemm.hpp:1341
static const T & at(const T *a, std::size_t lda, std::size_t i, std::size_t p) noexcept
A's element in row i and column p.
Definition gemm.hpp:1392
static const T * rows_of(const T *a, std::size_t lda, std::size_t, std::size_t, buffer< T > &, std::size_t &ld_out)
A row-major for the plain loop: itself.
Definition gemm.hpp:1380
A given as Aᵀ: row-major, k × m, its rows lda apart (C = Aᵀᵀ·B, no copy of the transpose).
Definition gemm.hpp:1403
static const T * rows_of(const T *at, std::size_t ldat, std::size_t rows, std::size_t cols, buffer< T > &scratch, std::size_t &ld_out)
A row-major for the plain loop, copied from at into scratch.
Definition gemm.hpp:1446
static const T & at(const T *at_, std::size_t ldat, std::size_t i, std::size_t p) noexcept
A's element in row i and column p: Aᵀ's in row p and column i.
Definition gemm.hpp:1458
static void pack(const T *at, std::size_t ldat, std::size_t i0, std::size_t p0, std::size_t rows, std::size_t kc, T *out)
The rows from i0 of A, rows × kc from column p0, packed as Kernel lays them out (gemm_pack_at).
Definition gemm.hpp:1408
B as gemm_packed reads it by default: row-major, k × n, its rows ldb apart.
Definition gemm.hpp:1216
static void pack(const T *b, std::size_t ldb, std::size_t p0, std::size_t j0, std::size_t kc, std::size_t cols, T *out)
The kc × cols block of B from row p0 and column j0 into slivers of Nr columns.
Definition gemm.hpp:1220
static constexpr bool in_place(std::size_t ldb) noexcept
Whether Kernel reads B's whole slivers in place.
Definition gemm.hpp:1233
static const T * rows_of(const T *x, std::size_t ld, std::size_t, std::size_t, buffer< T > &, std::size_t &ld_out)
The rows × cols operand row-major for the plain loop: x itself, ld apart.
Definition gemm.hpp:1240
static const T & at(const T *b, std::size_t ldb, std::size_t p, std::size_t j) noexcept
B's element in row p and column j.
Definition gemm.hpp:1252
B given as Bᵀ: row-major, n × k, its rows ldb apart (C = A·Bᵀᵀ, no copy of the transpose).
Definition gemm.hpp:1263
static const T & at(const T *bt, std::size_t ldbt, std::size_t p, std::size_t j) noexcept
B's element in row p and column j: Bᵀ's in row j and column p.
Definition gemm.hpp:1301
static constexpr bool in_place(std::size_t) noexcept
Never in place: a sliver's columns lie apart.
Definition gemm.hpp:1281
static const T * rows_of(const T *x, std::size_t ld, std::size_t rows, std::size_t cols, buffer< T > &scratch, std::size_t &ld_out)
The rows × cols operand row-major for the plain loop, copied from its transpose x (ld apart) into scr...
Definition gemm.hpp:1289
static void pack(const T *bt, std::size_t ldbt, std::size_t p0, std::size_t j0, std::size_t kc, std::size_t cols, T *out)
The kc × cols block of B from row p0 and column j0 (Bᵀ's rows from j0, its columns from p0) into sliv...
Definition gemm.hpp:1268
A left operand packed once for several products by it (an LU's −L₂₁ against blocks of its U₁₂): the t...
Definition gemm.hpp:4832
std::size_t lda
A's row distance.
Definition gemm.hpp:4837
std::size_t k
A's columns, the depth.
Definition gemm.hpp:4835
const T * packed
A packed for the tile (null where plain).
Definition gemm.hpp:4838
std::size_t m
A's rows.
Definition gemm.hpp:4834
const T * a
A, row-major.
Definition gemm.hpp:4836
left_kernel kernel
The tile.
Definition gemm.hpp:4833
std::size_t granule
The rows of the tile's slivers: a block of rows from a multiple of it reads the packing.
Definition gemm.hpp:4839
The real operands of gemm_split, kept by each thread from one product to the next (split_storage): ma...
Definition gemm.hpp:4186
buffer< R > b
B's parts for a block of columns and of depth: [Re B | Im B], or [[Re B, Im B], [Im B,...
Definition gemm.hpp:4188
buffer< R > c
[Re C | Im C], a block of columns.
Definition gemm.hpp:4189
buffer< R > a
A's parts for a block of depth: [Re A | −Im A | Im A], or [Re A | −Im A].
Definition gemm.hpp:4187
Nothing to turn on around a task.
Definition gemm.hpp:1488
The vector path's tile: gemm_mr × gemm_nr elements of C in registers (gemm_kernel),...
Definition gemm.hpp:1473
static constexpr bool reads_b_in_place(std::size_t ldb) noexcept
Whether B's rows ldb apart are read in place: no farther apart than a packed sliver's (and a B that o...
Definition gemm.hpp:1492
static constexpr std::size_t speedup
The weight of its work (gemm_kernel_speedup).
Definition gemm.hpp:1483
static constexpr std::size_t b_in_place_rows
The most rows of A for which gemm_packed reads B in place at any distance between its rows (where rea...
Definition gemm.hpp:1482
static constexpr std::size_t depth
The panel depth.
Definition gemm.hpp:1476
static void tile(std::size_t kc, const T *ap, const T *bp, std::size_t ldb, T *c, std::size_t ldc, std::size_t rows, std::size_t cols)
C += Aₚ·Bₚ for one tile of C (rows × cols) over a panel of depth kc, B's rows ldb apart.
Definition gemm.hpp:1508
static constexpr std::size_t mr
The tile's rows.
Definition gemm.hpp:1474
static constexpr std::size_t serial_below
Every product with work shares it (gemm_shares).
Definition gemm.hpp:1484
static constexpr std::size_t nr
The tile's columns.
Definition gemm.hpp:1475
static void pack_a(const T *a, std::size_t lda, std::size_t rows, std::size_t kc, T *out)
A's slivers for this tile (gemm_pack_a).
Definition gemm.hpp:1498
static constexpr std::size_t block_rows
Definition gemm.hpp:1477
The x86-64 tiles of the double matrix product: AVX-512 and AVX2 with FMA, chosen at run time.