14#ifndef NUMERICAL_BLAS_HPP
15#define NUMERICAL_BLAS_HPP
47 inline constexpr std::size_t
syrk_kept_limit {(std::size_t {32} << 20U) /
sizeof(
double)};
62 return op ==
'N' ||
op ==
'n' ||
op ==
'T' ||
op ==
't' ||
op ==
'C' ||
op ==
'c';
68 return op !=
'N' &&
op !=
'n';
74 return op ==
'C' ||
op ==
'c';
81 return std::isfinite(x);
88 return std::isfinite(x.real()) && std::isfinite(x.imag());
104 template <
typename T>
113 const auto r {
static_cast<std::size_t
>(rows)};
114 const auto c {
static_cast<std::size_t
>(cols)};
115 const auto ld {
static_cast<std::size_t
>(lda)};
117 T*
const to {out.data()};
123 [a, ld, c,
to, scale,
conjugate](std::size_t j, std::size_t i) {
124 const T x {a[i + (j * ld)]};
133 template <
typename T>
147 inline constexpr std::size_t
kept_bytes {std::size_t {32} << 20U};
154 template <
typename T>
157 thread_local std::array<buffer<T>, 2> kept;
163 template <
typename T>
170 template <
typename T>
185 template <
typename T>
193 if (std::memcmp(b,
middle, n *
sizeof(
T)) != 0) {
196 std::atomic<bool> holds {
true};
198 if (std::memcmp(b + ((j + 1) *
ldb),
middle - (j + 1), n *
sizeof(
T)) != 0) {
199 holds.store(
false, std::memory_order_relaxed);
202 return holds.load(std::memory_order_relaxed);
206 template <
typename T>
218 [
from, cols, a, lda](std::size_t i, std::size_t j) { a[i + (j * lda)] =
from[(i * cols) + j]; });
224 template <
typename T>
227 const std::size_t*
to,
236 [
from, n,
to, a, lda](std::size_t i, std::size_t j) { a[i + (
to[j] * lda)] =
from[(i * n) + j]; });
246 template <
typename T>
287 template <
typename T>
299 [a, lda, n, rows,
in](std::size_t j, std::size_t i) {
304 rows[(i * n) + j] = a[i + (j * lda)];
307 a[i + (j * lda)] = rows[(i * n) + j];
316 template <
typename T>
323 const auto r {
static_cast<std::size_t
>(rows)};
324 const auto c {
static_cast<std::size_t
>(cols)};
326 for (std::size_t j {0}; j < c; ++j) {
327 for (std::size_t i {0}; i < r; ++i) {
328 out[(j * r) + i] = scale * a[i + (j *
static_cast<std::size_t
>(lda))];
340 template <
typename T>
357 left =
row_major(b,
static_cast<std::ptrdiff_t
>(
ldb),
static_cast<std::ptrdiff_t
>(m),
static_cast<std::ptrdiff_t
>(k),
alpha,
361 left =
columns_as_rows(b,
static_cast<std::ptrdiff_t
>(
ldb),
static_cast<std::ptrdiff_t
>(k),
static_cast<std::ptrdiff_t
>(m),
alpha);
383 product_of_ops<double>(
transa,
transb,
alpha, m, n, k, a, lda, b,
ldb, c, ldc);
398 template <
typename Index>
402 const std::size_t size {pivots.size()};
403 std::vector<std::size_t> at(size);
404 std::vector<std::size_t>
where(size);
405 for (std::size_t i {0}; i < size; ++i) {
409 for (std::size_t i {0}; i < size; ++i) {
410 const std::size_t
from {
where[pivots[i]]};
413 where[pivots[i]] = i;
414 std::swap(at[i], at[
from]);
422 template <
typename T,
typename Index>
430 for (std::size_t j {0}; j < std::min(m, n); ++j) {
431 T*
const column {a + (j * lda)};
432 std::size_t pivot {j};
433 for (std::size_t i {j + 1}; i < m; ++i) {
443 for (std::size_t c {0}; c < n; ++c) {
444 std::swap(a[j + (c * lda)], a[pivot + (c * lda)]);
446 for (std::size_t i {j + 1}; i < m; ++i) {
449 for (std::size_t c {j + 1}; c < n; ++c) {
450 T*
const target {a + (c * lda)};
452 for (std::size_t i {j + 1}; i < m; ++i) {
468 template <
typename T>
501 else if (lda < std::max<std::ptrdiff_t>(1, a_rows)) {
504 else if (
ldb < std::max<std::ptrdiff_t>(1, b_rows)) {
507 else if (ldc < std::max<std::ptrdiff_t>(1, m)) {
510 if (
illegal != 0 || m == 0 || n == 0) {
513 const auto rows {
static_cast<std::size_t
>(m)};
514 const auto cols {
static_cast<std::size_t
>(n)};
515 const auto stride {
static_cast<std::size_t
>(ldc)};
516 for (std::size_t j {0}; j < cols; ++j) {
519 std::fill_n(
column, rows,
T {});
521 else if (beta !=
T {1}) {
525 if (
alpha ==
T {} || k == 0) {
529 static_cast<std::size_t
>(
ldb), c,
stride);
539 std::ptrdiff_t
inc)
noexcept
542 return inc > 0 ? i *
static_cast<std::size_t
>(
inc) : (length - 1 - i) * (std::size_t {0} -
static_cast<std::size_t
>(
inc));
546 template <
typename T>
556 const auto count {
static_cast<std::size_t
>(n)};
562 for (std::size_t i {0}; i < count; ++i) {
601 template <
typename T>
604 const std::complex<T>* x,
606 const std::complex<T>*
y,
607 std::ptrdiff_t
incy)
noexcept
609 std::complex<T>
sum {};
613 const auto count {
static_cast<std::size_t
>(n)};
617 for (std::size_t i {0}; i < count; ++i) {
661 if (lda < std::max<std::ptrdiff_t>(1, m)) {
670 if (m == 0 || n == 0 || (
alpha == 0.0 && beta == 1.0)) {
674 const auto rows {
static_cast<std::size_t
>(m)};
675 const auto cols {
static_cast<std::size_t
>(n)};
676 const auto ld {
static_cast<std::size_t
>(lda)};
677 const std::size_t
x_length {transposed ? rows : cols};
678 const std::size_t
y_length {transposed ? cols : rows};
683 for (std::size_t i {0}; i <
y_length; ++i) {
687 const bool in_place {transposed &&
incx == 1 &&
alpha == 1.0};
689 for (std::size_t i {0}; i < scaled.size(); ++i) {
693 const double*
const factors {in_place ? x : scaled.data()};
698 factors, rows,
dots.data() + first);
700 for (std::size_t j {0}; j < cols; ++j) {
706 for (
double& factor : scaled) {
716 for (std::size_t i {0};
incy != 1 && i <
y_length; ++i) {
776 for (std::size_t j {0}; j < w; ++j) {
777 for (std::size_t i {
operands.lower ? j : 0}; i < (
operands.lower ? w : j + 1); ++i) {
783 const std::size_t h {w / 2};
787 const std::size_t rows {
operands.lower ? w - h : h};
788 const std::size_t cols {
operands.lower ? h : w - h};
802 const std::size_t k {
operands.depth};
841 const bool lower {
uplo ==
'L' ||
uplo ==
'l'};
843 if (!lower &&
uplo !=
'U' &&
uplo !=
'u') {
855 if (lda < std::max<std::ptrdiff_t>(1, transposed ? k : n)) {
858 if (ldc < std::max<std::ptrdiff_t>(1, n)) {
864 const auto order {
static_cast<std::size_t
>(n)};
865 const auto ld {
static_cast<std::size_t
>(ldc)};
866 if (
alpha == 0.0 || k == 0) {
867 for (std::size_t j {0}; j < order && beta != 1.0; ++j) {
868 for (std::size_t i {lower ? j : 0}; i < (lower ? order : j + 1); ++i) {
869 c[i + (j * ld)] = beta == 0.0 ? 0.0 : beta * c[i + (j * ld)];
878 const auto depth {
static_cast<std::size_t
>(k)};
879 const auto lda_s {
static_cast<std::size_t
>(lda)};
880 for (std::size_t j {0}; j < order; ++j) {
881 for (std::size_t i {lower ? j : 0}; i < (lower ? order : j + 1); ++i) {
882 double&
entry {c[i + (j * ld)]};
887 detail::syrk_all({transposed ? a :
nullptr,
lda_s, transposed ?
nullptr : a,
lda_s, depth, c, ld, lower, transposed, !transposed}, order);
893 const std::size_t
copy_rows {transposed ? depth : order};
894 const std::size_t
copy_cols {transposed ? order : depth};
898 for (std::size_t i {0}; i <
copy_rows; ++i) {
899 for (std::size_t j {0}; j <
copy_cols; ++j) {
903 const double*
const p_rows {transposed ? a :
copy.data()};
904 const std::size_t ldp {transposed ?
lda_s : depth};
905 const double*
const q_cols {transposed ?
copy.data() : a};
906 const std::size_t ldq {transposed ? order :
lda_s};
907 detail::syrk_all({
p_rows, ldp,
q_cols, ldq, depth, c, ld, lower,
false,
false}, order);
934 for (std::size_t done {0}; done < k; done +=
trsm_block) {
935 const std::size_t w {std::min(
trsm_block, k - done)};
936 const std::size_t
k0 {lower ? done : k - done - w};
937 const std::size_t
k1 {
k0 + w};
940 double* x {b + (j *
ldb)};
941 for (std::size_t s {0}; s <
k1 -
k0; ++s) {
942 const std::size_t p {lower ?
k0 + s :
k1 - 1 - s};
944 x[p] /= a[p + (p * lda)];
946 const std::size_t
from {lower ? p + 1 :
k0};
947 const std::size_t
to {lower ?
k1 : p};
951 const std::size_t first {lower ?
k1 : 0};
952 const std::size_t count {lower ? k -
k1 :
k0};
954 (
void)
gemm(
'N',
'N',
static_cast<std::ptrdiff_t
>(count),
static_cast<std::ptrdiff_t
>(n),
static_cast<std::ptrdiff_t
>(w), -1.0,
955 a + first + (
k0 * lda),
static_cast<std::ptrdiff_t
>(lda), b +
k0,
static_cast<std::ptrdiff_t
>(
ldb), 1.0, b + first,
956 static_cast<std::ptrdiff_t
>(
ldb));
983 const bool lower {
uplo ==
'L' ||
uplo ==
'l'};
988 if (!lower &&
uplo !=
'U' &&
uplo !=
'u') {
1003 if (lda < std::max<std::ptrdiff_t>(1,
left ? m : n)) {
1006 if (
ldb < std::max<std::ptrdiff_t>(1, m)) {
1009 if (m == 0 || n == 0) {
1012 const auto rows {
static_cast<std::size_t
>(m)};
1013 const auto cols {
static_cast<std::size_t
>(n)};
1014 const auto ld {
static_cast<std::size_t
>(
ldb)};
1015 for (std::size_t j {0}; j < cols; ++j) {
1016 for (std::size_t i {0}; i < rows; ++i) {
1017 double&
entry {b[i + (j * ld)]};
1024 const std::size_t k {
left ? rows : cols};
1027 const double*
a_used {a};
1028 std::size_t
lda_used {
static_cast<std::size_t
>(lda)};
1031 for (std::size_t j {0}; j < k; ++j) {
1032 for (std::size_t i {lower ? j : 0}; i < (lower ? k : j + 1); ++i) {
1033 transposed_a[j + (i * k)] = a[i + (j *
static_cast<std::size_t
>(lda))];
1044 for (std::size_t j {0}; j < cols; ++j) {
1045 for (std::size_t i {0}; i < rows; ++i) {
1046 bt[j + (i * cols)] = b[i + (j * ld)];
1050 for (std::size_t j {0}; j < cols; ++j) {
1051 for (std::size_t i {0}; i < rows; ++i) {
1052 b[i + (j * ld)] =
bt[j + (i * cols)];
1068 template <
typename T,
typename Index>
1079 const auto at = [&](std::size_t i, std::size_t j) {
return a[i + (j * ld)]; };
1082 for (std::size_t c {0}; c <
nrhs; ++c) {
1083 T*
const x {b + (c *
ldb)};
1085 for (std::size_t i {0}; i < size; ++i) {
1086 std::swap(x[i], x[
static_cast<std::size_t
>(
ipiv[i]) - 1]);
1088 for (std::size_t j {0}; j < size; ++j) {
1089 for (std::size_t i {j + 1}; i < size; ++i) {
1090 x[i] -= at(i, j) * x[j];
1093 for (std::size_t j {size}; j-- > 0;) {
1095 for (std::size_t i {0}; i < j; ++i) {
1096 x[i] -= at(i, j) * x[j];
1101 for (std::size_t i {0}; i < size; ++i) {
1103 for (std::size_t k {0}; k < i; ++k) {
1108 for (std::size_t i {size}; i-- > 0;) {
1110 for (std::size_t k {i + 1}; k < size; ++k) {
1115 for (std::size_t i {size}; i-- > 0;) {
1116 std::swap(x[i], x[
static_cast<std::size_t
>(
ipiv[i]) - 1]);
1125 template <
typename Index>
1139 const auto n {
static_cast<std::ptrdiff_t
>(size)};
1140 const auto columns {
static_cast<std::ptrdiff_t
>(
nrhs)};
1141 const auto lda {
static_cast<std::ptrdiff_t
>(ld)};
1142 const auto ldb_ {
static_cast<std::ptrdiff_t
>(
ldb)};
1143 const auto exchange = [&](std::size_t i) {
1144 const auto other {
static_cast<std::size_t
>(
ipiv[i]) - 1};
1145 for (std::size_t c {0}; c <
nrhs; ++c) {
1146 std::swap(b[i + (c *
ldb)], b[other + (c *
ldb)]);
1150 for (std::size_t i {0}; i < size; ++i) {
1153 (
void)
trsm(
'L',
'L',
'N',
'U', n, columns, 1.0, a, lda, b,
ldb_);
1154 (
void)
trsm(
'L',
'U',
'N',
'N', n, columns, 1.0, a, lda, b,
ldb_);
1157 (
void)
trsm(
'L',
'U',
'T',
'N', n, columns, 1.0, a, lda, b,
ldb_);
1158 (
void)
trsm(
'L',
'L',
'T',
'U', n, columns, 1.0, a, lda, b,
ldb_);
1159 for (std::size_t i {size}; i-- > 0;) {
1165 template <
typename T,
typename Index>
1168 std::ptrdiff_t
nrhs,
1173 std::ptrdiff_t
ldb);
1185 template <
typename T,
typename Index>
1187 std::ptrdiff_t
nrhs,
1200 if (lda < std::max<std::ptrdiff_t>(1, n)) {
1203 if (
ldb < std::max<std::ptrdiff_t>(1, n)) {
1209 const auto size {
static_cast<std::size_t
>(n)};
1210 const auto ld {
static_cast<std::size_t
>(lda)};
1211 std::vector<std::size_t> pivots;
1213 bool singular {
false};
1219 pivots =
lu.pivots();
1220 singular =
lu.singular();
1221 if (!singular &&
nrhs > 0) {
1239 factors = std::move(
lu).factors();
1251 for (std::size_t i {0}; i < size; ++i) {
1252 if (a[i + (i * ld)] ==
T {}) {
1253 return static_cast<int>(i + 1);
1266 template <
typename T>
1273 const auto l = [&](std::size_t i, std::size_t j) ->
T& {
return lower ? a[i + (j * ld)] : a[j + (i * ld)]; };
1275 for (std::size_t j {0}; j < size; ++j) {
1276 auto diagonal {std::real(a[j + (j * ld)])};
1277 for (std::size_t k {0}; k < j; ++k) {
1283 return static_cast<int>(j + 1);
1287 for (std::size_t i {j + 1}; i < size; ++i) {
1289 for (std::size_t k {0}; k < j; ++k) {
1309 template <
typename T>
1315 const bool lower {
uplo ==
'L' ||
uplo ==
'l'};
1316 if (!lower &&
uplo !=
'U' &&
uplo !=
'u') {
1322 if (lda < std::max<std::ptrdiff_t>(1, n)) {
1328 const auto size {
static_cast<std::size_t
>(n)};
1329 const auto ld {
static_cast<std::size_t
>(lda)};
1339 for (std::size_t i {0}; i < size; ++i) {
1340 for (std::size_t j {0}; j <= i; ++j) {
1351 for (std::size_t i {0}; i < size; ++i) {
1352 for (std::size_t j {0}; j <= i; ++j) {
1360 return static_cast<int>(failed.
order());
1375 template <
typename T,
typename Index>
1388 if (lda < std::max<std::ptrdiff_t>(1, m)) {
1391 if (m == 0 || n == 0) {
1394 const auto rows {
static_cast<std::size_t
>(m)};
1395 const auto cols {
static_cast<std::size_t
>(n)};
1396 const auto ld {
static_cast<std::size_t
>(lda)};
1402 std::optional<lu_decomposition<T>>
factored;
1414 for (std::size_t i {0}; i < rows; ++i) {
1415 if (a[i + (i * ld)] ==
T {}) {
1416 return static_cast<int>(i + 1);
1428 template <
typename T,
typename Index>
1431 std::ptrdiff_t
nrhs,
1447 if (lda < std::max<std::ptrdiff_t>(1, n)) {
1450 if (
ldb < std::max<std::ptrdiff_t>(1, n)) {
1453 const auto size {
static_cast<std::size_t
>(n)};
1454 const auto ld {
static_cast<std::size_t
>(lda)};
1462 template <
typename T>
1472 const auto l = [&](std::size_t i, std::size_t j) {
return lower ? a[i + (j * ld)] :
numerical::detail::conj_of(a[j + (i * ld)]); };
1473 for (std::size_t c {0}; c <
nrhs; ++c) {
1474 T*
const x {b + (c *
ldb)};
1475 for (std::size_t i {0}; i < size; ++i) {
1477 for (std::size_t k {0}; k < i; ++k) {
1478 sum -= l(i, k) * x[k];
1480 x[i] =
sum / l(i, i);
1482 for (std::size_t i {size}; i-- > 0;) {
1484 for (std::size_t k {i + 1}; k < size; ++k) {
1506 const auto n {
static_cast<std::ptrdiff_t
>(size)};
1507 const auto columns {
static_cast<std::ptrdiff_t
>(
nrhs)};
1508 const char uplo {lower ?
'L' :
'U'};
1509 (
void)
trsm(
'L',
uplo, lower ?
'N' :
'T',
'N', n, columns, 1.0, a,
static_cast<std::ptrdiff_t
>(ld), b,
static_cast<std::ptrdiff_t
>(
ldb));
1510 (
void)
trsm(
'L',
uplo, lower ?
'T' :
'N',
'N', n, columns, 1.0, a,
static_cast<std::ptrdiff_t
>(ld), b,
static_cast<std::ptrdiff_t
>(
ldb));
1520 template <
typename T>
1523 std::ptrdiff_t
nrhs,
1529 const bool lower {
uplo ==
'L' ||
uplo ==
'l'};
1530 if (!lower &&
uplo !=
'U' &&
uplo !=
'u') {
1539 if (lda < std::max<std::ptrdiff_t>(1, n)) {
1542 if (
ldb < std::max<std::ptrdiff_t>(1, n)) {
1545 const auto size {
static_cast<std::size_t
>(n)};
1546 const auto ld {
static_cast<std::size_t
>(lda)};
1562 template <
typename Index>
1570 std::ptrdiff_t
lwork,
1574 const bool vectors {
jobz ==
'V' ||
jobz ==
'v'};
1575 const bool lower {
uplo ==
'L' ||
uplo ==
'l'};
1576 if (!vectors &&
jobz !=
'N' &&
jobz !=
'n') {
1579 if (!lower &&
uplo !=
'U' &&
uplo !=
'u') {
1585 if (lda < std::max<std::ptrdiff_t>(1, n)) {
1591 lwork_least = vectors ? 1 + (6 * n) + (2 * n * n) : (2 * n) + 1;
1608 const auto size {
static_cast<std::size_t
>(n)};
1609 const auto ld {
static_cast<std::size_t
>(lda)};
1611 for (std::size_t i {0}; i < size; ++i) {
1612 for (std::size_t j {0}; j <= i; ++j) {
1613 lower_rows[(i * size) + j] = lower ? a[i + (j * ld)] : a[j + (i * ld)];
1620 std::copy_n(&values[0], size, w);
1624 std::copy_n(&
eigen.values()[0], size, w);
1625 for (std::size_t j {0}; j < size; ++j) {
1626 for (std::size_t i {0}; i < size; ++i) {
1627 a[i + (j * ld)] =
eigen.vectors()(i, j);
1632 std::fill_n(w, size, std::numeric_limits<double>::quiet_NaN());
1634 for (std::size_t j {0}; j < size; ++j) {
1635 std::fill_n(a + (j * ld), size, std::numeric_limits<double>::quiet_NaN());
1638 return static_cast<int>(size);
1652 template <
typename Index>
1662 std::ptrdiff_t
ldvt,
1664 std::ptrdiff_t
lwork,
1681 if (lda < std::max<std::ptrdiff_t>(1, m)) {
1684 const std::ptrdiff_t
least {std::min(m, n)};
1685 const std::ptrdiff_t
most {std::max(m, n)};
1688 if (
ldu < std::max<std::ptrdiff_t>(1,
u_out ? m : 1)) {
1698 if (
ldvt < std::max<std::ptrdiff_t>(1,
vt_rows)) {
1709 work[0] =
static_cast<double>(std::max<std::ptrdiff_t>(1,
lwork_least));
1718 const auto rows {
static_cast<std::size_t
>(m)};
1719 const auto cols {
static_cast<std::size_t
>(n)};
1720 const auto k {
static_cast<std::size_t
>(
least)};
1725 std::copy_n(&values[0], k, s);
1729 std::copy_n(&
svd.singular_values()[0], k, s);
1734 const std::size_t
ld_u {
overwrite && m >= n ?
static_cast<std::size_t
>(lda) :
static_cast<std::size_t
>(
ldu)};
1737 for (std::size_t j {0}; j <
left.cols(); ++j) {
1738 for (std::size_t i {0}; i <
left.rows(); ++i) {
1742 for (std::size_t j {0}; j <
right.cols(); ++j) {
1743 for (std::size_t i {0}; i <
right.rows(); ++i) {
1770 std::ptrdiff_t
lwork)
1778 if (lda < std::max<std::ptrdiff_t>(1, m)) {
1781 const std::ptrdiff_t
lwork_least {std::min(m, n) == 0 ? 1 : n};
1789 if (std::min(m, n) == 0) {
1792 const auto rows {
static_cast<std::size_t
>(m)};
1793 const auto cols {
static_cast<std::size_t
>(n)};
1794 const auto ld {
static_cast<std::size_t
>(lda)};
1799#ifdef NUMERICAL_X86_COMPILED
1800 entries.resize(rows * cols);
1801 for (std::size_t j {0}; j < cols; ++j) {
1802 std::copy_n(a + (j * ld), rows, entries.data() + (j * rows));
1807#ifdef NUMERICAL_X86_COMPILED
1809 for (std::size_t j {0}; j < cols; ++j) {
1810 std::copy_n(entries.data() + (j * rows), rows, a + (j * ld));
1818 for (std::size_t j {0}; j < block.width; ++j) {
1819 tau[block.first + j] = block.t[(j * block.width) + j];
1841 std::ptrdiff_t
lwork)
1852 if (lda < std::max<std::ptrdiff_t>(1, m)) {
1855 const std::ptrdiff_t
lwork_least {std::max<std::ptrdiff_t>(1, n)};
1866 const auto rows {
static_cast<std::size_t
>(m)};
1867 const auto cols {
static_cast<std::size_t
>(n)};
1868 const auto depth {
static_cast<std::size_t
>(k)};
1869 const auto ld {
static_cast<std::size_t
>(lda)};
1871 std::vector<buffer<double>> reflectors(depth);
1872 const std::vector<double>
taus(tau, tau + depth);
1873 for (std::size_t j {0}; j < depth; ++j) {
1874 reflectors[j].resize(rows - j);
1875 reflectors[j][0] = 1.0;
1876 std::copy_n(a + (j + 1) + (j * ld), rows - j - 1, reflectors[j].data() + 1);
1878 std::vector<numerical::detail::reflector_block<double>> blocks;
1884 for (std::size_t j {0}; j < cols; ++j) {
1885 std::copy_n(columns.data() + (j * rows), rows, a + (j * ld));
1902 template <
typename Index>
1905 std::ptrdiff_t
nrhs,
1914 std::ptrdiff_t
lwork,
1926 if (lda < std::max<std::ptrdiff_t>(1, m)) {
1929 if (
ldb < std::max<std::ptrdiff_t>({1, m, n})) {
1933 constexpr std::ptrdiff_t
leaf {25};
1934 const std::ptrdiff_t
least {std::max<std::ptrdiff_t>(1, std::min(m, n))};
1935 const auto levels {std::max<std::ptrdiff_t>(
1936 static_cast<std::ptrdiff_t
>(std::log2(
static_cast<double>(
least) /
static_cast<double>(
leaf + 1))) + 1, 0)};
1948 if (m == 0 || n == 0) {
1951 const auto rows {
static_cast<std::size_t
>(m)};
1952 const auto cols {
static_cast<std::size_t
>(n)};
1953 const auto rhs {
static_cast<std::size_t
>(
nrhs)};
1954 const auto ld {
static_cast<std::size_t
>(
ldb)};
1956 for (std::size_t i {0}; i < rows; ++i) {
1957 for (std::size_t c {0}; c <
rhs; ++c) {
1958 right[(i *
rhs) + c] = b[i + (c * ld)];
1961 if (!std::ranges::all_of(
right, [](
double x) {
return std::isfinite(x); })) {
1968 std::copy_n(&
fit.singular_values[0],
fit.singular_values.size(), s);
1969 *rank =
static_cast<Index>(
fit.rank);
1970 for (std::size_t c {0}; c <
rhs; ++c) {
1971 for (std::size_t i {0}; i < cols; ++i) {
1972 b[i + (c * ld)] =
fit.solution(i, c);
1974 for (std::size_t i {cols}; i < rows; ++i) {
1975 b[i + (c * ld)] = i == cols &&
fit.residuals.size() > 0 ? std::sqrt(
fit.residuals[c]) : 0.0;
1991 const vector<std::complex<double>>& values,
1995 const std::size_t n {values.size()};
1996 for (std::size_t j {0}; j < n; ++j) {
1997 if (values[j].
imag() == 0.0) {
1998 for (std::size_t i {0}; i < n; ++i) {
1999 out[i + (j * ld)] = v(i, j).real();
2002 else if (values[j].
imag() > 0.0) {
2003 for (std::size_t i {0}; i < n; ++i) {
2004 out[i + (j * ld)] = v(i, j).real();
2005 out[i + ((j + 1) * ld)] = v(i, j).imag();
2032 std::ptrdiff_t
ldvl,
2034 std::ptrdiff_t
ldvr,
2036 std::ptrdiff_t
lwork)
2049 if (lda < std::max<std::ptrdiff_t>(1, n)) {
2069 const auto size {
static_cast<std::size_t
>(n)};
2074 for (std::size_t j {0}; j < size; ++j) {
2075 wr[j] = values[j].real();
2076 wi[j] = values[j].imag();
A Cholesky factorisation A = L Lᴴ of a Hermitian positive-definite matrix.
The eigenvalues, and optionally the right eigenvectors, of a general square matrix,...
An LU factorisation with partial pivoting of a square matrix: each column's pivot the first entry of ...
A dense, row-major matrix of T with value semantics.
buffer< T >::const_iterator begin() const noexcept
Const iterator to the first element (row-major).
buffer< T > elements() &&
The elements, row-major, handed over by a matrix about to expire, which is left 0 × 0: a large result...
Thrown when a decomposition's input holds an infinite or NaN entry: no factorisation of it means anyt...
Thrown when a Cholesky factorisation finds the matrix not positive-definite.
std::size_t order() const noexcept
The order of the first leading minor found not positive definite (LAPACK's info), 0 where unknown.
The magnitude at or under which a singular value counts as zero, in a rank (matrix_rank),...
static rank_cutoff relative(double ratio)
The largest singular value times ratio.
The singular value decomposition A = U Σ Vᴴ of an m × n matrix.
The eigenvalues and eigenvectors of a real symmetric matrix.
A dense vector of T with value semantics.
Matrix factorisations and the algorithms built on them.
The eigenvalues and eigenvectors of a general square matrix.
The matrix product kernel behind matrix * matrix and the einsum matmul path.
A dense, row-major, value-semantic matrix over a scalar type.
void potrs_columns(bool lower, std::size_t size, std::size_t nrhs, const T *a, std::size_t ld, T *b, std::size_t ldb)
potrs's solves, each column of B by forward then backward substitution with L (lower) or Uᴴ (size × n...
void syrk_product(const syrk_operands &operands, std::size_t c0, std::size_t cols, std::size_t r0, std::size_t rows, double *out, std::size_t ldo)
P(c0…) Q(…, r0…) added to the cols × rows row-major block at out (ldo apart): numerical::detail::gemm...
void syrk_triangle(syrk_operands operands, std::size_t j0, std::size_t w)
Adds P Q to the triangle of the diagonal block [j0, j0 + w) of C: halved until it is a block of syrk_...
buffer< double > & syrk_operand() noexcept
The storage of syrk's operand copy this thread keeps between calls.
void lower_triangle_rows(T *a, std::size_t lda, std::size_t n, T *rows, bool in)
The lower triangle of the column-major n × n a (lda apart) into that of the row-major rows (n apart) ...
buffer< T > & kept_storage(std::size_t slot) noexcept
Row-major storage of the scalar T this thread keeps between calls, in slot (0 for a call's A,...
bool is_identity(const T *b, std::size_t ldb, std::size_t n)
Whether the column-major n × n b (ldb apart) is the identity to the bit (a -0 is no zero of it,...
void triangular_solve(bool lower, bool unit, std::size_t k, std::size_t n, const double *a, std::size_t lda, double *b, std::size_t ldb)
A X = B in place for the column-major triangular k × k a (lower or upper, a unit diagonal not read) a...
constexpr std::size_t strided(std::size_t i, std::size_t length, std::ptrdiff_t inc) noexcept
Index of element i of a BLAS vector of length elements inc apart: from the last one where inc is nega...
bool finite(const T &x)
Whether the real x is finite (an overload per kind of scalar, as conj_of: each one expression).
void column_major_permuted_into(const T *from, std::size_t n, const std::size_t *to, T *a, std::size_t lda)
column_major_into for the n × n from, its column j written as column to[j] of a.
constexpr bool is_operation(char op) noexcept
Whether op is one of BLAS's N, T or C (either case): no transpose, transpose, conjugate one.
constexpr std::size_t kept_bytes
The most bytes a thread keeps in each of kept_storage's slots between calls (32 MiB).
int rectangular_lu(std::size_t m, std::size_t n, T *a, std::size_t lda, Index *ipiv)
The LU factorisation with partial pivoting of a rectangular column-major A in place,...
void give_back(std::size_t slot, buffer< T > &&storage)
storage given back to kept_storage's slot, unless past kept_bytes.
void interchanges(const std::vector< std::size_t > &pivots, Index *ipiv)
The permutation pivots (row i of P A is row pivots[i] of A) as LAPACK's successive row interchanges,...
void getrs_columns(char trans, std::size_t size, std::size_t nrhs, const T *a, std::size_t ld, const Index *ipiv, T *b, std::size_t ldb)
getrs's solves, each column of B by forward then backward substitution (size × nrhs,...
constexpr bool conjugates(char op) noexcept
Whether op is BLAS's C: the conjugate transpose (a real matrix's is its transpose).
void row_major_into(buffer< T > &out, const T *a, std::ptrdiff_t lda, std::ptrdiff_t rows, std::ptrdiff_t cols, T scale, bool conjugate=false)
The column-major rows × cols a (lda apart) into the row-major out (resized to it),...
void column_major_into(const T *from, std::size_t rows, std::size_t cols, T *a, std::size_t lda)
The row-major rows × cols from into the column-major a (lda apart), by tiles.
buffer< T > row_major(const T *a, std::ptrdiff_t lda, std::ptrdiff_t rows, std::ptrdiff_t cols, T scale, bool conjugate=false)
rows × cols of the column-major a (lda apart), as a row-major buffer, each element times scale (conju...
buffer< T > take_kept(std::size_t slot)
kept_storage's slot, taken over by the caller (left empty there meanwhile).
void product_by_op_a(char transa, std::size_t m, std::size_t n, std::size_t k, const T *l, std::size_t ldl, const T *a, std::size_t lda, T *c, std::size_t ldc)
C += L op(A)ᵀ (C row-major m × n, L m × k), a the column-major op(A)'s storage (lda apart): op(A)ᵀ is...
int column_cholesky(bool lower, std::size_t size, T *a, std::size_t ld)
The Cholesky factor of the column-major order-size A in place, column by column as the reference's dp...
void syrk_all(const syrk_operands &operands, std::size_t n)
The whole triangle of C (order n) from operands: a product the matrix product kernel packs runs in on...
buffer< T > columns_as_rows(const T *a, std::ptrdiff_t lda, std::ptrdiff_t rows, std::ptrdiff_t cols, T scale)
The cols columns of rows elements of the column-major a (lda apart) as the rows of a row-major buffer...
constexpr bool transposes(char op) noexcept
Whether op transposes (T or C: a real matrix's conjugate transpose is its transpose).
constexpr std::size_t trsm_columns_least
The fewest columns of B getrs and potrs of double solve by trsm: fewer take the substitution column b...
constexpr std::size_t transpose_weight
The weight of an element of the transposing copies against a multiply-add: bound by the memory,...
T real_dot(std::ptrdiff_t n, const T *x, std::ptrdiff_t incx, const T *y, std::ptrdiff_t incy)
numerical::blas::dot of the real scalar T.
constexpr std::size_t transpose_tile
The tile side of the transposing copies between column-major arguments and row-major storage: a tile'...
void real_form(const matrix< std::complex< double > > &v, const vector< std::complex< double > > &values, double *out, std::size_t ld)
The eigenvectors v (columns, complex, matching values) in dgeev's real form, into the column-major ou...
void product_of_ops(char transa, char transb, T alpha, std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc)
C += α op(B)ᵀ op(A)ᵀ, row-major C (m × n, ldc apart), the column-major C of BLAS's gemm as its transp...
std::complex< T > complex_dot(bool conjugate, std::ptrdiff_t n, const std::complex< T > *x, std::ptrdiff_t incx, const std::complex< T > *y, std::ptrdiff_t incy) noexcept
Σ x_i y_i (conjugate false, BLAS's zdotu) or Σ conj(x_i) y_i (true, zdotc) over n complex elements in...
int potrf(char uplo, std::ptrdiff_t n, T *a, std::ptrdiff_t lda)
The Cholesky factor of a symmetric positive-definite A, column-major, as LAPACK's dpotrf: with uplo L...
int gemm(char transa, char transb, std::ptrdiff_t m, std::ptrdiff_t n, std::ptrdiff_t k, T alpha, const T *a, std::ptrdiff_t lda, const T *b, std::ptrdiff_t ldb, T beta, T *c, std::ptrdiff_t ldc)
C ← α op(A) op(B) + β C, column-major, as BLAS's dgemm: op(A) is m × k, op(B) k × n,...
double dot(std::ptrdiff_t n, const double *x, std::ptrdiff_t incx, const double *y, std::ptrdiff_t incy)
Σ x_i y_i over n elements incx and incy apart (from the last where negative), as BLAS's ddot; 0 where...
int getrf(std::ptrdiff_t m, std::ptrdiff_t n, T *a, std::ptrdiff_t lda, Index *ipiv)
P A = L U of the column-major m × n A in place, as LAPACK's dgetrf: the factors as dgetrf leaves them...
int trsm(char side, char uplo, char transa, char diag, std::ptrdiff_t m, std::ptrdiff_t n, double alpha, const double *a, std::ptrdiff_t lda, double *b, std::ptrdiff_t ldb)
B ← α op(A)⁻¹ B (side L) or α B op(A)⁻¹ (R), column-major, B m × n and the triangular A (uplo U or L,...
constexpr std::size_t syrk_block
The columns of syrk's smallest diagonal blocks, formed whole: their wasted half, at most 64 n k multi...
int orgqr(std::ptrdiff_t m, std::ptrdiff_t n, std::ptrdiff_t k, double *a, std::ptrdiff_t lda, const double *tau, double *work, std::ptrdiff_t lwork)
The first n columns of Q = H₀ H₁ … H_{k-1}, from the reflectors of a dgeqrf (geqrf's or any implement...
constexpr double lapack_precision
LAPACK's relative machine precision, ‘dlamch('E’): the unit roundoff2⁻⁵³, half of std::numeric_limits...
constexpr std::size_t trsm_block
The rows of trsm's blocks: within one, each column solved alone; past it, the matrix product kernel.
int syevd(char jobz, char uplo, std::ptrdiff_t n, double *a, std::ptrdiff_t lda, double *w, double *work, std::ptrdiff_t lwork, Index *iwork, std::ptrdiff_t liwork)
The eigenvalues, and with jobz V the eigenvectors, of a symmetric column-major A, as LAPACK's dsyevd:...
int gesdd(char jobz, std::ptrdiff_t m, std::ptrdiff_t n, double *a, std::ptrdiff_t lda, double *s, double *u, std::ptrdiff_t ldu, double *vt, std::ptrdiff_t ldvt, double *work, std::ptrdiff_t lwork, Index *iwork)
The singular value decomposition A = U Σ Vᵀ of a column-major m × n A, as LAPACK's dgesdd: the singul...
constexpr std::size_t syrk_kept_limit
The most elements of syrk's operand copy a thread keeps between calls (32 MiB).
int gemv(char trans, std::ptrdiff_t m, std::ptrdiff_t n, double alpha, const double *a, std::ptrdiff_t lda, const double *x, std::ptrdiff_t incx, double beta, double *y, std::ptrdiff_t incy)
y ← α op(A) x + β y, column-major A m × n, as BLAS's dgemv: the vectors incx and incy apart (from the...
int geev(char jobvl, char jobvr, std::ptrdiff_t n, const double *a, std::ptrdiff_t lda, double *wr, double *wi, double *vl, std::ptrdiff_t ldvl, double *vr, std::ptrdiff_t ldvr, double *work, std::ptrdiff_t lwork)
The eigenvalues, and with jobvl or jobvr V the left or right eigenvectors, of a column-major n × n A,...
int potrs(char uplo, std::ptrdiff_t n, std::ptrdiff_t nrhs, const T *a, std::ptrdiff_t lda, T *b, std::ptrdiff_t ldb)
A X = B from the Cholesky factor of a dpotrf (by any implementation), column-major,...
int getrs(char trans, std::ptrdiff_t n, std::ptrdiff_t nrhs, const T *a, std::ptrdiff_t lda, const Index *ipiv, T *b, std::ptrdiff_t ldb)
A X = B, Aᵀ X = B or Aᴴ X = B (trans N, T or C), column-major, from the factors and interchanges of a...
int syrk(char uplo, char trans, std::ptrdiff_t n, std::ptrdiff_t k, double alpha, const double *a, std::ptrdiff_t lda, double beta, double *c, std::ptrdiff_t ldc)
C ← α op(A) op(A)ᵀ + β C on the triangle uplo (U or L) of the column-major n × n C,...
int gesv(std::ptrdiff_t n, std::ptrdiff_t nrhs, T *a, std::ptrdiff_t lda, Index *ipiv, T *b, std::ptrdiff_t ldb)
A X = B, column-major, as LAPACK's dgesv: A (n × n, lda apart) is factorised as P A = L U by NumericA...
int gelsd(std::ptrdiff_t m, std::ptrdiff_t n, std::ptrdiff_t nrhs, const double *a, std::ptrdiff_t lda, double *b, std::ptrdiff_t ldb, double *s, double rcond, Index *rank, double *work, std::ptrdiff_t lwork, Index *iwork)
The minimum-norm least-squares solutions of A X ≈ B, column-major A m × n and B max(m,...
constexpr std::size_t gemv_rows
The rows (or, for Aᵀ x, the columns) each task of gemv takes, 64 so that an order of a few hundred sh...
int geqrf(std::ptrdiff_t m, std::ptrdiff_t n, double *a, std::ptrdiff_t lda, double *tau, double *work, std::ptrdiff_t lwork)
The QR factorisation A = Q R of a column-major m × n A, as LAPACK's dgeqrf: R overwrites the upper tr...
constexpr std::size_t gemv_weight
The weight of gemv's multiply-adds against a compute-bound product's: each reads an element of A once...
constexpr std::size_t gemm_packed_threshold
Below this many multiply-adds the plain loop (gemm_rows, by blocks of rows held in registers) runs in...
C complex_dot_product(const C *x, const C *y, std::size_t count)
Σ x[i] · y[i] of complex numbers: on the x86-64 units with AVX2 (the AVX-512 path too) their fused ch...
bool on_release(void(*step)()) noexcept
Registers step among the calling thread's thread_releases, while a place is left (a workspace past th...
void transpose_walk(std::size_t i0, std::size_t i1, std::size_t j0, std::size_t j1, std::size_t piece, const Copy ©)
transpose_pieces of side piece (transpose_piece's 8 or the whole tile), each side its own instance: a...
void form_q_into(buffer< T > &out, const std::vector< reflector_block< T > > &blocks, std::size_t rows, std::size_t first, std::size_t count, bool transposed)
The columns [first, first + count) of Q = H₀ H₁ …, the blocks applied (the last first) to those of th...
T conjugate_dot_product(const T *x, const T *y, std::size_t count)
Σ conj(x[i]) · y[i], in order, where no native vector holds the parts.
void gemm_at(std::size_t m, std::size_t n, std::size_t k, const T *at, std::size_t ldat, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm_layouts with A given as Aᵀ (at, k × m, ldat apart).
void add_product(double *y, const double &f, const double *x, std::size_t count)
y[i] += f · x[i] for double, as subtract_product.
void row_dots(const std::complex< double > *m, std::size_t ld, std::size_t rows, const std::complex< double > *x, std::size_t count, std::complex< double > *out)
row_dots for std::complex<double>, through complex_row_dots.
void subtract_products(double *y, const double *f, const double *x, std::size_t ldx, std::size_t terms, std::size_t count)
y[i] -= Σₜ f[t] · x[(t · ldx) + i] for double, the units chosen as subtract_product chooses them,...
T conj_of(const T &x)
The conjugate of a real (or general) scalar — the value itself.
constexpr std::size_t narrow_solve
The most right-hand sides an LU solve takes row by row rather than by blocks through the product kern...
void gemm_atbt(std::size_t m, std::size_t n, std::size_t k, const T *at, std::size_t ldat, const T *bt, std::size_t ldbt, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm_layouts with both operands given as their transposes.
void gemm_bt(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *bt, std::size_t ldbt, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm_layouts with B given as Bᵀ (bt, n × k, ldbt apart).
magnitude_t< T > pivot_magnitude(const T &x)
The magnitude partial pivoting compares: |x| of a real x.
reflector_block< T > gather_reflectors(const std::vector< buffer< T > > &reflectors, std::size_t k0, std::size_t k1, std::size_t rows, const std::vector< T > *taus=nullptr)
Gathers the reflectors reflectors[k0…k1-1] (each I − τ_k v vᴴ: τ_k from taus, else 2 for a unit v; an...
void parallel_for(std::size_t count, std::size_t work_per_index, const Body &body)
Runs body(i) for every i in [0, count).
std::size_t transpose_piece(std::size_t from_bytes, std::size_t to_bytes) noexcept
The side of the square pieces a transposing copy walks a tile by, from the bytes between rows of its ...
void gemm(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
C += A·B, every element of C summed in ascending order of the depth.
constexpr std::size_t qr_block
The reflectors of one panel of the blocked QR.
std::vector< reflector_block< T > > reflect_in_rows(buffer< T > &a, std::size_t rows, std::size_t cols, buffer< T > *r=nullptr, bool reflectors=true)
The blocks of reflectors of a Householder QR of the row-major a (rows × cols), factorised in place as...
constexpr std::size_t gemm_min_depth
Below this depth the plain loop runs too: each panel reads and writes its tiles of C once,...
std::vector< reflector_block< T > > reflect_in_columns(buffer< T > &columns, std::size_t rows, std::size_t cols, bool reflectors=true)
The blocks of reflectors of a Householder QR of the column-major columns (rows per column,...
T chunked_dot(const T *a, const T *b, std::size_t count, const Dot &dot)
Σ a[i] · b[i] for a plain dot product: chunks of dot_chunk elements in parallel, each by dot,...
std::vector< T, detail::default_init_allocator< T > > buffer
The storage of vector, matrix and tensor, namely a std::vector whose buffer<T>(n) and resize(n) leave...
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
tensor< T > diag(const tensor< T > &v, std::ptrdiff_t k)
The square matrix with the vector v on its k diagonal (above the main one for a positive k) and zeros...
least_squares_solution< T > least_squares(const matrix< T > &a, const matrix< T > &b, rank_cutoff cutoff={})
lstsq, with the residuals, the rank and the singular values numpy.linalg.lstsq returns beside the sol...
@ amx
The Apple AMX coprocessor, fused multiply-adds; the products it does not suit on the NEON path's tile...
@ divide_and_conquer
Golub–Kahan bidiagonalisation then divide and conquer, as LAPACK's dgesdd: the default.
tensor< T > diagonal(const tensor< T > &t, std::ptrdiff_t offset)
The offset diagonal of each matrix of t (its last two axes), as numpy.linalg.diagonal: above the main...
vector< double > singular_values(const matrix< T > &a)
The singular values of a, descending, without its singular vectors, as numpy.linalg....
vector< double > symmetric_eigenvalues(const matrix< T > &a)
The eigenvalues of the real symmetric or complex Hermitian a, ascending, without its eigenvectors,...
A minimal std::thread parallel-for for the hot kernels.
The operands of syrk_triangle, which are row-major P = op(A) and Q = op(A)ᵀ (one of them scaled by α)...
std::size_t ldp
P's row distance.
bool q_is_pt
Q is Pᵀ read from P's rows (numerical::detail::gemm_bt), q unused.
const double * p
P, n × depth.
bool p_is_qt
P is Qᵀ read from Q's columns (numerical::detail::gemm_at), p unused.
const double * q
Q, depth × n.
std::size_t ldc
C's column distance.
std::size_t ldq
Q's row distance.
bool lower
The lower triangle (else the upper).
double * c
C, column-major.
The reflectors of one panel of the blocked QR, I − V T Vᴴ as LAPACK's dlarft forms it.
A least-squares solution and what numpy.linalg.lstsq reports beside it (least_squares).
The pivoted LU factorisation of an m × n matrix (lu_factors): A = P L U.