NumericAL
A header-only C++20 lexer built on REAL
Loading...
Searching...
No Matches
factorizations.hpp
Go to the documentation of this file.
1
22#ifndef NUMERICAL_FACTORIZATIONS_HPP
23#define NUMERICAL_FACTORIZATIONS_HPP
24
25#include <algorithm>
26#include <array>
27#include <atomic>
28#include <bit>
29#include <cmath>
30#include <concepts>
31#include <complex> // so std::abs resolves for a complex scalar, regardless of include order
32#include <cstddef>
33#include <cstdint>
34#include <cstring>
35#include <functional>
36#include <limits>
37#include <memory>
38#include <numbers>
39#include <numeric>
40#include <optional>
41#include <stdexcept>
42#include <string>
43#include <tuple>
44#include <type_traits>
45#include <utility>
46#include <vector>
47
48#include "concepts.hpp"
49#include "matrix.hpp"
50#include "simd.hpp"
51#include "parallel.hpp"
52#include "vector.hpp"
53
54namespace numerical {
55
56 namespace detail {
62 template <typename T>
63 [[nodiscard]] T pivot_divisor(const T& pivot)
64 {
65 return pivot;
66 }
67
70 template <typename R>
72 {
73 std::complex<R> reciprocal;
74 std::complex<R> pivot;
75 bool divides;
76 };
77
81 template <typename R>
82 [[nodiscard]] complex_divisor<R> pivot_divisor(const std::complex<R>& pivot)
83 {
84 const R largest {std::max(std::abs(pivot.real()), std::abs(pivot.imag()))};
85 return {std::complex<R> {1} / pivot, pivot, largest < std::numeric_limits<R>::min()};
86 }
87
89 template <typename T>
90 [[nodiscard]] T over_pivot(const T& x,
91 const T& divisor)
92 {
93 return x / divisor;
94 }
95
98 template <typename R>
99 [[nodiscard]] std::complex<R> over_pivot(const std::complex<R>& x,
101 {
102 return divisor.divides ? x / divisor.pivot : operator_times(x, divisor.reciprocal);
103 }
104
106 template <typename T>
108 const T& diagonal)
109 {
110 return x / diagonal;
111 }
112
118 template <typename R>
119 [[nodiscard]] std::complex<R> over_real_diagonal(const std::complex<R>& x,
120 const std::complex<R>& diagonal)
121 {
122 return std::complex<R> {x.real() / diagonal.real(), x.imag() / diagonal.real()};
123 }
124
126 template <typename T>
128 const T& a,
129 const T& b)
130 {
131 sum -= a * b;
132 }
133
135 template <typename R>
136 void subtract_term(std::complex<R>& sum,
137 const std::complex<R>& a,
138 const std::complex<R>& b)
139 {
140 sum -= operator_times(a, b);
141 }
142
145 inline constexpr std::size_t lu_block {64};
146
147#ifdef NUMERICAL_X86_COMPILED
152 inline constexpr std::size_t lu_small_order {384};
153#else
157 inline constexpr std::size_t lu_small_order {256};
158#endif
159
161 inline constexpr std::size_t lu_small_block {32};
162
168 inline constexpr std::size_t lu_wide_order {3072};
169
171 inline constexpr std::size_t lu_wide_block {128};
172
174 [[nodiscard]] constexpr std::size_t lu_block_for(std::size_t n) noexcept
175 {
176 static_assert(std::has_single_bit(lu_small_block) && std::has_single_bit(lu_block), "a panel is a mask away");
177 static_assert(std::has_single_bit(lu_wide_block), "a panel is a mask away");
178 if (n >= lu_wide_order) {
179 return lu_wide_block;
180 }
181 return n <= lu_small_order ? lu_small_block : lu_block;
182 }
183
187 inline constexpr std::size_t lu_lookahead_columns {1024};
188
192 inline constexpr std::size_t lu_fused_columns {512};
193
197#ifdef NUMERICAL_X86_COMPILED
198 inline constexpr std::size_t lu_update_speedup {gemm_kernel_speedup / 4};
199#else
200 inline constexpr std::size_t lu_update_speedup {gemm_kernel_speedup};
201#endif
202
204 inline constexpr std::size_t narrow_solve {16};
205
206#ifdef NUMERICAL_X86_COMPILED
210 inline constexpr std::size_t slice_columns {48};
211#else
215 inline constexpr std::size_t slice_columns {64};
216#endif
217
218#ifdef NUMERICAL_X86_COMPILED
222 inline constexpr std::size_t slice_solve_order {384};
223#endif
224
225
227 inline constexpr std::size_t solve_leaf {16};
228
230 inline constexpr std::size_t inverse_band {16};
231
233 inline constexpr std::size_t svd_band {32};
234
240 [[nodiscard]] inline double power_of_two_scale(double largest) noexcept
241 {
242 return largest == 0.0 ? 1.0 : std::ldexp(1.0, std::ilogb(largest));
243 }
244
246 inline constexpr std::size_t zeroed_rows {32};
247
248#ifdef NUMERICAL_X86_COMPILED
252 inline constexpr std::size_t panel_copy_rows {8};
253
258 inline constexpr std::size_t panel_tile_rows {256};
259#endif
260
262 inline constexpr std::size_t cholesky_block {64};
263
266 inline constexpr std::size_t cholesky_diagonal_rows {4};
267
270 inline constexpr std::size_t cholesky_together_width {16};
271
275#ifdef NUMERICAL_X86_COMPILED
276 inline constexpr std::size_t cholesky_one_panel {40};
277#else
278 inline constexpr std::size_t cholesky_one_panel {48};
279#endif
280 static_assert(cholesky_one_panel <= cholesky_block, "one panel of cholesky_block columns holds the whole matrix");
281
285 [[nodiscard]] constexpr std::size_t cholesky_block_for(std::size_t n) noexcept
286 {
287 if (n < cholesky_one_panel) {
288 return cholesky_block;
289 }
290 if (n <= 128) {
291 return cholesky_block / 4;
292 }
293 return n <= 256 ? cholesky_block / 2 : cholesky_block;
294 }
295
297 inline constexpr std::size_t cholesky_rows {64};
298
302 inline constexpr std::size_t cholesky_update_columns {64};
303
307 inline constexpr std::size_t cholesky_update_rows {256};
308
309
310 // conj_of / real_part are written as overloads (not `if constexpr`) so each
311 // is a single, fully-covered expression: a complex scalar selects the
312 // std::complex overload, any other scalar the primary template.
313
315 template <typename T>
316 [[nodiscard]] T conj_of(const T& x)
317 {
318 return x;
319 }
320
322 template <typename U>
323 [[nodiscard]] std::complex<U> conj_of(const std::complex<U>& x)
324 {
325 return std::conj(x);
326 }
327
329 template <typename T>
330 using magnitude_t = decltype(std::abs(std::declval<const T&>()));
331
333 template <typename T>
335 {
336 return std::abs(x);
337 }
338
341 template <typename U>
342 [[nodiscard]] U pivot_magnitude(const std::complex<U>& x)
343 {
344 return std::abs(x.real()) + std::abs(x.imag());
345 }
346
349 template <typename T>
350 using carried_t = std::conditional_t<std::is_same_v<magnitude_t<T>, long double>, long double, double>;
351
354 template <typename T>
356 {
357 return static_cast<T>(static_cast<magnitude_t<T>>(x));
358 }
359
361 template <typename T>
363 {
364 return static_cast<carried_t<T>>(x);
365 }
366
368 template <typename U>
369 [[nodiscard]] carried_t<U> real_part(const std::complex<U>& x)
370 {
371 return static_cast<carried_t<U>>(x.real());
372 }
373
376 template <typename T>
378 std::conditional_t<std::is_same_v<T, float> || std::is_same_v<T, std::complex<float>>, float, carried_t<T>>;
379
382 template <typename T>
383 inline constexpr carried_t<T> rescue_below {std::is_same_v<value_range_t<T>, float> ? 0x1p-60 : 0x1p-500};
384
389 template <typename T>
391 {
392 using real = carried_t<T>;
393 constexpr real least {static_cast<real>(std::numeric_limits<value_range_t<T>>::min())};
394 return largest < rescue_below<T> ? std::ldexp(real {1}, -std::ilogb(std::max(largest, least))) : real {1};
395 }
396
398 template <std::floating_point R>
400 int e)
401 {
402 return std::ldexp(x, e);
403 }
404
406 template <std::floating_point R>
408 {
409 return std::abs(x);
410 }
411
413 template <std::floating_point U>
414 [[nodiscard]] U part_size(const std::complex<U>& x)
415 {
416 return std::max(std::abs(x.real()), std::abs(x.imag()));
417 }
418
420 template <std::floating_point U>
421 [[nodiscard]] std::complex<U> times_power_of_two(const std::complex<U>& x,
422 int e)
423 {
424 return {std::ldexp(x.real(), e), std::ldexp(x.imag(), e)};
425 }
426
433 template <typename T,
434 typename Factor>
436 std::size_t count,
437 const Factor& factor)
438 {
439 T value {start};
440 int exponent {0};
441 for (std::size_t i {0}; i < count; ++i) {
442 // the factor brought near 1 first, exactly: a subnormal pivot times a value near 1 would round on the
443 // subnormals' coarse grid
444 int own {0};
445 const auto f {factor(i)};
446 static_cast<void>(std::frexp(part_size(f), &own));
447 value *= times_power_of_two(f, -own);
448 exponent += own;
449 int part {0};
450 static_cast<void>(std::frexp(part_size(value), &part)); // 0 for a zero product
451 // the value scaled itself, exactly: 2⁻ᵖᵃʳᵗ of a subnormal product (a pivot of 10⁻³¹⁰) is past the doubles
452 value = times_power_of_two(value, -part);
453 exponent += part;
454 }
455 return times_power_of_two(value, exponent);
456 }
457
463 template <typename T>
464 [[nodiscard]] T phase_of(const T& x)
465 {
466 const T scaled {x * detail::from_carried<T>(rescue_factor<T>(static_cast<carried_t<T>>(std::abs(x))))};
467 const carried_t<T> magnitude {static_cast<carried_t<T>>(std::abs(scaled))};
468 return magnitude == 0 ? T {1} : scaled / detail::from_carried<T>(magnitude);
469 }
470
477 template <typename T>
479 {
480 carried_t<T> largest {0};
481 for (const T& element : x) {
482 largest = std::max(largest, static_cast<carried_t<T>>(std::abs(element)));
483 }
484 const carried_t<T> factor {rescue_factor<T>(largest)};
485 for (T& element : x) {
487 }
488 return factor;
489 }
490
493 template <typename T>
495 std::size_t count)
496 {
497 using real = carried_t<T>;
498 real largest {0};
499 for (std::size_t i {0}; i < count; ++i) {
500 largest = std::max(largest, static_cast<real>(std::abs(x[i])));
501 }
502 if (largest == 0) { // the entries are finite: require_finite runs where every caller starts
503 return largest;
504 }
505 real sum {0};
506 for (std::size_t i {0}; i < count; ++i) {
507 const real ratio {static_cast<real>(std::abs(x[i])) / largest};
508 sum += ratio * ratio;
509 }
510 return largest * std::sqrt(sum);
511 }
512
515 inline constexpr double plain_squares_low {0x1p-900};
516
518 inline constexpr double plain_squares_high {0x1p900};
519
522 template <typename T>
524 const T* x,
525 std::size_t count)
526 {
528 return std::sqrt(plain);
529 }
530 return norm_at_scale(x, count);
531 }
532
534 template <typename T>
536 std::size_t count)
537 {
538 return norm_at_scale(x, count);
539 }
540
542 [[nodiscard]] inline double scaled_norm(const double* x,
543 std::size_t count)
544 {
545 return plain_or_at_scale(dot_product(x, x, count), x, count);
546 }
547
550 [[nodiscard]] inline double scaled_norm(const std::complex<double>* x,
551 std::size_t count)
552 {
553 const auto* const parts {reinterpret_cast<const double*>(x)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
554 return plain_or_at_scale(dot_product(parts, parts, 2 * count), x, count);
555 }
556
558 template <typename T>
560 {
561 return scaled_norm(x.data(), x.size());
562 }
563
571 [[nodiscard]] inline double rotation_length(double a,
572 double b) noexcept
573 {
574 constexpr double low {0x1p-500};
575 constexpr double high {0x1p500};
576 const double larger {std::max(std::abs(a), std::abs(b))};
577 if (larger > low && larger < high) {
578 return std::sqrt((a * a) + (b * b));
579 }
580 return std::hypot(a, b);
581 }
582 } // namespace detail
583
585 class singular_matrix_error : public detail::tagged<std::runtime_error>
586 {
587 public:
588
590 explicit singular_matrix_error(const std::string& message)
591 : tagged(message)
592 {}
593 };
594
597 class non_finite_error : public detail::tagged<std::domain_error>
598 {
599 public:
600
602 explicit non_finite_error(const std::string& message)
603 : tagged(message)
604 {}
605 };
606
607 namespace detail {
608
610 template <typename T>
611 [[nodiscard]] bool is_finite_entry(const T& x)
612 {
613 return std::isfinite(x);
614 }
615
617 template <typename U>
618 [[nodiscard]] bool is_finite_entry(const std::complex<U>& x)
619 {
620 return std::isfinite(x.real()) && std::isfinite(x.imag());
621 }
622
627 template <typename T>
629 const char* what)
630 {
631 for (std::size_t i {0}; i < a.rows(); ++i) {
632 for (std::size_t j {0}; j < a.cols(); ++j) {
633 if (!is_finite_entry(a(i, j))) {
634 throw non_finite_error(std::string {what} + ": the matrix holds an infinite or NaN entry");
635 }
636 }
637 }
638 }
639
645 template <typename T>
647 {
648 buffer<T> elements(a.rows() * a.cols());
649 parallel_copy(std::to_address(a.begin()), elements.size(), elements.data());
650 return matrix<T> {a.rows(), a.cols(), std::move(elements)};
651 }
652
654 inline constexpr std::size_t finite_chunk {std::size_t {1} << 15U};
655
661 [[nodiscard]] inline bool all_finite_shared(const double* x,
662 std::size_t count)
663 {
664 if (count < 2 * finite_chunk) {
665 return all_finite(x, count);
666 }
667 std::atomic<bool> finite {true};
668 parallel_for((count + finite_chunk - 1) / finite_chunk, memory_element_weight * finite_chunk / 2, [x, count, &finite](std::size_t chunk) {
669 const std::size_t first {chunk * finite_chunk};
670 if (!all_finite(x + first, std::min(finite_chunk, count - first))) {
671 finite.store(false, std::memory_order_relaxed);
672 }
673 });
674 return finite.load(std::memory_order_relaxed);
675 }
676
678 inline void require_finite(const matrix<double>& a,
679 const char* what)
680 {
681 if (!all_finite_shared(std::to_address(a.begin()), a.rows() * a.cols())) {
682 throw non_finite_error(std::string {what} + ": the matrix holds an infinite or NaN entry");
683 }
684 }
685
687 inline void require_finite(const matrix<std::complex<double>>& a,
688 const char* what)
689 {
690 const auto* const parts {reinterpret_cast<const double*>(std::to_address(a.begin()))}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
691 if (!all_finite_shared(parts, 2 * a.rows() * a.cols())) {
692 throw non_finite_error(std::string {what} + ": the matrix holds an infinite or NaN entry");
693 }
694 }
695
698 template <typename T>
700 {
701 for (std::size_t i {0}; i < a.rows(); ++i) {
702 bool finite {std::isfinite(real_part(a(i, i)))};
703 for (std::size_t j {0}; j < i; ++j) {
704 finite = finite && is_finite_entry(a(i, j));
705 }
706 if (!finite) {
707 return false;
708 }
709 }
710 return true;
711 }
712
718 template <typename T>
720 const char* what)
721 {
722 if (!lower_is_finite(a)) {
723 throw non_finite_error(std::string {what} + ": the matrix holds an infinite or NaN entry");
724 }
725 }
726
733 const char* what)
734 {
735 if (!all_finite_shared(std::to_address(a.begin()), a.rows() * a.cols()) && !lower_is_finite(a)) {
736 throw non_finite_error(std::string {what} + ": the matrix holds an infinite or NaN entry");
737 }
738 }
739
741 NUMERICAL_OUT_OF_LINE inline void require_finite_lower(const matrix<std::complex<double>>& a,
742 const char* what)
743 {
744 const auto* const parts {reinterpret_cast<const double*>(std::to_address(a.begin()))}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
745 if (!all_finite_shared(parts, 2 * a.rows() * a.cols()) && !lower_is_finite(a)) {
746 throw non_finite_error(std::string {what} + ": the matrix holds an infinite or NaN entry");
747 }
748 }
749
755 template <typename T>
757 const T* columns,
758 std::size_t height,
759 const T* l,
760 std::size_t terms)
761 {
762 for (std::size_t t {0}; t < terms; ++t) {
763 subtract_product(column, columns + (t * height), conj_of(l[t]), height);
764 }
765 }
766
770 inline void subtract_panel_columns(double* column,
771 const double* columns,
772 std::size_t height,
773 const double* l,
774 std::size_t terms)
775 {
776 subtract_products(column, l, columns, height, terms, height);
777 }
778 } // namespace detail
779
780 namespace detail {
781
784 template <typename T>
785 inline constexpr bool real_rows {std::is_same_v<T, double> || std::is_same_v<T, float>};
786 } // namespace detail
787
797 template <field T>
799 {
800 public:
801
807 explicit lu_decomposition(const matrix<T>& a)
808 : lu_decomposition(detail::parallel_copy_of(a))
809 {}
810
817 : n_(a.rows())
818 {
819 if (a.cols() != n_) {
820 throw dimension_error("LU decomposition requires a square matrix");
821 }
822 detail::require_finite(a, "LU decomposition");
823 lu_ = std::move(a).elements();
824 pivots_.resize(n_);
825 for (std::size_t i {0}; i < n_; ++i) {
826 pivots_[i] = i;
827 }
828 factorize();
829 }
830
838 [[nodiscard]] vector<T> solve(const vector<T>& b) const
839 {
840 if (b.size() != n_) {
841 throw dimension_error("solve: right-hand side size differs from the matrix");
842 }
843 if (singular_) {
844 throw singular_matrix_error("solve: matrix is singular");
845 }
846 buffer<T> y(n_, T {});
847 for (std::size_t i {0}; i < n_; ++i) { // forward substitution (unit lower L)
848 T sum {b[pivots_[i]]};
849 for (std::size_t j {0}; j < i; ++j) {
850 detail::subtract_term(sum, lu_[(i * n_) + j], y[j]);
851 }
852 y[i] = sum;
853 }
854 for (std::size_t i {n_}; i-- > 0;) { // back substitution (upper U)
855 T sum {y[i]};
856 for (std::size_t j {i + 1}; j < n_; ++j) {
857 detail::subtract_term(sum, lu_[(i * n_) + j], y[j]);
858 }
860 }
861 return vector<T> {std::move(y)};
862 }
863
866 {
867 // by powers of 2 apart (detail::scaled_product): a partial product past the doubles (pivots of 10¹⁵⁰ then 10⁻¹⁵⁰)
868 // gave ±∞ or NaN for a determinant of 10⁻⁸⁵; a zero pivot gives a zero, never 0·∞
869 return detail::scaled_product(detail::from_carried<T>(sign_), n_, [this](std::size_t i) { return lu_[(i * n_) + i]; });
870 }
871
879 [[nodiscard]] std::pair<T, double> log_determinant() const
880 {
881 if (singular_) {
882 return {T {0}, -std::numeric_limits<double>::infinity()};
883 }
884 T sign {static_cast<T>(sign_)};
885 double logarithm {0.0};
886 for (std::size_t i {0}; i < n_; ++i) {
887 const T pivot {lu_[(i * n_) + i]};
888 sign *= detail::phase_of(pivot);
889 logarithm += std::log(static_cast<double>(std::abs(pivot)));
890 }
891 return {sign, logarithm};
892 }
893
903 {
904 return inverse(buffer<T> {});
905 }
906
915 {
916 buffer<T> rows {pivoted_inverse(std::move(storage)).elements()};
917 const std::size_t n {n_};
918 T* const at {rows.data()};
919 const std::size_t* const to {pivots_.data()};
920 // Bounds captured by value: by reference, the stores might have changed them, and each would be read again.
922 buffer<T> held(n); // column to[i] of A⁻¹ is column i of U⁻¹ L⁻¹
923 for (std::size_t r {band * detail::inverse_band}; r < std::min(n, (band + 1) * detail::inverse_band); ++r) {
924 std::copy_n(at + (r * n), n, held.data());
925 for (std::size_t i {0}; i < n; ++i) {
926 at[(r * n) + to[i]] = held[i];
927 }
928 }
929 });
930 return matrix<T> {n, n, std::move(rows)};
931 }
932
942 {
943 if (singular_) {
944 throw singular_matrix_error("inverse: matrix is singular");
945 }
946 const std::size_t n {n_};
947 storage.resize(n * n);
948 T* const rows {storage.data()};
950 for (std::size_t r {band * detail::inverse_band}; r < std::min(n, (band + 1) * detail::inverse_band); ++r) {
951 std::fill_n(rows + (r * n), n, T {});
952 rows[(r * n) + r] = T {1};
953 }
954 });
955 solve_rows(storage, n, true);
956 return matrix<T> {n, n, std::move(storage)};
957 }
958
970 [[nodiscard]] matrix<T> solve(const matrix<T>& b) const
971 {
972 if (b.rows() != n_) {
973 throw dimension_error("solve: right-hand side row count differs from the matrix");
974 }
975 if (singular_) {
976 throw singular_matrix_error("solve: matrix is singular");
977 }
978 const std::size_t cols {b.cols()};
979 buffer<T> rows(n_ * cols); // every element written below
980 for (std::size_t i {0}; i < n_; ++i) {
981 for (std::size_t c {0}; c < cols; ++c) {
982 rows[(i * cols) + c] = b(pivots_[i], c);
983 }
984 }
985 solve_rows(rows, cols);
986 return matrix<T> {n_, cols, std::move(rows)};
987 }
988
998 {
999 if (b.rows() != n_) {
1000 throw dimension_error("solve: right-hand side row count differs from the matrix");
1001 }
1002 if (singular_) {
1003 throw singular_matrix_error("solve: matrix is singular");
1004 }
1005 const std::size_t cols {b.cols()};
1006 buffer<T> rows {std::move(b).elements()};
1007 // Row i of P B is row pivots_[i] of B: each cycle i → pivots_[i] → … walked once, its first row held aside.
1008 std::vector<bool> placed(n_, false);
1009 buffer<T> held(cols);
1010 for (std::size_t start {0}; start < n_; ++start) {
1011 if (placed[start] || pivots_[start] == start) {
1012 continue;
1013 }
1014 std::copy_n(rows.data() + (start * cols), cols, held.data());
1015 std::size_t i {start};
1016 for (std::size_t from {pivots_[i]}; from != start; i = from, from = pivots_[i]) {
1017 std::copy_n(rows.data() + (from * cols), cols, rows.data() + (i * cols));
1018 placed[i] = true;
1019 }
1020 std::copy_n(held.data(), cols, rows.data() + (i * cols));
1021 placed[i] = true;
1022 }
1023 solve_rows(rows, cols);
1024 return matrix<T> {n_, cols, std::move(rows)};
1025 }
1026
1030 {
1031 return matrix<T> {n_, n_, lu_};
1032 }
1033
1036 {
1037 return matrix<T> {n_, n_, std::move(lu_)};
1038 }
1039
1041 [[nodiscard]] const std::vector<std::size_t>& pivots() const noexcept
1042 {
1043 return pivots_;
1044 }
1045
1048 {
1049 return singular_;
1050 }
1051
1061 [[nodiscard]] double reciprocal_condition(double one_norm) const;
1062
1063 private:
1064
1066
1076 std::size_t cols,
1077 bool lower = false) const
1078 {
1079 if constexpr (detail::real_rows<T>) {
1080 // The rows beat the blocked passes on the x86-64 units at any order (the portable path's products there fuse
1081 // nothing, the rows would), and at any order elsewhere, by groups (arm64, four columns: order 256 0.65 times the
1082 // blocked passes' time under Clang, 0.45 under GCC; order 1024 on every thread 0.91 and 0.62); in float, where
1083 // the blocked passes copy each block of the factor negated for a product of four columns, order 3000 on one
1084 // arm64 core 5.9 → 2.1 ms.
1085#ifdef NUMERICAL_X86_COMPILED
1086 const bool narrow {!lower && !y.empty() && cols <= detail::narrow_solve && detail::gemm_path_current() != gemm_path::portable};
1087#else
1088 const bool narrow {!lower && !y.empty() && cols <= detail::narrow_solve};
1089#endif
1090 if (narrow) {
1091 solve_narrow(y, cols);
1092 return;
1093 }
1094 }
1095 const std::size_t slices {column_slices(cols)};
1096 if (slices > 1) { // independent right-hand sides: each thread its own columns, all the blocks, no join per block
1097 const std::size_t slice {(((cols + slices - 1) / slices) + 7) / 8 * 8};
1098 const auto start {[this, slice, slices, cols, lower](std::size_t s) {
1099 return lower ? lower_slice_start(s, slices, cols) : std::min(cols, s * slice);
1100 }};
1101 detail::parallel_for(lower ? slices : (cols + slice - 1) / slice, n_ * n_ * slice / detail::gemm_kernel_speedup, [&](std::size_t s) {
1102 solve_columns(y, cols, start(s), start(s + 1), lower);
1103 });
1104 return;
1105 }
1106 solve_columns(y, cols, 0, cols, lower);
1107 }
1108
1115 [[nodiscard]] std::size_t column_slices(std::size_t cols) const
1116 {
1117#ifdef NUMERICAL_X86_COMPILED
1119 return 1;
1120 }
1121#endif
1122 if (cols < 2 * detail::slice_columns) {
1123 return 1;
1124 }
1125 return std::min(max_threads(), cols / detail::slice_columns);
1126 }
1127
1134 [[nodiscard]] std::size_t lower_slice_start(std::size_t s,
1135 std::size_t slices,
1136 std::size_t cols) const
1137 {
1138 const auto n {static_cast<double>(n_)};
1139 const auto weight {[n](double c) { return (n * n * c) + (((n * n * n) - ((n - c) * (n - c) * (n - c))) / 3.0); }};
1140 const double target {weight(static_cast<double>(cols)) * static_cast<double>(s) / static_cast<double>(slices)};
1141 std::size_t low {0};
1142 std::size_t high {cols};
1143 while (low < high) { // the fewest columns that weigh the target
1144 const std::size_t middle {low + ((high - low) / 2)};
1145 if (weight(static_cast<double>(middle)) < target) {
1146 low = middle + 1;
1147 }
1148 else {
1149 high = middle;
1150 }
1151 }
1152 return std::min(cols, (low + 7) / 8 * 8);
1153 }
1154
1160 std::size_t cols,
1161 std::size_t c0,
1162 std::size_t c1,
1163 bool lower) const
1164 {
1165 constexpr std::size_t width {64};
1166 const std::size_t chunks {(c1 - c0 + width - 1) / width};
1167 for (std::size_t k0 {0}; k0 < n_; k0 += block_) { // forward: unit-lower L
1168 const std::size_t end {std::min(k0 + block_, n_)};
1169 const std::size_t reach {lower ? std::min(end, c1) : c1}; // the block's rows' columns that can be nonzero
1170 if (reach <= c0) {
1171 continue; // the slice's columns are zero in these rows and every row past them
1172 }
1173 detail::parallel_for((reach - c0 + width - 1) / width, (end - k0) * (end - k0) * width / 2, [&](std::size_t chunk) {
1174 const std::size_t first {c0 + (chunk * width)};
1175 const std::size_t last {std::min(first + width, reach)};
1176 for (std::size_t i {k0 + 1}; i < end; ++i) {
1177 detail::subtract_products(y.data() + (i * cols) + first, lu_.data() + (i * n_) + k0, y.data() + (k0 * cols) + first, cols, i - k0, last - first);
1178 }
1179 });
1180 subtract_block(y, cols, end, n_, k0, end, c0, reach);
1181 }
1182 for (std::size_t k1 {n_}; k1 > 0;) { // backward: upper U, the blocks of the forward pass, last first
1183 const std::size_t k0 {panel_of(k1 - 1)};
1184 detail::parallel_for(chunks, (k1 - k0) * (k1 - k0) * width / 2, [&](std::size_t chunk) {
1185 const std::size_t first {c0 + (chunk * width)};
1186 const std::size_t last {std::min(first + width, c1)};
1187 for (std::size_t i {k1}; i-- > k0;) {
1188 detail::subtract_products(y.data() + (i * cols) + first, lu_.data() + (i * n_) + i + 1, y.data() + ((i + 1) * cols) + first, cols, k1 - (i + 1), last - first);
1189 const auto divisor {detail::pivot_divisor(lu_[(i * n_) + i])};
1190 for (std::size_t c {first}; c < last; ++c) {
1191 y[(i * cols) + c] = detail::over_pivot(y[(i * cols) + c], divisor);
1192 }
1193 }
1194 });
1195 subtract_block(y, cols, 0, k0, k0, k1, c0, c1);
1196 k1 = k0;
1197 }
1198 }
1199
1210 std::size_t cols) const
1211 {
1212 // Forward, by groups of rows: their terms before the group are independent of one another, and go together
1213 // (\ref detail::subtract_rows), the group's own triangle after; each element's terms in ascending order still.
1214 constexpr std::size_t group {detail::product_rows};
1215 static_assert(detail::lu_small_block % group == 0 && detail::lu_block % group == 0, "a panel is a whole number of groups");
1216 static_assert(detail::lu_wide_block % group == 0, "a panel is a whole number of groups");
1217 std::size_t i0 {0};
1218 for (; i0 + group <= n_; i0 += group) {
1219 if (i0 > 0) {
1220 detail::subtract_rows(y.data() + (i0 * cols), cols, lu_.data() + (i0 * n_), n_, y.data(), cols, i0, cols);
1221 }
1222 for (std::size_t i {i0 + 1}; i < i0 + group; ++i) {
1223 detail::subtract_products(y.data() + (i * cols), lu_.data() + (i * n_) + i0, y.data() + (i0 * cols), cols, i - i0, cols);
1224 }
1225 }
1226 for (std::size_t i {std::max<std::size_t>(i0, 1)}; i < n_; ++i) {
1227 detail::subtract_products(y.data() + (i * cols), lu_.data() + (i * n_), y.data(), cols, i, cols);
1228 }
1229 // Backward, each row's terms block by block from the last down to its own, ascending within each: a group's
1230 // terms in the blocks past its own are independent of one another and go together (\ref detail::subtract_rows),
1231 // in that same order; its own block's after, row by row (\p finish). The rows past the last whole group lie in
1232 // the last block, a panel being a whole number of groups: they have their own block's terms alone.
1233 const std::size_t last {panel_of(n_ - 1)};
1234 const auto finish {[this, &y, cols](std::size_t i) {
1235 const std::size_t to {std::min(panel_of(i) + block_, n_)};
1236 if (i + 1 < to) {
1237 detail::subtract_products(y.data() + (i * cols), lu_.data() + (i * n_) + i + 1, y.data() + ((i + 1) * cols), cols, to - (i + 1), cols);
1238 }
1239 const auto divisor {detail::pivot_divisor(lu_[(i * n_) + i])};
1240 for (std::size_t c {0}; c < cols; ++c) {
1241 y[(i * cols) + c] = detail::over_pivot(y[(i * cols) + c], divisor);
1242 }
1243 }};
1244 std::size_t i1 {n_};
1245 for (; i1 % group != 0; --i1) {
1246 finish(i1 - 1);
1247 }
1248 for (; i1 > 0; i1 -= group) {
1249 const std::size_t top {i1 - group};
1250 const std::size_t own {panel_of(top)};
1251 for (std::size_t b0 {last}; b0 > own; b0 -= block_) {
1252 detail::subtract_rows(y.data() + (top * cols), cols, lu_.data() + (top * n_) + b0, n_, y.data() + (b0 * cols), cols, std::min(block_, n_ - b0), cols);
1253 }
1254 for (std::size_t i {i1}; i-- > top;) {
1255 finish(i);
1256 }
1257 }
1258 }
1259
1261 [[nodiscard]] std::size_t panel_of(std::size_t i) const noexcept
1262 {
1263 return i & ~(block_ - 1);
1264 }
1265
1269 std::size_t cols,
1270 std::size_t r0,
1271 std::size_t r1,
1272 std::size_t c0,
1273 std::size_t c1,
1274 std::size_t first,
1275 std::size_t last) const
1276 {
1277 if (r0 < r1) {
1278 const buffer<T> minus {negated_block(r0, r1, c0, c1)};
1279 detail::gemm(r1 - r0, last - first, c1 - c0, minus.data(), c1 - c0, y.data() + (c0 * cols) + first, cols,
1280 y.data() + (r0 * cols) + first, cols);
1281 }
1282 }
1283
1286 std::size_t r0,
1287 std::size_t r1,
1288 std::size_t c0,
1289 std::size_t c1) const
1290 {
1291 out.resize((r1 - r0) * (c1 - c0));
1292 for (std::size_t r {r0}; r < r1; ++r) {
1293 for (std::size_t c {c0}; c < c1; ++c) {
1294 out[((r - r0) * (c1 - c0)) + (c - c0)] = -lu_[(r * n_) + c];
1295 }
1296 }
1297 }
1298
1301 std::size_t r1,
1302 std::size_t c0,
1303 std::size_t c1) const
1304 {
1305 buffer<T> out((r1 - r0) * (c1 - c0), T {});
1306 for (std::size_t r {r0}; r < r1; ++r) {
1307 for (std::size_t c {c0}; c < c1; ++c) {
1308 out[((r - r0) * (c1 - c0)) + (c - c0)] = -lu_[(r * n_) + c];
1309 }
1310 }
1311 return out;
1312 }
1313
1314 // Right-looking, by panels of block_ columns, as LAPACK's dgetrf: each panel is factorised
1315 // with partial pivoting (whole rows swapped), its rows of U to the right are solved against the panel's
1316 // unit-lower L, and the trailing block takes the product of the two through detail::gemm, where the
1317 // work of a large factorisation is. Each element still receives its updates in ascending order of the
1318 // pivot, so without multiply-add fusion the factors are the unblocked algorithm's to the bit; an order
1319 // up to one panel is that algorithm.
1320 //
1321 // With a look-ahead of one panel: the trailing update reaches the next panel's columns first, then the
1322 // next panel is factorised on the calling thread while the workers update the columns right of it (the
1323 // panel, a column at a time, would otherwise hold every other thread idle). Its row swaps reach those
1324 // columns once both are done: an update reads its rows of L from a copy and writes row by row, so a swap
1325 // after it is the swap before it, and the factors stay the ones above, to the bit.
1326 //
1327 // A pivot's swap reaches the columns left of its panel only at the end (\ref swap_left_columns): nothing reads
1328 // them before, and on the calling thread each swap held up the panel the other threads wait for.
1330 {
1331 // Every panel in the one column-major storage, of the first (and tallest) one's size: a fresh one each panel was
1332 // mapped anew and faulted in page by page, on the critical path (order 1000 on six AVX2 cores: 24 µs of a panel's
1333 // 470, and its copy in 97 µs instead of about 40).
1334 buffer<T> panel(n_ * std::min(block_, n_));
1335 const std::size_t end {std::min(block_, n_)};
1336 factorize_panel(panel, 0, end);
1337 if (end < n_) {
1339 }
1341 }
1342
1349 std::size_t first_end)
1350 {
1351 // −L₂₁ and its packing for the product's tile, in storage kept from panel to panel: packed once, it serves every
1352 // block of the trailing update, where each block's product packed it again (6.9 % of the time on six AVX2 cores
1353 // at order 1000).
1356 std::size_t end {first_end};
1357 for (std::size_t k0 {0}; end < n_;) {
1358 const std::size_t next {std::min(end + block_, n_)};
1359 negated_block_into(minus_l, end, n_, k0, end);
1360 const detail::packed_left<T> left {detail::pack_left(n_ - end, block_, end - k0, minus_l.data(), end - k0, packing)};
1361 const std::size_t right {n_ - next};
1362 if (right > detail::lu_lookahead_columns) { // one product of the whole trailing block, then the panel
1363 solve_panel_rows(k0, end);
1364 update_columns(left, k0, end, next);
1365 update_columns(left, k0, next, n_);
1366 factorize_panel(panel, end, next);
1367 k0 = end;
1368 end = next;
1369 continue;
1370 }
1371 // About two blocks of columns for each thread the work pays for: a product too small to share ran one block for
1372 // each thread the machine has (order 256 on eight arm64 threads: 329 → 311 µs); two, not one, as on x86-64 a
1373 // block of half the columns stays in its cache, and the balance beat one (six AVX2 cores, order 1000: 12 %).
1374 const std::size_t work {(n_ - end) * right * (end - k0) / detail::lu_update_speedup};
1375 const std::size_t helpers {std::clamp(work / detail::work_per_thread, std::size_t {1}, max_threads())};
1376 const std::size_t width {std::max(block_, (right + (2 * helpers) - 1) / (2 * helpers))};
1377 const std::size_t blocks {(right + width - 1) / width};
1378 swap_limit_ = next;
1379 // The next panel's columns first, their update shared among the threads; then, where the blocks right of it
1380 // run beside the panel, each block solves its rows of U₁₂ and takes its update on one thread, one parallel loop
1381 // a panel where the rows of U₁₂ were solved in one of their own (order 531 on ten AVX-512 cores: 2.24 → 2.09 ms):
1382 // the columns of a block depend on no other's, and the rows they read from the panel's L are above every swap
1383 // the next panel makes. On one thread, all of U₁₂ first (in that order, orders 200 and 300 on one AVX-512 core:
1384 // 4 to 5 %).
1385 const std::size_t per_block {((n_ - end) * width * (end - k0) / detail::lu_update_speedup) + ((end - k0) * (end - k0) * width / 2)};
1386 const bool fused {right <= detail::lu_fused_columns && detail::parallel_threads(blocks, per_block) >= 2};
1387 if (fused) {
1388 solve_columns(k0, end, end, next);
1389 }
1390 else {
1391 solve_panel_rows(k0, end);
1392 }
1393 update_columns(left, k0, end, next);
1395 [&](std::size_t block) {
1396 const std::size_t first {next + (block * width)};
1397 const std::size_t last {std::min(first + width, n_)};
1398 if (fused) {
1399 solve_columns(k0, end, first, last);
1400 }
1401 update_columns(left, k0, first, last);
1402 },
1403 [&] { factorize_panel(panel, end, next); });
1404 swap_limit_ = n_;
1405 for (const auto& [a, b] : deferred_swaps_) {
1406 for (std::size_t c {next}; c < n_; ++c) {
1407 std::swap(lu_[(a * n_) + c], lu_[(b * n_) + c]);
1408 }
1409 }
1410 deferred_swaps_.clear();
1411 k0 = end;
1412 end = next;
1413 }
1414 }
1415
1425 std::size_t k0,
1426 std::size_t end)
1427 {
1428 const std::size_t rows {n_ - k0};
1429 const std::size_t width {end - k0};
1430 transpose_panel(panel, rows, k0, end, true); // column j of the panel at panel[j * rows], every element written
1431 factorize_columns(panel, rows, 0, width, k0, end);
1432 transpose_panel(panel, rows, k0, end, false);
1433 }
1434
1440 std::size_t rows,
1441 std::size_t k0,
1442 std::size_t end,
1443 bool in)
1444 {
1445 const std::size_t width {end - k0};
1446#ifdef NUMERICAL_X86_COMPILED
1447 if (rows >= detail::panel_tile_rows) {
1448 T* const base {lu_.data() + (k0 * n_) + k0};
1449 for (std::size_t r0 {0}; r0 < rows; r0 += detail::panel_copy_rows) {
1450 const std::size_t r1 {std::min(r0 + detail::panel_copy_rows, rows)};
1451 if (in) {
1452 for (std::size_t j {0}; j < width; ++j) {
1453 for (std::size_t r {r0}; r < r1; ++r) {
1454 panel[(j * rows) + r] = base[(r * n_) + j];
1455 }
1456 }
1457 }
1458 else {
1459 for (std::size_t j {0}; j < width; ++j) {
1460 for (std::size_t r {r0}; r < r1; ++r) {
1461 base[(r * n_) + j] = panel[(j * rows) + r];
1462 }
1463 }
1464 }
1465 }
1466 return;
1467 }
1468#endif
1469 for (std::size_t r {0}; r < rows; ++r) {
1470 for (std::size_t j {0}; j < width; ++j) {
1471 if (in) {
1472 panel[(j * rows) + r] = lu_[((k0 + r) * n_) + k0 + j];
1473 }
1474 else {
1475 lu_[((k0 + r) * n_) + k0 + j] = panel[(j * rows) + r];
1476 }
1477 }
1478 }
1479 }
1480
1491 std::size_t rows,
1492 std::size_t first,
1493 std::size_t last,
1494 std::size_t k0,
1495 std::size_t end)
1496 {
1497 // A leaf, or a panel too short for its products to pay for packing (order 64 on arm64: 30 µs unblocked,
1498 // 37 recursive).
1499 if (last - first <= detail::solve_leaf || rows - first < 2 * block_) {
1500 for (std::size_t j {first}; j < last; ++j) {
1501 T * column {panel.data() + (j * rows)};
1502 std::size_t pivot {0}; // from row j
1504 for (std::size_t r {detail::largest_magnitude_vectors(column + j, rows - j, best, pivot)}; r < rows - j; ++r) {
1506 if (mag > best) {
1507 best = mag;
1508 pivot = r;
1509 }
1510 }
1511 pivot += j;
1512 if (best == detail::magnitude_t<T> {}) {
1513 singular_ = true;
1514 continue; // a zero pivot column: leave it, the determinant becomes 0
1515 }
1516 if (pivot != j) {
1517 for (std::size_t c {0}; c < end - k0; ++c) { // the whole panel row
1518 std::swap(panel[(c * rows) + j], panel[(c * rows) + pivot]);
1519 }
1520 swap_rows_outside(k0 + j, k0 + pivot, k0, end);
1521 std::swap(pivots_[k0 + j], pivots_[k0 + pivot]);
1522 sign_ = -sign_;
1523 }
1524 const auto divisor {detail::pivot_divisor(column[j])};
1525 for (std::size_t r {j + 1}; r < rows; ++r) {
1526 column[r] = detail::over_pivot(column[r], divisor); // the multiplier, in L's slot
1527 }
1528 for (std::size_t c {j + 1}; c < last; ++c) {
1529 T* target {panel.data() + (c * rows)};
1530 detail::subtract_product(target + j + 1, column + j + 1, target[j], rows - (j + 1));
1531 }
1532 }
1533 return;
1534 }
1535 const std::size_t mid {first + ((last - first) / 2)};
1536 factorize_columns(panel, rows, first, mid, k0, end);
1537 for (std::size_t c {mid}; c < last; ++c) { // the top of the right half: rows [first, mid)
1538 T* target {panel.data() + (c * rows)};
1539 for (std::size_t j {first}; j < mid; ++j) {
1540 detail::subtract_product(target + j + 1, panel.data() + (j * rows) + j + 1, target[j], mid - (j + 1));
1541 }
1542 }
1543 // The bottom, rows [mid, rows): A₂₂ −= L₂₁ U₁₂, column-major, so as its transpose, row-major:
1544 // A₂₂ᵀ += (−U₁₂ᵀ) L₂₁ᵀ.
1545 const std::size_t depth {mid - first};
1546 buffer<T> minus_u((last - mid) * depth, T {});
1547 for (std::size_t c {mid}; c < last; ++c) {
1548 for (std::size_t j {first}; j < mid; ++j) {
1549 minus_u[((c - mid) * depth) + (j - first)] = -panel[(c * rows) + j];
1550 }
1551 }
1552 // The plain loop, which rounds as the tiles do (the same bits): a product this narrow packed a sliver of the
1553 // panel's whole height for each few columns (six x86-64 AVX2 cores, LU of order 1000: 6.86 → 6.36 ms).
1554 detail::gemm_rows(last - mid, rows - mid, depth, minus_u.data(), depth, panel.data() + (first * rows) + mid, rows,
1555 panel.data() + (mid * rows) + mid, rows);
1556 factorize_columns(panel, rows, mid, last, k0, end);
1557 }
1558
1562 void swap_rows_outside(std::size_t a,
1563 std::size_t b,
1564 std::size_t k0,
1565 std::size_t end)
1566 {
1567 if (k0 > 0) {
1568 left_swaps_.emplace_back(a, b);
1569 }
1570 for (std::size_t c {end}; c < swap_limit_; ++c) {
1571 std::swap(lu_[(a * n_) + c], lu_[(b * n_) + c]);
1572 }
1573 if (swap_limit_ < n_) {
1574 deferred_swaps_.emplace_back(a, b);
1575 }
1576 }
1577
1584 {
1585 if (left_swaps_.empty()) {
1586 return;
1587 }
1588 const std::size_t panels {(n_ - 1) / block_}; // every panel but the last, which no later swap reaches
1589 detail::parallel_for(panels, left_swaps_.size() * block_, [&](std::size_t p) {
1590 const std::size_t c0 {p * block_};
1591 const std::size_t c1 {c0 + block_};
1592 const auto first {std::lower_bound(left_swaps_.begin(), left_swaps_.end(), c1,
1593 [](const auto& swap, std::size_t row) { return swap.first < row; })};
1594 for (auto swap {first}; swap != left_swaps_.end(); ++swap) {
1595 std::swap_ranges(lu_.data() + (swap->first * n_) + c0, lu_.data() + (swap->first * n_) + c1,
1596 lu_.data() + (swap->second * n_) + c0);
1597 }
1598 });
1599 left_swaps_.clear();
1600 }
1601
1605 void solve_panel_rows(std::size_t k0,
1606 std::size_t end)
1607 {
1608 constexpr std::size_t width {64};
1609 const std::size_t blocks {((n_ - end) + width - 1) / width};
1610 detail::parallel_for(blocks, (end - k0) * (end - k0) * width / 2, [&](std::size_t block) {
1611 const std::size_t first {end + (block * width)};
1612 solve_columns(k0, end, first, std::min(first + width, n_));
1613 });
1614 }
1615
1618 void solve_columns(std::size_t k0,
1619 std::size_t end,
1620 std::size_t first,
1621 std::size_t last)
1622 {
1623 for (std::size_t i {k0 + 1}; i < end; ++i) {
1624 detail::subtract_products(lu_.data() + (i * n_) + first, lu_.data() + (i * n_) + k0, lu_.data() + (k0 * n_) + first, n_, i - k0, last - first);
1625 }
1626 }
1627
1631 std::size_t k0,
1632 std::size_t c0,
1633 std::size_t c1)
1634 {
1635 const std::size_t end {k0 + left.k};
1636 detail::gemm_with(left, c1 - c0, lu_.data() + (k0 * n_) + c0, n_, lu_.data() + (end * n_) + c0, n_);
1637 }
1638
1639 std::size_t n_;
1640 std::size_t block_ {detail::lu_block_for(n_)};
1642 std::vector<std::size_t> pivots_;
1643 int sign_ {1};
1644 bool singular_ {false};
1645 std::size_t swap_limit_ {n_};
1646 std::vector<std::pair<std::size_t, std::size_t>> deferred_swaps_;
1647 std::vector<std::pair<std::size_t, std::size_t>> left_swaps_;
1648 };
1649
1650 namespace detail {
1651
1657 [[nodiscard]] inline std::pair<matrix<float>, int> scaled_to_float(const matrix<double>& m)
1658 {
1659 const double* x {std::to_address(m.begin())};
1660 const std::size_t count {m.rows() * m.cols()};
1661 double l0 {0.0}; // four chains: a maximum is exact in any order
1662 double l1 {0.0};
1663 double l2 {0.0};
1664 double l3 {0.0};
1665 std::size_t i {0};
1666 for (; i + 4 <= count; i += 4) {
1667 l0 = std::max(l0, std::abs(x[i]));
1668 l1 = std::max(l1, std::abs(x[i + 1]));
1669 l2 = std::max(l2, std::abs(x[i + 2]));
1670 l3 = std::max(l3, std::abs(x[i + 3]));
1671 }
1672 for (; i < count; ++i) {
1673 l0 = std::max(l0, std::abs(x[i]));
1674 }
1675 int exponent {0};
1676 static_cast<void>(std::frexp(std::max(std::max(l0, l1), std::max(l2, l3)), &exponent));
1677 exponent = std::max(exponent, -1021); // 2⁻ᵉ finite: entries below 2⁻¹⁰²¹ vanish in single precision whatever the scale
1678 const double scale {std::ldexp(1.0, -exponent)};
1679 buffer<float> out(count);
1680 std::transform(m.begin(), m.end(), out.begin(), [scale](double v) { return static_cast<float>(v * scale); });
1681 return {matrix<float> {m.rows(), m.cols(), std::move(out)}, exponent};
1682 }
1683
1686 [[nodiscard]] inline double residual_ratio(const matrix<double>& x,
1687 const matrix<double>& r)
1688 {
1689 // An infinity or a NaN of the solution reaches the residual: no column of A is zero (A is not singular).
1690 if (!all_finite(std::to_address(r.begin()), r.rows() * r.cols())) {
1691 return std::numeric_limits<double>::quiet_NaN();
1692 }
1693 double ratio {0.0};
1694 for (std::size_t j {0}; j < x.cols(); ++j) {
1695 double x_norm {0.0};
1696 double r_norm {0.0};
1697 for (std::size_t i {0}; i < x.rows(); ++i) {
1698 x_norm = std::max(x_norm, std::abs(x(i, j)));
1699 r_norm = std::max(r_norm, std::abs(r(i, j)));
1700 }
1701 ratio = std::max(ratio, r_norm / x_norm);
1702 }
1703 return ratio;
1704 }
1705
1724 [[nodiscard]] inline std::optional<matrix<double>> mixed_precision_solve(const matrix<double>& a,
1725 const matrix<double>& b,
1726 double* reciprocal_condition = nullptr)
1727 {
1728 const std::size_t n {a.rows()};
1729 double a_norm {0.0};
1730 for (auto row = a.begin(); row != a.end(); row += static_cast<std::ptrdiff_t>(n)) {
1731 std::array<double, 4> sums {}; // four chains: the bound needs no particular rounding of the sum
1732 for (std::size_t j {0}; j < n; ++j) {
1733 sums[j % 4] += std::abs(row[static_cast<std::ptrdiff_t>(j)]);
1734 }
1735 a_norm = std::max(a_norm, (sums[0] + sums[1]) + (sums[2] + sums[3]));
1736 }
1737 if (!all_finite(std::to_address(a.begin()), n * n) || !all_finite(std::to_address(b.begin()), b.rows() * b.cols())) {
1738 return std::nullopt;
1739 }
1740 const double bound {a_norm * (std::numeric_limits<double>::epsilon() / 2.0) * std::sqrt(static_cast<double>(n))};
1741 auto [single_a, a_exponent] {scaled_to_float(a)};
1742 const lu_decomposition<float> factored {std::move(single_a)};
1743 if (factored.singular()) {
1744 return std::nullopt;
1745 }
1746 const auto solved = [&factored, a_exponent](std::pair<matrix<float>, int> rhs) { // A⁻¹ times the unscaled rhs
1747 const matrix<float> y {factored.solve(std::move(rhs.first))};
1748 const double up {std::ldexp(1.0, rhs.second)};
1749 const double down {std::ldexp(1.0, -a_exponent)};
1750 buffer<double> out(y.rows() * y.cols());
1751 std::transform(y.begin(), y.end(), out.begin(), [up, down](float v) { return (static_cast<double>(v) * up) * down; });
1752 return matrix<double> {y.rows(), y.cols(), std::move(out)};
1753 };
1754 matrix<double> x {solved(scaled_to_float(b))};
1755 matrix<double> r {b - (a * x)};
1756 double ratio {residual_ratio(x, r)};
1757 double previous {std::numeric_limits<double>::infinity()};
1758 while (!(ratio <= bound)) {
1759 if (!(ratio <= previous / 8.0)) { // the last correction fell short, or the residual is not finite
1760 return std::nullopt;
1761 }
1762 previous = ratio;
1763 x = x + solved(scaled_to_float(r));
1764 r = b - (a * x);
1765 ratio = residual_ratio(x, r);
1766 }
1767 if (reciprocal_condition != nullptr) { // the scaled A's ‖·‖₁, its condition A's
1768 std::vector<double> sums(n, 0.0);
1769 for (std::size_t i {0}; i < n; ++i) {
1770 for (std::size_t j {0}; j < n; ++j) {
1771 sums[j] += std::abs(a(i, j));
1772 }
1773 }
1774 *reciprocal_condition = factored.reciprocal_condition(std::ldexp(*std::max_element(sums.begin(), sums.end()), -a_exponent));
1775 }
1776 return x;
1777 }
1778 } // namespace detail
1779
1788 template <field T>
1789 [[nodiscard]] vector<T> solve(matrix<T> a,
1790 const vector<T>& b)
1791 {
1792 return lu_decomposition<T> {std::move(a)}.solve(b);
1793 }
1794
1804 template <field T>
1805 [[nodiscard]] matrix<T> solve(matrix<T> a,
1806 const matrix<T>& b)
1807 {
1808 return lu_decomposition<T> {std::move(a)}.solve(b);
1809 }
1810
1812 template <field T>
1813 [[nodiscard]] T determinant(matrix<T> a)
1814 {
1815 return lu_decomposition<T> {std::move(a)}.determinant();
1816 }
1817
1825 template <field T>
1826 [[nodiscard]] matrix<T> inverse(matrix<T> a)
1827 {
1828 return lu_decomposition<T> {std::move(a)}.inverse();
1829 }
1830
1833 enum class triangular_op : unsigned char
1834 {
1835 none,
1836 transpose,
1837 adjoint
1838 };
1839
1840 namespace detail {
1841
1843 inline constexpr std::size_t triangular_block {32};
1844
1846 inline constexpr std::size_t triangular_slice {64};
1847
1861 template <typename T>
1862 void triangular_halves(const T* a,
1863 std::size_t n,
1864 bool lower,
1865 bool transposed,
1866 bool unit,
1867 T* x,
1868 std::size_t ldx,
1869 std::size_t k,
1870 std::size_t first,
1871 std::size_t last,
1872 T* minus)
1873 {
1874 const std::size_t width {last - first};
1875 if (width <= triangular_block) {
1876 std::array<T, triangular_block> terms {}; // a column of a, gathered, where M is its transpose
1877 for (std::size_t step {0}; step < width; ++step) {
1878 const std::size_t i {lower ? first + step : last - 1 - step};
1879 T* const row {x + (i * ldx)};
1880 const std::size_t from {lower ? first : i + 1};
1881 const std::size_t to {lower ? i : last};
1882 const T* f {a + (i * n) + from};
1883 if (transposed) {
1884 for (std::size_t j {from}; j < to; ++j) {
1885 terms[j - from] = a[(j * n) + i];
1886 }
1887 f = terms.data();
1888 }
1889 subtract_products(row, f, x + (from * ldx), ldx, to - from, k); // the terms in order, in registers
1890 if (!unit) {
1891 const T diagonal {a[(i * n) + i]};
1892 for (std::size_t c {0}; c < k; ++c) {
1893 row[c] /= diagonal;
1894 }
1895 }
1896 }
1897 return;
1898 }
1899 const std::size_t middle {first + (((width / 2) + triangular_block - 1) / triangular_block * triangular_block)};
1900 const std::size_t solved_first {lower ? first : middle}; // the half solved first
1901 const std::size_t solved_last {lower ? middle : last};
1902 const std::size_t other_first {lower ? middle : first};
1903 const std::size_t other_last {lower ? last : middle};
1904 triangular_halves(a, n, lower, transposed, unit, x, ldx, k, solved_first, solved_last, minus);
1905 const std::size_t depth {solved_last - solved_first};
1906 for (std::size_t r {0}; r < depth; ++r) {
1907 for (std::size_t c {0}; c < k; ++c) {
1908 minus[(r * k) + c] = -x[((solved_first + r) * ldx) + c];
1909 }
1910 }
1911 const std::size_t rows {other_last - other_first};
1912 if (transposed) { // M's block is the transpose of a's block at (solved, other)
1913 gemm_at(rows, k, depth, a + (solved_first * n) + other_first, n, minus, k, x + (other_first * ldx), ldx);
1914 }
1915 else {
1916 gemm(rows, k, depth, a + (other_first * n) + solved_first, n, minus, k, x + (other_first * ldx), ldx);
1917 }
1918 triangular_halves(a, n, lower, transposed, unit, x, ldx, k, other_first, other_last, minus);
1919 }
1920
1929 template <typename T>
1930 void triangular_rows(const T* a,
1931 std::size_t n,
1932 bool lower,
1933 bool transposed,
1934 bool unit,
1935 T* x,
1936 std::size_t k)
1937 {
1938 if (k == 1 && !transposed) { // each unknown its row's dot product with those solved, the row read where it runs
1939 for (std::size_t step {0}; step < n; ++step) {
1940 const std::size_t i {lower ? step : n - 1 - step};
1941 const std::size_t from {lower ? 0 : i + 1};
1942 const std::size_t count {lower ? i : n - i - 1};
1943 const T value {x[i] - dot_product(a + (i * n) + from, x + from, count)};
1944 x[i] = unit ? value : value / a[(i * n) + i];
1945 }
1946 return;
1947 }
1948 if (k == 1) { // aᵀ: each unknown solved, then its terms out of the rest along its row of a
1949 for (std::size_t step {0}; step < n; ++step) {
1950 const std::size_t j {lower ? n - 1 - step : step};
1951 if (!unit) {
1952 x[j] /= a[(j * n) + j];
1953 }
1954 const std::size_t from {lower ? 0 : j + 1};
1955 subtract_product(x + from, a + (j * n) + from, x[j], lower ? j : n - j - 1);
1956 }
1957 return;
1958 }
1959 const bool m_lower {lower != transposed};
1960 if (k <= triangular_slice) { // one block of columns: the products take the threads
1961 buffer<T> minus((((n + 1) / 2) + triangular_block) * k); // a half's solution, negated
1962 triangular_halves(a, n, m_lower, transposed, unit, x, k, k, 0, n, minus.data());
1963 return;
1964 }
1965 // The columns apart, a block of them to a thread, each solved by halves: the same operations on each element
1966 // whatever the threads.
1967 parallel_for((k + triangular_slice - 1) / triangular_slice, n * n * triangular_slice / 2, [&](std::size_t block) {
1968 const std::size_t start {block * triangular_slice};
1969 const std::size_t count {std::min(triangular_slice, k - start)};
1970 buffer<T> minus((((n + 1) / 2) + triangular_block) * count); // a half's solution, negated
1971 triangular_halves(a, n, m_lower, transposed, unit, x + start, k, count, 0, n, minus.data());
1972 });
1973 }
1974 } // namespace detail
1975
1976 namespace detail {
1977
1979 template <typename T>
1980 [[nodiscard]] bool elements_finite(const T* x,
1981 std::size_t count)
1982 {
1983 return std::all_of(x, x + count, [](const T& value) { return is_finite_entry(value); });
1984 }
1985
1987 [[nodiscard]] inline bool elements_finite(const double* x,
1988 std::size_t count)
1989 {
1990 return all_finite_shared(x, count);
1991 }
1992
1994 [[nodiscard]] inline bool elements_finite(const std::complex<double>* x,
1995 std::size_t count)
1996 {
1997 return all_finite_shared(reinterpret_cast<const double*>(x), 2 * count); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
1998 }
1999
2002 template <typename T>
2004 std::size_t n,
2005 bool lower)
2006 {
2007 for (std::size_t i {0}; i < n; ++i) {
2008 const T* const row {a + (i * n)};
2009 if (!(lower ? elements_finite(row, i + 1) : elements_finite(row + i, n - i))) {
2010 return false;
2011 }
2012 }
2013 return true;
2014 }
2015
2022 template <typename T>
2024 std::size_t n,
2025 bool lower,
2026 const T* x,
2027 std::size_t count)
2028 {
2029 if (!elements_finite(a, n * n) && !triangle_is_finite(a, n, lower)) {
2030 throw non_finite_error("solve_triangular: the matrix holds an infinite or NaN entry");
2031 }
2032 if (!elements_finite(x, count)) {
2033 throw non_finite_error("solve_triangular: the right-hand side holds an infinite or NaN entry");
2034 }
2035 }
2036
2041 template <typename T>
2043 std::size_t n,
2044 bool lower,
2045 triangular_op op,
2046 bool unit_diagonal,
2047 T* x,
2048 std::size_t k)
2049 {
2050 for (std::size_t i {0}; i < n && !unit_diagonal; ++i) {
2051 if (a[(i * n) + i] == T {}) {
2052 throw singular_matrix_error("singular matrix: resolution failed at diagonal " + std::to_string(i + 1));
2053 }
2054 }
2055 const bool adjoint {op == triangular_op::adjoint}; // a real one's conjugates are itself
2056 if (adjoint) {
2057 std::transform(x, x + (n * k), x, [](const T& value) { return conj_of(value); });
2058 }
2059 triangular_rows(a, n, lower, op != triangular_op::none, unit_diagonal, x, k);
2060 if (adjoint) {
2061 std::transform(x, x + (n * k), x, [](const T& value) { return conj_of(value); });
2062 }
2063 }
2064
2070 template <typename T>
2072 std::size_t n,
2073 bool lower,
2074 triangular_op op,
2075 bool unit_diagonal,
2076 T* x,
2077 std::size_t k)
2078 {
2079 require_finite_triangle(a, n, lower, x, n * k);
2080 solve_finite_triangular_in_place(a, n, lower, op, unit_diagonal, x, k);
2081 }
2082
2085 template <std::floating_point R>
2086 [[nodiscard]] R sign_entry(R x,
2087 int& sign) noexcept
2088 {
2089 sign = x >= R {} ? 1 : -1;
2090 return static_cast<R>(sign);
2091 }
2092
2095 template <std::floating_point R>
2096 [[nodiscard]] std::complex<R> sign_entry(std::complex<R> x,
2097 int& sign) noexcept
2098 {
2099 sign = 0;
2100 const R modulus {std::abs(x)};
2101 return modulus > std::numeric_limits<R>::min() ? x / modulus : std::complex<R> {1};
2102 }
2103
2105 template <std::floating_point R>
2106 [[nodiscard]] bool same_sign(R x,
2107 int sign) noexcept
2108 {
2109 return (x >= R {} ? 1 : -1) == sign;
2110 }
2111
2113 template <std::floating_point R>
2114 [[nodiscard]] bool same_sign(std::complex<R> /*x*/,
2115 int /*sign*/) noexcept
2116 {
2117 return false;
2118 }
2119
2122 template <std::floating_point R>
2123 [[nodiscard]] bool climb_ends(R last,
2124 R largest) noexcept
2125 {
2126 return last == std::abs(largest);
2127 }
2128
2130 template <std::floating_point R>
2131 [[nodiscard]] bool climb_ends(std::complex<R> last,
2132 std::complex<R> largest) noexcept
2133 {
2134 return std::abs(last) == std::abs(largest);
2135 }
2136
2149 template <typename T>
2150 [[nodiscard]] double one_norm_estimate(std::size_t n,
2151 const std::function<void(T*, bool)>& apply)
2152 {
2153 constexpr std::size_t most_climbs {5};
2154 buffer<T> x(n, T {static_cast<decltype(std::abs(T {}))>(1.0 / static_cast<double>(n))});
2155 std::vector<int> signs(n, 0); // a real climb's last sign vector
2156 const auto product = [&](bool adjoint) {
2157 apply(x.data(), adjoint);
2158 return elements_finite(x.data(), n);
2159 };
2160 const auto sum = [&] {
2161 double total {0.0};
2162 for (const T& value : x) {
2163 total += static_cast<double>(std::abs(value));
2164 }
2165 return total;
2166 };
2167 const auto largest_at = [&] { // the first of the largest moduli, as idamax and izmax1
2168 std::size_t at {0};
2169 for (std::size_t i {1}; i < n; ++i) {
2170 if (std::abs(x[i]) > std::abs(x[at])) {
2171 at = i;
2172 }
2173 }
2174 return at;
2175 };
2176 const auto to_signs = [&] {
2177 for (std::size_t i {0}; i < n; ++i) {
2178 x[i] = sign_entry(x[i], signs[i]);
2179 }
2180 };
2181 if (!product(false)) {
2182 return std::numeric_limits<double>::infinity();
2183 }
2184 if (n == 1) {
2185 return static_cast<double>(std::abs(x[0]));
2186 }
2187 double estimate {sum()};
2188 to_signs();
2189 if (!product(true)) {
2190 return std::numeric_limits<double>::infinity();
2191 }
2192 std::size_t j {largest_at()};
2193 for (std::size_t climb {2};; ++climb) {
2194 std::fill(x.begin(), x.end(), T {});
2195 x[j] = T {1};
2196 if (!product(false)) {
2197 return std::numeric_limits<double>::infinity();
2198 }
2199 const double previous {estimate};
2200 estimate = sum();
2201 bool repeated {true}; // a real climb's signs as before: it has converged
2202 for (std::size_t i {0}; i < n && repeated; ++i) {
2203 repeated = same_sign(x[i], signs[i]);
2204 }
2205 if (repeated || estimate <= previous) {
2206 break;
2207 }
2208 to_signs();
2209 if (!product(true)) {
2210 return std::numeric_limits<double>::infinity();
2211 }
2212 const std::size_t last {j};
2213 j = largest_at();
2214 if (climb_ends(x[last], x[j]) || climb >= most_climbs) {
2215 break;
2216 }
2217 }
2218 for (std::size_t i {0}; i < n; ++i) { // the alternating vector, of growing moduli
2219 const double magnitude {1.0 + (static_cast<double>(i) / static_cast<double>(n - 1))};
2220 x[i] = T {static_cast<decltype(std::abs(T {}))>(i % 2 == 0 ? magnitude : -magnitude)};
2221 }
2222 if (!product(false)) {
2223 return std::numeric_limits<double>::infinity();
2224 }
2225 return std::max(estimate, 2.0 * (sum() / static_cast<double>(3 * n)));
2226 }
2227
2231 template <typename T>
2232 [[nodiscard]] double reciprocal_condition(std::size_t n,
2233 double one_norm,
2234 const std::function<void(T*, bool)>& apply)
2235 {
2236 if (n == 0) {
2237 return 1.0;
2238 }
2239 if (one_norm == 0.0) {
2240 return 0.0;
2241 }
2242 return (1.0 / one_norm_estimate<T>(n, apply)) / one_norm;
2243 }
2244
2250 template <typename T>
2251 [[nodiscard]] double lu_reciprocal_condition(const T* lu,
2252 std::size_t n,
2253 double one_norm)
2254 {
2255 for (std::size_t i {0}; i < n; ++i) {
2256 if (lu[(i * n) + i] == T {}) {
2257 return 0.0;
2258 }
2259 }
2260 return reciprocal_condition<T>(n, one_norm, [lu, n](T* x, bool adjoint) {
2261 const triangular_op op {adjoint ? triangular_op::adjoint : triangular_op::none};
2264 });
2265 }
2266
2272 template <typename T>
2273 [[nodiscard]] double cholesky_reciprocal_condition(const T* factor,
2274 std::size_t n,
2275 bool lower,
2276 double one_norm)
2277 {
2278 return reciprocal_condition<T>(n, one_norm, [factor, n, lower](T* x, bool /*adjoint*/) {
2279 solve_finite_triangular_in_place(factor, n, lower, lower ? triangular_op::none : triangular_op::adjoint, false, x, 1);
2280 solve_finite_triangular_in_place(factor, n, lower, lower ? triangular_op::adjoint : triangular_op::none, false, x, 1);
2281 });
2282 }
2283
2289 template <typename T>
2290 [[nodiscard]] double triangular_reciprocal_condition(const T* a,
2291 std::size_t n,
2292 bool lower)
2293 {
2294 std::vector<double> sums(n, 0.0);
2295 for (std::size_t i {0}; i < n; ++i) {
2296 if (a[(i * n) + i] == T {}) {
2297 return 0.0;
2298 }
2299 for (std::size_t j {lower ? 0 : i}; j <= (lower ? i : n - 1); ++j) {
2300 sums[j] += static_cast<double>(std::abs(a[(i * n) + j]));
2301 }
2302 }
2303 const double one_norm {n == 0 ? 0.0 : *std::max_element(sums.begin(), sums.end())};
2304 return reciprocal_condition<T>(n, one_norm, [a, n, lower](T* x, bool adjoint) {
2305 solve_finite_triangular_in_place(a, n, lower, adjoint ? triangular_op::adjoint : triangular_op::none, false, x, 1);
2306 });
2307 }
2308 } // namespace detail
2309
2310 template <field T>
2311 double lu_decomposition<T>::reciprocal_condition(double one_norm) const
2312 {
2313 return detail::lu_reciprocal_condition(lu_.data(), n_, one_norm);
2314 }
2315
2326 template <field T>
2327 [[nodiscard]] double triangular_reciprocal_condition(const matrix<T>& a,
2328 bool lower)
2329 {
2330 const std::size_t n {a.rows()};
2331 if (a.cols() != n) {
2332 throw dimension_error("triangular_reciprocal_condition: expected a square matrix");
2333 }
2334 const T* const at {std::to_address(a.begin())};
2335 detail::require_finite_triangle(at, n, lower, at, 0);
2336 return detail::triangular_reciprocal_condition(at, n, lower);
2337 }
2338
2354 template <field T>
2355 [[nodiscard]] matrix<T> solve_triangular(const matrix<T>& a,
2356 matrix<T> b,
2357 bool lower,
2358 triangular_op op = triangular_op::none,
2359 bool unit_diagonal = false)
2360 {
2361 const std::size_t n {a.rows()};
2362 if (a.cols() != n) {
2363 throw dimension_error("solve_triangular: expected a square matrix");
2364 }
2365 if (b.rows() != n) {
2366 throw dimension_error("solve_triangular: right-hand side row count differs from the matrix");
2367 }
2368 const std::size_t k {b.cols()};
2369 buffer<T> x {std::move(b).elements()};
2370 detail::solve_triangular_in_place(std::to_address(a.begin()), n, lower, op, unit_diagonal, x.data(), k);
2371 return matrix<T> {n, k, std::move(x)};
2372 }
2373
2375 template <field T>
2376 [[nodiscard]] vector<T> solve_triangular(const matrix<T>& a,
2377 const vector<T>& b,
2378 bool lower,
2379 triangular_op op = triangular_op::none,
2380 bool unit_diagonal = false)
2381 {
2382 matrix<T> column {solve_triangular(a, matrix<T> {b.size(), 1, buffer<T>(b.begin(), b.end())}, lower, op, unit_diagonal)};
2383 return vector<T> {std::move(column).elements()};
2384 }
2385
2387 template <typename T>
2389 {
2390 std::vector<std::size_t> rows;
2393 };
2394
2395 namespace detail {
2396
2399 template <typename T>
2400 [[nodiscard]] std::pair<matrix<T>, matrix<T>> split_lu(const T* lu,
2401 std::size_t rows,
2402 std::size_t cols,
2403 std::size_t k)
2404 {
2405 buffer<T> l(rows * k, T {});
2406 buffer<T> u(k * cols, T {});
2407 for (std::size_t i {0}; i < rows; ++i) {
2408 for (std::size_t j {0}; j < k; ++j) {
2409 l[(i * k) + j] = i == j ? T {1} : (j < i ? lu[(i * cols) + j] : T {});
2410 }
2411 }
2412 for (std::size_t i {0}; i < k; ++i) {
2413 for (std::size_t j {i}; j < cols; ++j) {
2414 u[(i * cols) + j] = lu[(i * cols) + j];
2415 }
2416 }
2417 return {matrix<T> {rows, k, std::move(l)}, matrix<T> {k, cols, std::move(u)}};
2418 }
2419
2428 template <typename T>
2429 void tall_lu_columns(T* panel,
2430 std::size_t m,
2431 std::size_t n,
2432 std::size_t first,
2433 std::size_t last,
2434 std::vector<std::size_t>& rows)
2435 {
2436 if (last - first <= solve_leaf || m - first < 2 * lu_block_for(m)) {
2437 for (std::size_t j {first}; j < last; ++j) {
2438 T* const column {panel + (j * m)};
2439 std::size_t pivot {0}; // from row j
2440 magnitude_t<T> best {pivot_magnitude(column[j])};
2441 for (std::size_t r {largest_magnitude_vectors(column + j, m - j, best, pivot)}; r < m - j; ++r) {
2442 const magnitude_t<T> mag {pivot_magnitude(column[j + r])};
2443 if (mag > best) {
2444 best = mag;
2445 pivot = r;
2446 }
2447 }
2448 pivot += j;
2449 if (best == magnitude_t<T> {}) {
2450 continue; // a zero column below the diagonal: nothing to eliminate
2451 }
2452 if (pivot != j) {
2453 for (std::size_t c {0}; c < n; ++c) {
2454 std::swap(panel[(c * m) + j], panel[(c * m) + pivot]);
2455 }
2456 std::swap(rows[j], rows[pivot]);
2457 }
2458 const auto divisor {pivot_divisor(column[j])};
2459 for (std::size_t r {j + 1}; r < m; ++r) {
2460 column[r] = over_pivot(column[r], divisor); // the multiplier, in L's slot
2461 }
2462 for (std::size_t c {j + 1}; c < last; ++c) {
2463 T* const target {panel + (c * m)};
2464 subtract_product(target + j + 1, column + j + 1, target[j], m - (j + 1));
2465 }
2466 }
2467 return;
2468 }
2469 const std::size_t mid {first + ((last - first) / 2)};
2470 tall_lu_columns(panel, m, n, first, mid, rows);
2471 for (std::size_t c {mid}; c < last; ++c) { // the top of the right half: rows [first, mid)
2472 T* const target {panel + (c * m)};
2473 for (std::size_t j {first}; j < mid; ++j) {
2474 subtract_product(target + j + 1, panel + (j * m) + j + 1, target[j], mid - (j + 1));
2475 }
2476 }
2477 // The bottom, rows [mid, m): A₂₂ −= L₂₁ U₁₂, column-major, so as its transpose, row-major: A₂₂ᵀ += (−U₁₂ᵀ) L₂₁ᵀ.
2478 const std::size_t depth {mid - first};
2479 buffer<T> minus_u((last - mid) * depth, T {});
2480 for (std::size_t c {mid}; c < last; ++c) {
2481 for (std::size_t j {first}; j < mid; ++j) {
2482 minus_u[((c - mid) * depth) + (j - first)] = -panel[(c * m) + j];
2483 }
2484 }
2485 gemm_rows(last - mid, m - mid, depth, minus_u.data(), depth, panel + (first * m) + mid, m, panel + (mid * m) + mid, m);
2486 tall_lu_columns(panel, m, n, mid, last, rows);
2487 }
2488
2492 inline constexpr std::size_t lu_tall_ratio {2};
2493 } // namespace detail
2494
2505 template <typename T>
2506 requires std::same_as<T, double> || std::same_as<T, std::complex<double>>
2508 {
2509 const std::size_t m {a.rows()};
2510 const std::size_t n {a.cols()};
2511 const std::size_t k {std::min(m, n)};
2512 detail::require_finite(a, "LU decomposition");
2513 std::vector<std::size_t> identity(m);
2514 std::iota(identity.begin(), identity.end(), std::size_t {0});
2515 if (k == 0) {
2516 return {std::move(identity), matrix<T> {m, k, buffer<T>(m * k)}, matrix<T> {k, n, buffer<T>(k * n)}};
2517 }
2518 if (m <= n) { // the square part's factorisation; U's other columns L⁻¹ (its rows in the pivoted order)
2519 buffer<T> square(m * m);
2520 for (std::size_t i {0}; i < m; ++i) {
2521 std::copy_n(&a(i, 0), m, square.data() + (i * m));
2522 }
2523 const lu_decomposition<T> f {matrix<T> {m, m, std::move(square)}};
2524 const matrix<T> lu {f.factors()};
2525 buffer<T> combined(m * n);
2526 for (std::size_t i {0}; i < m; ++i) {
2527 std::copy_n(&lu(i, 0), m, combined.data() + (i * n));
2528 }
2529 if (n > m) {
2530 buffer<T> rest(m * (n - m));
2531 for (std::size_t i {0}; i < m; ++i) {
2532 std::copy_n(&a(f.pivots()[i], m), n - m, rest.data() + (i * (n - m)));
2533 }
2534 detail::solve_triangular_in_place(std::to_address(lu.begin()), m, true, triangular_op::none, true, rest.data(), n - m); // L⁻¹
2535 for (std::size_t i {0}; i < m; ++i) {
2536 std::copy_n(rest.data() + (i * (n - m)), n - m, combined.data() + (i * n) + m);
2537 }
2538 }
2539 auto [l, u] {detail::split_lu(combined.data(), m, n, k)};
2540 return {f.pivots(), std::move(l), std::move(u)};
2541 }
2542 if (m <= detail::lu_tall_ratio * n) { // beside zero columns, square
2543 buffer<T> padded(m * m, T {});
2544 for (std::size_t i {0}; i < m; ++i) {
2545 std::copy_n(&a(i, 0), n, padded.data() + (i * m));
2546 }
2547 const lu_decomposition<T> f {matrix<T> {m, m, std::move(padded)}};
2548 const matrix<T> lu {f.factors()};
2549 auto [l, u] {detail::split_lu(std::to_address(lu.begin()), m, m, k)};
2550 buffer<T> top(k * n); // U's first n columns
2551 for (std::size_t i {0}; i < k; ++i) {
2552 std::copy_n(&u(i, 0), n, top.data() + (i * n));
2553 }
2554 return {f.pivots(), std::move(l), matrix<T> {k, n, std::move(top)}};
2555 }
2556 buffer<T> panel(m * n); // column-major: column j at panel[j m]
2557 for (std::size_t i {0}; i < m; ++i) {
2558 for (std::size_t j {0}; j < n; ++j) {
2559 panel[(j * m) + i] = a(i, j);
2560 }
2561 }
2562 detail::tall_lu_columns(panel.data(), m, n, 0, n, identity);
2563 buffer<T> combined(m * n);
2564 for (std::size_t i {0}; i < m; ++i) {
2565 for (std::size_t j {0}; j < n; ++j) {
2566 combined[(i * n) + j] = panel[(j * m) + i];
2567 }
2568 }
2569 auto [l, u] {detail::split_lu(combined.data(), m, n, k)};
2570 return {std::move(identity), std::move(l), std::move(u)};
2571 }
2572
2574 class not_positive_definite_error : public detail::tagged<std::runtime_error>
2575 {
2576 public:
2577
2579 explicit not_positive_definite_error(const std::string& message,
2580 std::size_t order = 0)
2581 : tagged(message), order_ {order}
2582 {}
2583
2585 [[nodiscard]] std::size_t order() const noexcept
2586 {
2587 return order_;
2588 }
2589
2590 private:
2591
2592 std::size_t order_;
2593 };
2594
2605 template <field T>
2607 {
2608 public:
2609
2617 : cholesky_decomposition(detail::parallel_copy_of(a))
2618 {}
2619
2628 : n_(a.rows())
2629 {
2630 if (a.cols() != n_) {
2631 throw dimension_error("Cholesky decomposition requires a square matrix");
2632 }
2633 detail::require_finite_lower(a, "Cholesky decomposition");
2634 lower_ = std::move(a).elements(); // the upper triangle is never read, and zeroed once factorised
2635 factorize();
2636 }
2637
2639 [[nodiscard]] matrix<T> lower() const&
2640 {
2641 return matrix<T> {n_, n_, lower_};
2642 }
2643
2645 [[nodiscard]] matrix<T> lower() &&
2646 {
2647 return matrix<T> {n_, n_, std::move(lower_)};
2648 }
2649
2656 [[nodiscard]] vector<T> solve(const vector<T>& b) const
2657 {
2658 if (b.size() != n_) {
2659 throw dimension_error("solve: right-hand side size differs from the matrix");
2660 }
2661 buffer<T> y(n_, T {});
2662 for (std::size_t i {0}; i < n_; ++i) { // forward: L y = b
2663 T sum {b[i]};
2664 for (std::size_t k {0}; k < i; ++k) {
2665 sum -= lower_[(i * n_) + k] * y[k];
2666 }
2667 y[i] = detail::over_real_diagonal(sum, lower_[(i * n_) + i]);
2668 }
2669 buffer<T> x(n_, T {});
2670 for (std::size_t i {n_}; i-- > 0;) { // back: Lᴴ x = y, (Lᴴ)[i][k] = conj(L[k][i])
2671 T sum {y[i]};
2672 for (std::size_t k {i + 1}; k < n_; ++k) {
2673 sum -= detail::conj_of(lower_[(k * n_) + i]) * x[k];
2674 }
2675 x[i] = detail::over_real_diagonal(sum, lower_[(i * n_) + i]); // its own conjugate
2676 }
2677 return vector<T> {std::move(x)};
2678 }
2679
2688 [[nodiscard]] matrix<T> solve(const matrix<T>& b) const
2689 {
2690 if (b.rows() != n_) {
2691 throw dimension_error("solve: right-hand side row count differs from the matrix");
2692 }
2693 const std::size_t cols {b.cols()};
2694 buffer<T> x(b.begin(), b.end());
2695 const std::size_t slices {std::max<std::size_t>(1, std::min(max_threads(), cols / 8))};
2696 const std::size_t slice {(cols + slices - 1) / slices};
2697 detail::parallel_for(slices, n_ * n_ * slice, [&](std::size_t s) {
2698 const std::size_t c0 {s * slice};
2699 const std::size_t c1 {std::min(cols, c0 + slice)};
2700 for (std::size_t i {0}; i < n_; ++i) { // forward: L Y = B
2701 T* const row {x.data() + (i * cols)};
2702 for (std::size_t k {0}; k < i; ++k) {
2703 const T l {lower_[(i * n_) + k]};
2704 const T* const above {x.data() + (k * cols)};
2705 for (std::size_t c {c0}; c < c1; ++c) {
2706 row[c] -= l * above[c];
2707 }
2708 }
2709 for (std::size_t c {c0}; c < c1; ++c) {
2710 row[c] = detail::over_real_diagonal(row[c], lower_[(i * n_) + i]);
2711 }
2712 }
2713 for (std::size_t i {n_}; i-- > 0;) { // back: Lᴴ X = Y, (Lᴴ)[i][k] = conj(L[k][i])
2714 T* const row {x.data() + (i * cols)};
2715 for (std::size_t k {i + 1}; k < n_; ++k) {
2716 const T l {detail::conj_of(lower_[(k * n_) + i])};
2717 const T* const below {x.data() + (k * cols)};
2718 for (std::size_t c {c0}; c < c1; ++c) {
2719 row[c] -= l * below[c];
2720 }
2721 }
2722 for (std::size_t c {c0}; c < c1; ++c) {
2723 row[c] = detail::over_real_diagonal(row[c], lower_[(i * n_) + i]);
2724 }
2725 }
2726 });
2727 return matrix<T> {n_, cols, std::move(x)};
2728 }
2729
2732 [[nodiscard]] T determinant() const
2733 {
2734 // each diagonal entry twice, not its square, which may itself pass the doubles
2735 return detail::scaled_product(static_cast<T>(1), 2 * n_, [this](std::size_t k) { return lower_[((k / 2) * n_) + (k / 2)]; });
2736 }
2737
2745 [[nodiscard]] double reciprocal_condition(double one_norm) const
2746 {
2747 return detail::cholesky_reciprocal_condition(lower_.data(), n_, true, one_norm);
2748 }
2749
2750 private:
2751
2752 // By panels of block_ columns: the panel's diagonal block row by row, the rows below it
2753 // by leaves (each depends only on the diagonal block; factor_rows_below), then the lower part of the trailing block takes
2754 // A22 += (-L21) L21ᴴ from the matrix product kernel. Every element still receives its terms in ascending
2755 // order, so without multiply-add fusion the factor is the unblocked algorithm's to the bit.
2757 {
2758 buffer<T> packing; // L₂₁ packed for the trailing update's tiles, kept from panel to panel
2759 buffer<T> minus_adjoint; // −L₂₁ᴴ, row-major, the same
2760 for (std::size_t k0 {0}; k0 < n_; k0 += block_) {
2761 const std::size_t end {std::min(k0 + block_, n_)};
2762 std::size_t i {k0};
2763 for (; end - k0 >= detail::cholesky_together_width && i + detail::cholesky_diagonal_rows <= end; i += detail::cholesky_diagonal_rows) {
2764 factor_rows_together(i, k0);
2765 for (std::size_t r {0}; r < detail::cholesky_diagonal_rows; ++r) { // their triangle, row after row
2766 factor_row(i + r, k0, i, i + r + 1);
2767 }
2768 }
2769 for (; i < end; ++i) {
2770 factor_row(i, k0, k0, i + 1);
2771 }
2772 if (end < n_) {
2773 minus_adjoint.resize((end - k0) * (n_ - end));
2774 factor_rows_below(k0, end, minus_adjoint);
2775 update_trailing(k0, end, minus_adjoint, packing);
2776 }
2777 }
2778 T* const rows {lower_.data()};
2779 const std::size_t n {n_};
2780 // The upper triangle zeroed by bands of rows on the threads: a pass bound by the memory, half a row an element.
2781 detail::parallel_for((n + detail::zeroed_rows - 1) / detail::zeroed_rows, detail::memory_element_weight * detail::zeroed_rows * n / 2, [rows, n](std::size_t band) {
2782 for (std::size_t i {band * detail::zeroed_rows}; i < std::min(n, (band + 1) * detail::zeroed_rows); ++i) {
2783 std::fill(rows + (i * n) + i + 1, rows + ((i + 1) * n), T {});
2784 }
2785 });
2786 }
2787
2795 void factor_rows_together(std::size_t i,
2796 std::size_t k0)
2797 {
2798 std::array<T, detail::cholesky_diagonal_rows> sums {};
2799 for (std::size_t j {k0}; j < i; ++j) {
2800 const T* const above {lower_.data() + (j * n_)};
2802 for (std::size_t r {0}; r < detail::cholesky_diagonal_rows; ++r) {
2803 sums[r] = lower_[((i + r) * n_) + j];
2804 }
2805 for (std::size_t k {k0}; k < j; ++k) {
2806 const T other {detail::conj_of(above[k])};
2808 for (std::size_t r {0}; r < detail::cholesky_diagonal_rows; ++r) {
2809 sums[r] -= lower_[((i + r) * n_) + k] * other;
2810 }
2811 }
2813 for (std::size_t r {0}; r < detail::cholesky_diagonal_rows; ++r) {
2814 lower_[((i + r) * n_) + j] = detail::over_real_diagonal(sums[r], above[j]);
2815 }
2816 }
2817 }
2818
2824 void factor_row(std::size_t i,
2825 std::size_t k0,
2826 std::size_t first,
2827 std::size_t last)
2828 {
2829 for (std::size_t j {first}; j < last; ++j) {
2830 T sum {lower_[(i * n_) + j]};
2831 for (std::size_t k {k0}; k < j; ++k) {
2832 sum -= lower_[(i * n_) + k] * detail::conj_of(lower_[(j * n_) + k]);
2833 }
2834 if (i == j) {
2835 const detail::carried_t<T> diagonal {detail::real_part(sum)};
2836 if (diagonal <= 0) {
2837 throw not_positive_definite_error("Cholesky: matrix is not positive-definite", i + 1);
2838 }
2839 lower_[(i * n_) + i] = detail::from_carried<T>(std::sqrt(diagonal));
2840 }
2841 else {
2842 lower_[(i * n_) + j] = detail::over_real_diagonal(sum, lower_[(j * n_) + j]);
2843 }
2844 }
2845 }
2846
2859 void factor_rows_below(std::size_t k0,
2860 std::size_t end,
2861 buffer<T>& minus_adjoint)
2862 {
2863 const std::size_t width {end - k0};
2864 const std::size_t chunks {((n_ - end) + detail::cholesky_rows - 1) / detail::cholesky_rows};
2865 detail::parallel_for(chunks, detail::cholesky_rows * width * width / 2, [&](std::size_t chunk) {
2866 const std::size_t first {end + (chunk * detail::cholesky_rows)};
2867 const std::size_t height {std::min(detail::cholesky_rows, n_ - first)};
2868 buffer<T> panel(height * width); // column j at panel[j * height], every element written
2869 for (std::size_t r {0}; r < height; ++r) {
2870 for (std::size_t j {0}; j < width; ++j) {
2871 panel[(j * height) + r] = lower_[((first + r) * n_) + k0 + j];
2872 }
2873 }
2874 constexpr std::size_t sub {16};
2875 buffer<T> minus_conj;
2876 for (std::size_t s0 {0}; s0 < width; s0 += sub) {
2877 const std::size_t s1 {std::min(s0 + sub, width)};
2878 if (s0 > 0) { // the earlier sub-blocks' terms, through the product kernel: Pᵀ[s0, s1) += (−conj L₁₁[s0, s1) × [0, s0)) Pᵀ[0, s0)
2879 minus_conj.resize((s1 - s0) * s0);
2880 for (std::size_t j {s0}; j < s1; ++j) {
2881 for (std::size_t k {0}; k < s0; ++k) {
2882 minus_conj[((j - s0) * s0) + k] = -detail::conj_of(lower_[((k0 + j) * n_) + k0 + k]);
2883 }
2884 }
2885 // The plain loop, which rounds as the tiles do (the same bits), where the packed
2886 // product packed slivers too small to pay for it (six AVX2 cores' one, order 1000:
2887 // this pass 2.82 → 2.23 ms).
2888 detail::gemm_rows(s1 - s0, height, s0, minus_conj.data(), s0, panel.data(), height, panel.data() + (s0 * height), height);
2889 }
2890 for (std::size_t j {s0}; j < s1; ++j) {
2891 T* const column {panel.data() + (j * height)};
2892 detail::subtract_panel_columns(column, panel.data() + (s0 * height), height, lower_.data() + ((k0 + j) * n_) + k0 + s0, j - s0);
2893 const T pivot {lower_[((k0 + j) * n_) + k0 + j]};
2894 for (std::size_t r {0}; r < height; ++r) {
2895 column[r] = detail::over_real_diagonal(column[r], pivot);
2896 }
2897 }
2898 }
2899 const std::size_t below {n_ - end};
2900#ifdef NUMERICAL_X86_COMPILED
2901 // On the x86-64 units the factor's rows, then −L₂₁ᴴ's columns, each written along: written
2902 // together, one of them went across (one AVX2 core, order 1000: this pass 2.23 → 1.97 ms).
2903 // Together on arm64, where apart was 3 % slower under GCC 14 (orders 48 to 128).
2904 for (std::size_t r {0}; r < height; ++r) {
2905 for (std::size_t j {0}; j < width; ++j) {
2906 lower_[((first + r) * n_) + k0 + j] = panel[(j * height) + r];
2907 }
2908 }
2909 for (std::size_t j {0}; j < width; ++j) {
2910 const T* const from {panel.data() + (j * height)};
2911 T* const to {minus_adjoint.data() + (j * below) + (first - end)};
2912 for (std::size_t r {0}; r < height; ++r) {
2913 to[r] = -detail::conj_of(from[r]);
2914 }
2915 }
2916#else
2917 for (std::size_t r {0}; r < height; ++r) {
2918 for (std::size_t j {0}; j < width; ++j) {
2919 const T value {panel[(j * height) + r]};
2920 lower_[((first + r) * n_) + k0 + j] = value;
2921 minus_adjoint[(j * below) + (first - end + r)] = -detail::conj_of(value);
2922 }
2923 }
2924#endif
2925 });
2926 }
2927
2939 void update_trailing(std::size_t k0,
2940 std::size_t end,
2941 const buffer<T>& minus_adjoint,
2942 buffer<T>& packing)
2943 {
2944 const std::size_t rows {n_ - end};
2945 const std::size_t depth {end - k0};
2946#ifdef NUMERICAL_X86_COMPILED
2947 // On the x86-64 units a real product the kernel packs runs on the triangle's tiles alone (detail::gemm_triangle,
2948 // each element the tiles' sum below), where the blocks below computed the part of each diagonal block past the
2949 // diagonal: six AVX2 cores, order 500 787 → 721 µs; a float one on the float tiles, where the blocks' narrow
2950 // kernel held it near the double's time. On arm64 the blocks stay, faster there (all cores, order 500:
2951 // 0.85 ms, by the triangle 1.03: the AMX takes −L₂₁ packed once for every block). A smaller or complex product by
2952 // the blocks.
2953 if constexpr (std::is_same_v<T, double> || std::is_same_v<T, float>) {
2954 if (depth >= detail::gemm_min_depth && rows * rows >= ((detail::gemm_packed_threshold + depth) - 1) / depth) {
2955 detail::gemm_triangle<detail::a_rows, detail::b_rows>(rows, depth, lower_.data() + (end * n_) + k0, n_, minus_adjoint.data(), rows,
2956 lower_.data() + (end * n_) + end, n_, true);
2957 return;
2958 }
2959 }
2960#endif
2961 const std::size_t step {std::max(block_, detail::cholesky_update_columns)};
2962 // Packed once where a block of rows serves several tiles: past two tiles' rows (one AVX2 core, the shared packing
2963 // cost order 300 5 % and 500 3 %, where a block of columns has one or two tiles), else each tile packs its own.
2964 const bool shared {rows > 2 * detail::cholesky_update_rows};
2965 const T* const l21 {lower_.data() + (end * n_) + k0};
2966 const detail::packed_left<T> left {shared ? detail::pack_left(rows, step, depth, l21, n_, packing)
2967 : detail::read_in_place(rows, depth, l21, n_)};
2968 const std::size_t granule {std::max<std::size_t>(1, left.granule)}; // a row at least
2969 const std::size_t height {std::max(granule, detail::cholesky_update_rows / granule * granule)};
2970 std::vector<std::pair<std::size_t, std::size_t>> tiles; // each one's first column and first row, below the panel
2971 for (std::size_t jb {0}; jb < rows; jb += step) {
2972 for (std::size_t r0 {jb / granule * granule}; r0 < rows; r0 += height) {
2973 tiles.emplace_back(jb, r0);
2974 }
2975 }
2976 detail::parallel_for(tiles.size(), height * step * depth / detail::gemm_kernel_speedup, [&](std::size_t t) {
2977 const auto [jb, r0] {tiles[t]};
2978 detail::gemm_with(detail::left_rows(left, r0, std::min(height, rows - r0)), std::min(step, rows - jb),
2979 minus_adjoint.data() + jb, rows, lower_.data() + ((end + r0) * n_) + end + jb, n_);
2980 });
2981 }
2982
2983 std::size_t n_;
2984 std::size_t block_ {detail::cholesky_block_for(n_)};
2986 };
2987
2989 template <field T>
2991 {
2992 return cholesky_decomposition<T> {std::move(a)};
2993 }
2994
2995 namespace detail {
2996
2999 template <typename T>
3000 struct thin_qr
3001 {
3004 };
3005
3020 template <typename T>
3022 std::size_t cols,
3023 const buffer<T>& v,
3024 std::size_t first,
3025 std::size_t from,
3026 const T& s = T {2})
3027 {
3028 constexpr std::size_t width {64};
3029 const std::size_t span {cols - from};
3030 parallel_for((span + width - 1) / width, 2 * v.size() * width, [&](std::size_t chunk) {
3031 const std::size_t start {from + (chunk * width)};
3032 const std::size_t count {std::min(width, cols - start)};
3033 buffer<T> w(count, T {});
3034 for (std::size_t i {0}; i < v.size(); ++i) {
3035 const T weight {conj_of(v[i])};
3036 const T* row {a.data() + ((first + i) * cols) + start};
3037 add_product(w.data(), weight, row, count);
3038 }
3039 for (std::size_t i {0}; i < v.size(); ++i) {
3040 const T scale {s * v[i]};
3041 T* row {a.data() + ((first + i) * cols) + start};
3042 subtract_product(row, scale, w.data(), count);
3043 }
3044 });
3045 }
3046
3048 template <typename T>
3050 {
3051 std::size_t first;
3052 std::size_t height;
3053 std::size_t width;
3056 };
3057
3063 template <typename T>
3064 [[nodiscard]] reflector_block<T> gather_reflectors(const std::vector<buffer<T>>& reflectors,
3065 std::size_t k0,
3066 std::size_t k1,
3067 std::size_t rows,
3068 const std::vector<T>* taus = nullptr)
3069 {
3070 reflector_block<T> block {.first = k0, .height = rows - k0, .width = k1 - k0, .v = {}, .t = {}};
3071 const std::size_t b {block.width};
3072 block.v.assign(block.height * b, T {});
3073 block.t.assign(b * b, T {});
3074 for (std::size_t j {0}; j < b; ++j) {
3075 const buffer<T>& v {reflectors[k0 + j]};
3076 for (std::size_t r {0}; r < v.size(); ++r) {
3077 block.v[((j + r) * b) + j] = v[r];
3078 }
3079 }
3080 // Vᴴ V once, through the matrix product kernel: its column j above the diagonal is V(:, 0…j-1)ᴴ v_j.
3081 buffer<T> v_adjoint(b * block.height, T {});
3082 for (std::size_t r {0}; r < block.height; ++r) {
3083 for (std::size_t j {0}; j < b; ++j) {
3084 v_adjoint[(j * block.height) + r] = conj_of(block.v[(r * b) + j]);
3085 }
3086 }
3087 buffer<T> gram(b * b, T {});
3088 gemm(b, b, block.height, v_adjoint.data(), block.height, block.v.data(), b, gram.data(), b);
3089 for (std::size_t j {0}; j < b; ++j) {
3090 if (reflectors[k0 + j].empty()) {
3091 continue; // the identity: a zero column of T
3092 }
3093 const T tau {taus == nullptr ? static_cast<T>(2) : (*taus)[k0 + j]};
3094 for (std::size_t i {0}; i < j; ++i) { // T(0…j-1, j) = −τ_j T(0…j-1, 0…j-1) z, z = V(:, 0…j-1)ᴴ v_j
3095 T sum {};
3096 for (std::size_t p {i}; p < j; ++p) {
3097 sum += block.t[(i * b) + p] * gram[(p * b) + j];
3098 }
3099 block.t[(i * b) + j] = -tau * sum;
3100 }
3101 block.t[(j * b) + j] = tau;
3102 }
3103 return block;
3104 }
3105
3111 template <typename T>
3113 bool adjoint,
3114 buffer<T>& a,
3115 std::size_t cols,
3116 std::size_t from,
3117 std::size_t to)
3118 {
3119 const std::size_t b {block.width};
3120 const std::size_t h {block.height};
3121 const std::size_t w {to - from};
3122 buffer<T> v_adjoint(b * h, T {});
3123 buffer<T> minus_v(h * b, T {});
3124 for (std::size_t r {0}; r < h; ++r) {
3125 for (std::size_t j {0}; j < b; ++j) {
3126 v_adjoint[(j * h) + r] = conj_of(block.v[(r * b) + j]);
3127 minus_v[(r * b) + j] = -block.v[(r * b) + j];
3128 }
3129 }
3130 T* top {a.data() + (block.first * cols) + from};
3131 buffer<T> projection(b * w, T {});
3132 gemm(b, w, h, v_adjoint.data(), h, top, cols, projection.data(), w);
3133 buffer<T> scaled(b * w, T {});
3134 for (std::size_t i {0}; i < b; ++i) {
3135 for (std::size_t p {0}; p < b; ++p) {
3136 const T factor {adjoint ? conj_of(block.t[(p * b) + i]) : block.t[(i * b) + p]};
3137 if (factor == T {}) {
3138 continue; // T is triangular: half of these are zero
3139 }
3140 add_product(scaled.data() + (i * w), factor, projection.data() + (p * w), w);
3141 }
3142 }
3143 gemm(h, w, b, minus_v.data(), b, scaled.data(), w, top, cols);
3144 }
3145
3154 template <typename T>
3156 buffer<T>& at,
3157 std::size_t cols,
3158 std::size_t from,
3159 std::size_t to,
3160 bool adjoint = false)
3161 {
3162 const std::size_t b {block.width};
3163 const std::size_t h {block.height};
3164 const std::size_t r {to - from};
3165 const auto v_conjugate {std::make_unique_for_overwrite<T[]>(h * b)}; // conj(V), h × b, written whole
3166 const auto minus_v_t {std::make_unique_for_overwrite<T[]>(b * h)}; // −Vᵀ, b × h
3167 const auto t_transposed {std::make_unique_for_overwrite<T[]>(b * b)}; // Tᵀ, or conj(T)
3168 for (std::size_t row {0}; row < h; ++row) {
3169 for (std::size_t j {0}; j < b; ++j) {
3170 v_conjugate[(row * b) + j] = conj_of(block.v[(row * b) + j]);
3171 minus_v_t[(j * h) + row] = -block.v[(row * b) + j];
3172 }
3173 }
3174 for (std::size_t i {0}; i < b; ++i) {
3175 for (std::size_t p {0}; p < b; ++p) {
3176 t_transposed[(p * b) + i] = adjoint ? conj_of(block.t[(p * b) + i]) : block.t[(i * b) + p];
3177 }
3178 }
3179 T* left {at.data() + (from * cols) + block.first};
3180 buffer<T> projection(r * b, T {});
3181 gemm(r, b, h, left, cols, v_conjugate.get(), b, projection.data(), b);
3182 buffer<T> scaled(r * b, T {});
3183 gemm(r, b, b, projection.data(), b, t_transposed.get(), b, scaled.data(), b);
3184 gemm(r, h, b, scaled.data(), b, minus_v_t.get(), h, left, cols);
3185 }
3186
3188 template <typename T>
3189 requires std::is_floating_point_v<T>
3190 [[nodiscard]] T conjugate_dot(const T* x,
3191 const T* y,
3192 std::size_t count)
3193 {
3194 return dot_product(x, y, count);
3195 }
3196
3198 template <typename T>
3199 [[nodiscard]] T conjugate_dot(const T* x,
3200 const T* y,
3201 std::size_t count)
3202 {
3203 return conjugate_dot_product(x, y, count);
3204 }
3205
3213 template <typename T>
3214 [[nodiscard]] std::pair<T, T> larfg_in_place(buffer<T>& w)
3215 {
3216 const carried_t<T> factor {rescale_tiny(w)}; // w, τ and the signs the same at any scale; β scaled back
3217 const T alpha {w[0]};
3218 const carried_t<T> beta {-std::copysign(scaled_norm(w), detail::real_part(alpha))};
3219 const T tau {(detail::from_carried<T>(beta) - alpha) / detail::from_carried<T>(beta)};
3220 const T scale {T {1} / (alpha - detail::from_carried<T>(beta))}; // |α − β| ≥ ‖x‖: no cancellation
3221 w[0] = T {1};
3222 for (std::size_t r {1}; r < w.size(); ++r) {
3223 w[r] *= scale;
3224 }
3225 return {detail::from_carried<T>(beta / factor), tau};
3226 }
3227
3233 template <typename T>
3234 void reflect_columns(T* panel,
3235 std::size_t height,
3236 const buffer<T>& v,
3237 std::size_t first,
3238 std::size_t from,
3239 std::size_t to,
3240 const T& s = T {2})
3241 {
3242 for (std::size_t j {from}; j < to; ++j) {
3243 T* const column {panel + (j * height) + first};
3244 const T scaled {s * conjugate_dot(v.data(), column, v.size())};
3245 subtract_product(column, scaled, v.data(), v.size());
3246 }
3247 }
3248
3250 inline constexpr std::size_t qr_block {32};
3251
3265 template <typename T>
3266 void factor_qr_panel(T* panel,
3267 std::size_t ld,
3268 std::size_t top,
3269 std::vector<buffer<T>>& reflectors,
3270 std::vector<T>& taus,
3271 std::size_t k0,
3272 std::size_t k1)
3273 {
3274 for (std::size_t k {k0}; k < k1; ++k) {
3275 const std::size_t c {k - k0};
3276 T* const x {panel + (c * ld) + (k - top)};
3277 buffer<T> w(x, x + (ld + top - k));
3278 const T alpha {w[0]};
3279 if (scaled_norm(w.data() + 1, w.size() - 1) == 0 && alpha == detail::from_carried<T>(detail::real_part(alpha))) {
3280 continue; // the identity: nothing below the diagonal, and the diagonal real
3281 }
3282 const auto [beta, tau] {larfg_in_place(w)};
3283 reflect_columns(panel, ld, w, k - top, c + 1, k1 - k0, conj_of(tau)); // Hᴴ on the panel's columns past k
3284 x[0] = beta;
3285 std::copy(w.begin() + 1, w.end(), x + 1);
3286 reflectors[k] = std::move(w);
3287 taus[k] = tau;
3288 }
3289 }
3290
3292 template <typename T>
3294 {
3296 std::vector<reflector_block<T>> blocks;
3297 };
3298
3305 inline constexpr std::size_t qr_look_ahead_threads {3};
3306
3310 inline constexpr std::size_t qr_update_slice {32};
3311
3322 template <typename T>
3323 [[nodiscard]] std::vector<reflector_block<T>> reflect_in_columns(buffer<T>& columns,
3324 std::size_t rows,
3325 std::size_t cols,
3326 bool reflectors = true)
3327 {
3328 const std::size_t depth {std::min(rows, cols)};
3329 std::vector<buffer<T>> reflections(depth); // w_k on rows k…m-1; empty for the identity
3330 std::vector<T> taus(depth, T {});
3331 std::vector<reflector_block<T>> blocks;
3332 if (depth == 0) {
3333 return blocks;
3334 }
3335 factor_qr_panel(columns.data(), rows, 0, reflections, taus, 0, std::min(qr_block, depth));
3336 if (reflectors || std::min(qr_block, depth) < cols) { // a block no update reads, where R alone is asked: not gathered
3337 blocks.push_back(gather_reflectors(reflections, 0, std::min(qr_block, depth), rows, &taus));
3338 }
3339 for (std::size_t k1 {std::min(qr_block, depth)}; k1 < cols;) { // the panels left of k1 factorised, the last not applied
3340 if (k1 >= depth) { // a wide matrix: its last block on the columns past every panel
3341 apply_transposed_reflector_block(blocks.back(), columns, rows, k1, cols, true);
3342 break;
3343 }
3344 const std::size_t n1 {std::min(k1 + qr_block, depth)}; // the next panel: columns k1…n1-1
3345 const std::size_t threads {max_threads()};
3346 if (threads < qr_look_ahead_threads) { // right of the panel: H_{k1-1} … H_{k0} A = (I − V T Vᴴ)ᴴ A, then the next
3347 apply_transposed_reflector_block(blocks.back(), columns, rows, k1, cols, true);
3348 factor_qr_panel(columns.data() + (k1 * rows), rows, 0, reflections, taus, k1, n1);
3349 if (reflectors || n1 < cols) {
3350 blocks.push_back(gather_reflectors(reflections, k1, n1, rows, &taus));
3351 }
3352 k1 = n1;
3353 continue;
3354 }
3355 // Look-ahead: one task updates the next panel's columns and factorises them while the others update the rest,
3356 // a slice of columns each (the same products on fewer columns: the same bits).
3357 const std::size_t rest {cols - n1};
3358 const std::size_t slice {std::max(qr_update_slice, (rest + threads - 2) / (threads - 1))};
3359 const std::size_t slices {(rest + slice - 1) / slice};
3360 const reflector_block<T>& block {blocks.back()};
3361 reflector_block<T> next {};
3362 parallel_for(1 + slices, rows * slice * qr_block / 4, [&](std::size_t task) {
3363 if (task == 0) {
3364 apply_transposed_reflector_block(block, columns, rows, k1, n1, true);
3365 factor_qr_panel(columns.data() + (k1 * rows), rows, 0, reflections, taus, k1, n1);
3366 if (reflectors || n1 < cols) {
3367 next = gather_reflectors(reflections, k1, n1, rows, &taus);
3368 }
3369 return;
3370 }
3371 const std::size_t from {n1 + ((task - 1) * slice)};
3372 apply_transposed_reflector_block(block, columns, rows, from, std::min(from + slice, cols), true);
3373 });
3374 if (reflectors || n1 < cols) {
3375 blocks.push_back(std::move(next));
3376 }
3377 k1 = n1;
3378 }
3379 return blocks;
3380 }
3381
3384 inline constexpr std::size_t qr_transpose_tile {32};
3385
3388 template <typename T>
3389 void transpose_into(const T* from,
3390 std::size_t rows,
3391 std::size_t cols,
3392 T* to)
3393 {
3394 parallel_for((rows + qr_transpose_tile - 1) / qr_transpose_tile, memory_element_weight * qr_transpose_tile * cols, [from, rows, cols, to](std::size_t band) {
3395 const std::size_t i0 {band * qr_transpose_tile};
3396 for (std::size_t j0 {0}; j0 < cols; j0 += qr_transpose_tile) {
3397 for (std::size_t j {j0}; j < std::min(j0 + qr_transpose_tile, cols); ++j) {
3398 for (std::size_t i {i0}; i < std::min(i0 + qr_transpose_tile, rows); ++i) {
3399 to[(j * rows) + i] = from[(i * cols) + j];
3400 }
3401 }
3402 }
3403 });
3404 }
3405
3411 template <typename T>
3413 std::size_t rows,
3414 std::size_t cols,
3415 std::vector<buffer<T>>& reflectors,
3416 std::vector<T>& taus,
3417 std::size_t k0,
3418 std::size_t k1)
3419 {
3420 const std::size_t height {rows - k0};
3421 const std::size_t width {k1 - k0};
3422 buffer<T> panel(height * width, T {});
3423 for (std::size_t i {0}; i < height; ++i) {
3424 for (std::size_t j {0}; j < width; ++j) {
3425 panel[(j * height) + i] = a[((k0 + i) * cols) + k0 + j];
3426 }
3427 }
3428 factor_qr_panel(panel.data(), height, k0, reflectors, taus, k0, k1);
3429 for (std::size_t i {0}; i < height; ++i) {
3430 for (std::size_t j {0}; j < width; ++j) {
3431 a[((k0 + i) * cols) + k0 + j] = panel[(j * height) + i];
3432 }
3433 }
3434 }
3435
3445 template <typename T>
3446 [[nodiscard]] std::vector<reflector_block<T>> reflect_in_rows(buffer<T>& a,
3447 std::size_t rows,
3448 std::size_t cols,
3449 buffer<T>* r = nullptr,
3450 bool reflectors = true)
3451 {
3452 const std::size_t depth {std::min(rows, cols)};
3453 std::vector<buffer<T>> reflections(depth); // w_k on rows k…m-1; empty for the identity
3454 std::vector<T> taus(depth, T {});
3455 if (r != nullptr) {
3456 *r = buffer<T>(depth * cols, T {}); // constructed, not assigned: the generic fill of assign took 4 % of qr at order 128
3457 }
3458 std::vector<reflector_block<T>> blocks;
3459 if (depth > 0) {
3460 factor_row_major_panel(a, rows, cols, reflections, taus, 0, std::min(qr_block, depth));
3461 if (reflectors || std::min(qr_block, depth) < cols) { // as reflect_in_columns
3462 blocks.push_back(gather_reflectors(reflections, 0, std::min(qr_block, depth), rows, &taus));
3463 }
3464 }
3465 for (std::size_t k1 {std::min(qr_block, depth)}; depth > 0 && k1 < cols;) { // as reflect_in_columns (none if empty)
3466 if (k1 >= depth) {
3467 apply_reflector_block(blocks.back(), true, a, cols, k1, cols);
3468 break;
3469 }
3470 const std::size_t n1 {std::min(k1 + qr_block, depth)};
3471 const std::size_t threads {max_threads()};
3472 if (threads < qr_look_ahead_threads) {
3473 apply_reflector_block(blocks.back(), true, a, cols, k1, cols);
3474 factor_row_major_panel(a, rows, cols, reflections, taus, k1, n1);
3475 if (reflectors || n1 < cols) {
3476 blocks.push_back(gather_reflectors(reflections, k1, n1, rows, &taus));
3477 }
3478 k1 = n1;
3479 continue;
3480 }
3481 const std::size_t rest {cols - n1};
3482 const std::size_t slice {std::max(qr_update_slice, (rest + threads - 2) / (threads - 1))};
3483 const std::size_t slices {(rest + slice - 1) / slice};
3484 const reflector_block<T>& block {blocks.back()};
3485 reflector_block<T> next {};
3486 parallel_for(1 + slices, rows * slice * qr_block / 4, [&](std::size_t task) {
3487 if (task == 0) {
3488 apply_reflector_block(block, true, a, cols, k1, n1);
3489 factor_row_major_panel(a, rows, cols, reflections, taus, k1, n1);
3490 if (reflectors || n1 < cols) {
3491 next = gather_reflectors(reflections, k1, n1, rows, &taus);
3492 }
3493 return;
3494 }
3495 const std::size_t from {n1 + ((task - 1) * slice)};
3496 apply_reflector_block(block, true, a, cols, from, std::min(from + slice, cols));
3497 });
3498 if (reflectors || n1 < cols) {
3499 blocks.push_back(std::move(next));
3500 }
3501 k1 = n1;
3502 }
3503 for (std::size_t i {0}; r != nullptr && i < depth; ++i) {
3504 for (std::size_t j {i}; j < cols; ++j) {
3505 (*r)[(i * cols) + j] = a[(i * cols) + j];
3506 }
3507 }
3508 return blocks;
3509 }
3510
3520 template <typename T>
3522 std::size_t rows,
3523 std::size_t cols,
3524 bool reflectors = true)
3525 {
3526#ifdef NUMERICAL_X86_COMPILED
3527 const std::size_t depth {std::min(rows, cols)};
3528 buffer<T> columns(rows * cols);
3529 transpose_into(a.data(), rows, cols, columns.data());
3530 qr_reflections<T> out {.r = buffer<T>(depth * cols), .blocks = reflect_in_columns(columns, rows, cols, reflectors)};
3531 for (std::size_t j0 {0}; j0 < cols; j0 += qr_transpose_tile) { // R, row-major: written whole, zeros below
3532 for (std::size_t i0 {0}; i0 < depth; i0 += qr_transpose_tile) {
3533 for (std::size_t i {i0}; i < std::min(i0 + qr_transpose_tile, depth); ++i) {
3534 for (std::size_t j {j0}; j < std::min(j0 + qr_transpose_tile, cols); ++j) {
3535 out.r[(i * cols) + j] = i <= j ? columns[(j * rows) + i] : T {};
3536 }
3537 }
3538 }
3539 }
3540#else
3542 out.blocks = reflect_in_rows(a, rows, cols, &out.r, reflectors);
3543#endif
3544 return out;
3545 }
3546
3555 template <typename T>
3557 const std::vector<reflector_block<T>>& blocks,
3558 std::size_t rows,
3559 std::size_t first,
3560 std::size_t count,
3561 bool transposed)
3562 {
3563#ifdef NUMERICAL_X86_COMPILED
3564 buffer<T> formed;
3565 buffer<T>& qt {transposed ? out : formed};
3566 qt = buffer<T>(count * rows, T {});
3567 for (std::size_t c {0}; c < count; ++c) {
3568 qt[(c * rows) + first + c] = T {1};
3569 }
3570 for (std::size_t b {blocks.size()}; b-- > 0;) {
3571 apply_transposed_reflector_block(blocks[b], qt, rows, std::max(first, blocks[b].first) - first, count);
3572 }
3573 if (!transposed) {
3574 out.resize(rows * count);
3575 transpose_into(qt.data(), count, rows, out.data());
3576 }
3577#else
3578 buffer<T> formed;
3579 buffer<T>& q {transposed ? formed : out};
3580 q = buffer<T>(rows * count, T {});
3581 for (std::size_t c {0}; c < count; ++c) {
3582 q[((first + c) * count) + c] = T {1};
3583 }
3584 for (std::size_t b {blocks.size()}; b-- > 0;) {
3585 apply_reflector_block(blocks[b], false, q, count, std::max(first, blocks[b].first) - first, count);
3586 }
3587 if (transposed) {
3588 out.resize(count * rows);
3589 transpose_into(q.data(), rows, count, out.data());
3590 }
3591#endif
3592 }
3593
3600 template <typename T>
3602 const std::vector<reflector_block<T>>& blocks,
3603 [[maybe_unused]] std::size_t rows,
3604 std::size_t count)
3605 {
3606#ifdef NUMERICAL_X86_COMPILED
3607 buffer<T> pt(count * rows);
3608 transpose_into(p.data(), rows, count, pt.data());
3609 for (std::size_t b {blocks.size()}; b-- > 0;) {
3610 apply_transposed_reflector_block(blocks[b], pt, rows, 0, count);
3611 }
3612 transpose_into(pt.data(), count, rows, p.data());
3613#else
3614 for (std::size_t b {blocks.size()}; b-- > 0;) {
3615 apply_reflector_block(blocks[b], false, p, count, 0, count);
3616 }
3617#endif
3618 }
3619
3637 template <typename T>
3639 std::size_t rows,
3640 std::size_t cols,
3641 bool complete = false)
3642 {
3643 const std::size_t depth {std::min(rows, cols)};
3644 const std::size_t width {complete ? rows : depth};
3645 qr_reflections<T> factors {householder_reflect(std::move(a), rows, cols)};
3646 // Q = H₀ H₁ … applied to the columns [first, first + count) of I, each column's own product: the first `depth`
3647 // formed alone whether Q is complete or not, so that the complete Q's are the reduced one's to the bit (formed
3648 // together, the products' tiles differ, and a fused multiply-add may round a column otherwise).
3649 const auto formed {[&](std::size_t first, std::size_t count) {
3650 buffer<T> columns;
3651 form_q_into(columns, factors.blocks, rows, first, count, false);
3652 return columns;
3653 }};
3654 thin_qr<T> out {.q = formed(0, depth), .r = std::move(factors.r)};
3655 if (width > depth) {
3656 const buffer<T> rest {formed(depth, width - depth)};
3657 buffer<T> whole(rows * width, T {});
3658 for (std::size_t i {0}; i < rows; ++i) {
3659 std::copy_n(out.q.data() + (i * depth), depth, whole.data() + (i * width));
3660 std::copy_n(rest.data() + (i * (width - depth)), width - depth, whole.data() + (i * width) + depth);
3661 }
3662 out.q = std::move(whole);
3663 }
3664 for (std::size_t k {0}; k < depth; ++k) { // R's diagonal real and non-negative: rotate row k, column k
3665 const T phase {phase_of(out.r[(k * cols) + k])};
3666 for (std::size_t j {k}; j < cols; ++j) {
3667 out.r[(k * cols) + j] *= conj_of(phase);
3668 }
3669 for (std::size_t i {0}; i < rows; ++i) {
3670 out.q[(i * width) + k] *= phase;
3671 }
3672 }
3673 return out;
3674 }
3675
3685 template <typename T>
3687 std::size_t rows,
3688 std::size_t cols)
3689 {
3690 const std::size_t depth {std::min(rows, cols)};
3691 buffer<T> r {householder_reflect(std::move(a), rows, cols, false).r};
3692 for (std::size_t k {0}; k < depth; ++k) { // as householder_qr rotates R's row k
3693 const T phase {phase_of(r[(k * cols) + k])};
3694 for (std::size_t j {k}; j < cols; ++j) {
3695 r[(k * cols) + j] *= conj_of(phase);
3696 }
3697 }
3698 return r;
3699 }
3700
3703 [[nodiscard]] inline double rank_tolerance(double largest,
3704 std::size_t rows,
3705 std::size_t cols)
3706 {
3707 return largest * std::numeric_limits<double>::epsilon() * static_cast<double>(std::max(rows, cols));
3708 }
3709 } // namespace detail
3710
3718 {
3719 public:
3720
3722 constexpr rank_cutoff() noexcept = default;
3723
3728 [[nodiscard]] static rank_cutoff relative(double ratio)
3729 {
3730 return rank_cutoff {checked(ratio), false};
3731 }
3732
3737 [[nodiscard]] static rank_cutoff absolute(double magnitude)
3738 {
3739 return rank_cutoff {checked(magnitude), true};
3740 }
3741
3743 [[nodiscard]] double magnitude(double largest,
3744 std::size_t rows,
3745 std::size_t cols) const noexcept
3746 {
3747 if (absolute_) {
3748 return value_;
3749 }
3750 return value_ < 0.0 ? detail::rank_tolerance(largest, rows, cols) : largest * value_;
3751 }
3752
3754 [[nodiscard]] rank_cutoff scaled(int exponent) const noexcept
3755 {
3756 return absolute_ ? rank_cutoff {std::ldexp(value_, -exponent), true} : *this;
3757 }
3758
3759 private:
3760
3761 constexpr rank_cutoff(double value,
3762 bool absolute) noexcept
3763 : value_(value), absolute_(absolute)
3764 {}
3765
3766 [[nodiscard]] static double checked(double value)
3767 {
3768 if (!(value >= 0.0)) {
3769 throw argument_error("rank cutoff: a tolerance is non-negative");
3770 }
3771 return value;
3772 }
3773
3774 double value_ {-1.0};
3775 bool absolute_ {false};
3776 };
3777
3779 enum class qr_mode
3780 {
3781 reduced,
3782 complete
3783 };
3784
3797 template <field T>
3799 {
3800 public:
3801
3807 explicit qr_decomposition(const matrix<T>& a,
3808 qr_mode mode = qr_mode::reduced)
3809 : qr_decomposition(detail::parallel_copy_of(a),
3810 mode)
3811 {}
3812
3820 qr_mode mode = qr_mode::reduced)
3821 : rows_(a.rows()), cols_(a.cols()), depth_(std::min(rows_, cols_)),
3822 width_(mode == qr_mode::complete ? rows_ : depth_)
3823 {
3824 detail::require_finite(a, "QR decomposition");
3825 detail::thin_qr<T> factors {detail::householder_qr(std::move(a).elements(), rows_, cols_, mode == qr_mode::complete)};
3826 q_ = std::move(factors.q);
3827 r_ = std::move(factors.r);
3828 r_.resize(width_ * cols_, T {}); // complete: R's rows past k are zero
3829 }
3830
3832 [[nodiscard]] matrix<T> q() const&
3833 {
3834 return matrix<T> {rows_, width_, q_};
3835 }
3836
3838 [[nodiscard]] matrix<T> q() &&
3839 {
3840 return matrix<T> {rows_, width_, std::move(q_)};
3841 }
3842
3844 [[nodiscard]] matrix<T> r() const&
3845 {
3846 return matrix<T> {width_, cols_, r_};
3847 }
3848
3850 [[nodiscard]] matrix<T> r() &&
3851 {
3852 return matrix<T> {width_, cols_, std::move(r_)};
3853 }
3854
3864 [[nodiscard]] vector<T> solve(const vector<T>& b) const
3865 {
3866 if (b.size() != rows_) {
3867 throw dimension_error("solve: right-hand side size differs from the matrix");
3868 }
3869 if (rows_ < cols_) {
3870 throw dimension_error("QR solve: the system is underdetermined (lstsq gives its minimum-norm solution)");
3871 }
3872 double largest {0.0};
3873 for (std::size_t i {0}; i < cols_; ++i) {
3874 largest = std::max(largest, static_cast<double>(detail::real_part(r_[(i * cols_) + i])));
3875 }
3876 const double tolerance {detail::rank_tolerance(largest, rows_, cols_)};
3877 buffer<T> y(cols_, T {}); // y = Qᴴ b
3878 for (std::size_t i {0}; i < cols_; ++i) {
3879 if (detail::real_part(r_[(i * cols_) + i]) <= tolerance) {
3880 throw singular_matrix_error("QR solve: the matrix is rank-deficient (lstsq gives its minimum-norm solution)");
3881 }
3882 T sum {};
3883 for (std::size_t k {0}; k < rows_; ++k) {
3884 sum += detail::conj_of(q_[(k * width_) + i]) * b[k];
3885 }
3886 y[i] = sum;
3887 }
3888 buffer<T> x(cols_, T {}); // back substitution R x = y
3889 for (std::size_t i {cols_}; i-- > 0;) {
3890 T sum {y[i]};
3891 for (std::size_t j {i + 1}; j < cols_; ++j) {
3892 sum -= r_[(i * cols_) + j] * x[j];
3893 }
3894 x[i] = sum / r_[(i * cols_) + i];
3895 }
3896 return vector<T> {std::move(x)};
3897 }
3898
3908 [[nodiscard]] matrix<T> solve(const matrix<T>& b) const
3909 {
3910 if (b.rows() != rows_) {
3911 throw dimension_error("solve: right-hand side row count differs from the matrix");
3912 }
3913 if (rows_ < cols_) {
3914 throw dimension_error("QR solve: the system is underdetermined (lstsq gives its minimum-norm solution)");
3915 }
3916 double largest {0.0};
3917 for (std::size_t i {0}; i < cols_; ++i) {
3918 largest = std::max(largest, static_cast<double>(detail::real_part(r_[(i * cols_) + i])));
3919 }
3920 const double tolerance {detail::rank_tolerance(largest, rows_, cols_)};
3921 for (std::size_t i {0}; i < cols_; ++i) {
3922 if (detail::real_part(r_[(i * cols_) + i]) <= tolerance) {
3923 throw singular_matrix_error("QR solve: the matrix is rank-deficient (lstsq gives its minimum-norm solution)");
3924 }
3925 }
3926 const std::size_t sides {b.cols()};
3927 buffer<T> q_adjoint(cols_ * rows_); // the first n columns of Q, conjugated, as rows
3928 for (std::size_t k {0}; k < rows_; ++k) {
3929 for (std::size_t i {0}; i < cols_; ++i) {
3930 q_adjoint[(i * rows_) + k] = detail::conj_of(q_[(k * width_) + i]);
3931 }
3932 }
3933 buffer<T> x(cols_ * sides); // Y = Qᴴ B, then X in its place
3934 gemm(cols_, sides, rows_, q_adjoint.data(), rows_, std::to_address(b.begin()), sides, x.data(), sides, detail::c_start::zero);
3935 for (std::size_t i {cols_}; i-- > 0;) { // back substitution R X = Y, a row of R's terms at a time
3936 T* const row {x.data() + (i * sides)};
3937 for (std::size_t j {i + 1}; j < cols_; ++j) {
3938 const T r {r_[(i * cols_) + j]};
3939 const T* const below {x.data() + (j * sides)};
3940 for (std::size_t c {0}; c < sides; ++c) {
3941 row[c] -= r * below[c];
3942 }
3943 }
3944 for (std::size_t c {0}; c < sides; ++c) {
3945 row[c] /= r_[(i * cols_) + i];
3946 }
3947 }
3948 return matrix<T> {cols_, sides, std::move(x)};
3949 }
3950
3951 private:
3952
3953 std::size_t rows_;
3954 std::size_t cols_;
3955 std::size_t depth_;
3956 std::size_t width_;
3959 };
3960
3962 template <typename T>
3963 [[nodiscard]] qr_decomposition<T> qr(const matrix<T>& a)
3964 {
3965 return qr_decomposition<T> {a};
3966 }
3967
3968 namespace detail {
3969
3971 [[nodiscard]] inline double adjoint_dot(const double* x,
3972 const double* y,
3973 std::size_t count)
3974 {
3975 return dot_product(x, y, count);
3976 }
3977
3979 [[nodiscard]] inline std::complex<double> adjoint_dot(const std::complex<double>* x,
3980 const std::complex<double>* y,
3981 std::size_t count)
3982 {
3983 return conjugate_dot_product(x, y, count);
3984 }
3985
3987 template <typename T>
3988 [[nodiscard]] double contiguous_norm(const T* x,
3989 std::size_t count)
3990 {
3991 double largest {0.0};
3992 for (std::size_t i {0}; i < count; ++i) {
3993 largest = std::max(largest, std::abs(x[i]));
3994 }
3995 if (largest == 0.0) {
3996 return 0.0;
3997 }
3998 double sum {0.0};
3999 for (std::size_t i {0}; i < count; ++i) {
4000 const double scaled {std::abs(x[i]) / largest};
4001 sum += scaled * scaled;
4002 }
4003 return largest * std::sqrt(sum);
4004 }
4005 } // namespace detail
4006
4015 template <double_field T>
4017 {
4018 public:
4019
4026 qr_mode mode = qr_mode::reduced)
4027 {
4028 const std::size_t m {a.rows()};
4029 const std::size_t n {a.cols()};
4030 const std::size_t k {std::min(m, n)};
4031 detail::require_finite(a, "pivoted QR");
4032 buffer<T> w(m * n); // by columns, each contiguous: the steps' sums, updates and exchanges go down them
4033 for (std::size_t i {0}; i < m; ++i) {
4034 for (std::size_t j {0}; j < n; ++j) {
4035 w[(j * m) + i] = a(i, j);
4036 }
4037 }
4038 buffer<double> current(n);
4039 buffer<double> reference(n);
4040 permutation_.resize(n);
4041 for (std::size_t j {0}; j < n; ++j) {
4042 permutation_[j] = j;
4043 current[j] = detail::contiguous_norm(w.data() + (j * m), m);
4044 reference[j] = current[j];
4045 }
4046 const double tolerance {std::sqrt(std::numeric_limits<double>::epsilon())};
4047 std::vector<buffer<T>> reflectors(k);
4048 std::vector<T> taus(k);
4049 const std::size_t top {k > pivoted_qr_crossover ? k - pivoted_qr_crossover : 0};
4050 std::size_t start {0};
4051 while (start < top) {
4052 start += factor_block(w, m, n, start, std::min(pivoted_qr_block, top - start), current, reference, reflectors, taus);
4053 }
4054 for (std::size_t step {start}; step < k; ++step) {
4055 const std::size_t pivot {static_cast<std::size_t>(std::max_element(current.begin() + static_cast<std::ptrdiff_t>(step), current.end()) - current.begin())};
4056 T* const head {w.data() + (step * m)};
4057 if (pivot != step) {
4058 std::swap_ranges(head, head + m, w.data() + (pivot * m));
4059 std::swap(permutation_[step], permutation_[pivot]);
4060 std::swap(current[step], current[pivot]);
4061 std::swap(reference[step], reference[pivot]);
4062 }
4063 buffer<T> v(head + step, head + m);
4064 // The identity where nothing lies below the diagonal and the diagonal is real (a zero column among them), as
4065 // the QR's panel takes it: larfg would divide by a zero β.
4066 const T alpha {v[0]};
4067 const bool identity {detail::scaled_norm(v.data() + 1, v.size() - 1) == 0 && alpha == detail::from_carried<T>(detail::real_part(alpha))};
4068 T tau {};
4069 if (identity) {
4070 std::fill(v.begin(), v.end(), T {});
4071 v[0] = T {1};
4072 }
4073 else {
4074 const auto [beta, factor] {detail::larfg_in_place(v)};
4075 tau = factor;
4076 head[step] = beta;
4077 std::fill(head + step + 1, head + m, T {});
4078 }
4079 for (std::size_t j {step + 1}; j < n; ++j) { // Hᴴ = I − τ̄ v vᴴ on the columns right of the pivot
4080 T* const column {w.data() + (j * m)};
4081 const T dot {detail::adjoint_dot(v.data(), column + step, m - step)};
4082 detail::subtract_product(column + step, detail::conj_of(tau) * dot, v.data(), m - step);
4083 if (current[j] != 0.0) { // LAPACK's downdate of the norm by the entry the reflection took into row step
4084 const double ratio {std::abs(column[step]) / current[j]};
4085 const double left {std::max(0.0, (1.0 + ratio) * (1.0 - ratio))};
4086 const double drift {left * (current[j] / reference[j]) * (current[j] / reference[j])};
4087 if (drift <= tolerance) {
4088 current[j] = detail::contiguous_norm(column + step + 1, m - step - 1);
4089 reference[j] = current[j];
4090 }
4091 else {
4092 current[j] *= std::sqrt(left);
4093 }
4094 }
4095 }
4096 reflectors[step] = std::move(v);
4097 taus[step] = tau;
4098 }
4099 const std::size_t rows {mode == qr_mode::complete ? m : k};
4100 buffer<T> r(rows * n, T {});
4101 for (std::size_t i {0}; i < rows; ++i) {
4102 for (std::size_t j {0}; j < n; ++j) {
4103 r[(i * n) + j] = w[(j * m) + i];
4104 }
4105 }
4106 std::vector<detail::reflector_block<T>> blocks; // Q = H₀ H₁ …, formed by blocks as the QR's
4107 for (std::size_t k0 {0}; k0 < k; k0 += pivoted_qr_block) {
4108 blocks.push_back(detail::gather_reflectors(reflectors, k0, std::min(k0 + pivoted_qr_block, k), m, &taus));
4109 }
4110 buffer<T> q;
4111 detail::form_q_into(q, blocks, m, 0, rows, false);
4112 for (std::size_t i {0}; i < k; ++i) { // R's diagonal real and non-negative, Q's column i taking the phase
4113 const double modulus {std::abs(r[(i * n) + i])};
4114 if (modulus == 0.0) {
4115 continue;
4116 }
4117 const T phase {r[(i * n) + i] / T {modulus}};
4118 for (std::size_t j {i}; j < n; ++j) {
4119 r[(i * n) + j] *= detail::conj_of(phase);
4120 }
4121 r[(i * n) + i] = T {modulus};
4122 for (std::size_t row {0}; row < m; ++row) {
4123 q[(row * rows) + i] *= phase;
4124 }
4125 }
4126 q_ = matrix<T> {m, rows, std::move(q)};
4127 r_ = matrix<T> {rows, n, std::move(r)};
4128 }
4129
4131 [[nodiscard]] const matrix<T>& q() const noexcept
4132 {
4133 return q_;
4134 }
4135
4137 [[nodiscard]] const matrix<T>& r() const noexcept
4138 {
4139 return r_;
4140 }
4141
4143 [[nodiscard]] const std::vector<std::size_t>& permutation() const noexcept
4144 {
4145 return permutation_;
4146 }
4147
4148 private:
4149
4150 static constexpr std::size_t pivoted_qr_block {32};
4151 static constexpr std::size_t pivoted_qr_crossover {128};
4152 static constexpr std::size_t pivoted_qr_band {16};
4153
4161 std::size_t factor_block(buffer<T>& w,
4162 std::size_t m,
4163 std::size_t n,
4164 std::size_t start,
4165 std::size_t width,
4166 buffer<double>& current,
4167 buffer<double>& reference,
4168 std::vector<buffer<T>>& reflectors,
4169 std::vector<T>& taus)
4170 {
4171 const double tolerance {std::sqrt(std::numeric_limits<double>::epsilon())};
4172 const std::size_t cols {n - start};
4173 buffer<T> f(cols * pivoted_qr_block, T {}); // F by rows: F(j, t) at f[j·block + t], j from start
4174 buffer<T> coefficients(pivoted_qr_block);
4175 std::vector<std::size_t> deferred;
4176 std::size_t done {0};
4177 const auto row_of_f = [&f](std::size_t j) { return f.data() + (j * pivoted_qr_block); };
4178 while (done < width && deferred.empty()) {
4179 const std::size_t kk {done};
4180 const std::size_t step {start + kk};
4181 const std::size_t pivot {static_cast<std::size_t>(std::max_element(current.begin() + static_cast<std::ptrdiff_t>(step), current.end()) - current.begin())};
4182 T* const head {w.data() + (step * m)};
4183 if (pivot != step) {
4184 std::swap_ranges(head, head + m, w.data() + (pivot * m));
4185 std::swap_ranges(row_of_f(kk), row_of_f(kk) + kk, row_of_f(pivot - start));
4186 std::swap(permutation_[step], permutation_[pivot]);
4187 std::swap(current[step], current[pivot]);
4188 std::swap(reference[step], reference[pivot]);
4189 }
4190 if (kk > 0) { // the column's share of the earlier reflections: A(step:, step) −= V(step:, ·) F(kk, ·)ᴴ
4191 for (std::size_t t {0}; t < kk; ++t) {
4192 coefficients[t] = detail::conj_of(row_of_f(kk)[t]);
4193 }
4194 detail::subtract_products(head + step, coefficients.data(), w.data() + (start * m) + step, m, kk, m - step);
4195 }
4196 buffer<T> v(head + step, head + m);
4197 const T alpha {v[0]};
4198 const bool identity {detail::scaled_norm(v.data() + 1, v.size() - 1) == 0 && alpha == detail::from_carried<T>(detail::real_part(alpha))};
4199 T tau {};
4200 T diagonal {alpha};
4201 if (identity) {
4202 std::fill(v.begin(), v.end(), T {});
4203 v[0] = T {1};
4204 }
4205 else {
4206 const auto [beta, factor] {detail::larfg_in_place(v)};
4207 tau = factor;
4208 diagonal = beta;
4209 }
4210 std::copy(v.begin(), v.end(), head + step); // v in place (its 1 on the diagonal) while the block lasts
4211 const auto conjugate_dot = [&v, m, step](const T* column) { // A(step:, j)ᴴ v, vectorised
4212 return detail::adjoint_dot(column + step, v.data(), m - step);
4213 };
4214 // F(j, kk) = τ A(step:, j)ᴴ v, each column its own dot product: by bands of columns on the threads
4215 const std::size_t later {cols - (kk + 1)};
4216 detail::parallel_for((later + pivoted_qr_band - 1) / pivoted_qr_band, (m - step) * pivoted_qr_band, [&](std::size_t band) {
4217 const std::size_t first {kk + 1 + (band * pivoted_qr_band)};
4218 const std::size_t last {std::min(cols, first + pivoted_qr_band)};
4219 for (std::size_t j {first}; j < last; ++j) {
4220 row_of_f(j)[kk] = tau * conjugate_dot(w.data() + ((start + j) * m));
4221 }
4222 });
4223 for (std::size_t j {0}; j <= kk; ++j) {
4224 row_of_f(j)[kk] = T {};
4225 }
4226 if (kk > 0) { // less the earlier reflectors' share: F(·, kk) += F(·, 0:kk) (−τ V(step:, 0:kk)ᴴ v)
4227 for (std::size_t t {0}; t < kk; ++t) {
4228 coefficients[t] = -tau * conjugate_dot(w.data() + ((start + t) * m));
4229 }
4230 for (std::size_t j {0}; j < cols; ++j) {
4231 T sum {};
4232 for (std::size_t t {0}; t < kk; ++t) {
4233 sum += row_of_f(j)[t] * coefficients[t];
4234 }
4235 row_of_f(j)[kk] += sum;
4236 }
4237 }
4238 for (std::size_t j {kk + 1}; j < cols; ++j) { // the pivot row: A(step, j) −= A(step, start:step+1) F(j, 0:kk+1)ᴴ
4239 T sum {};
4240 for (std::size_t t {0}; t <= kk; ++t) {
4241 sum += w[((start + t) * m) + step] * detail::conj_of(row_of_f(j)[t]);
4242 }
4243 w[((start + j) * m) + step] -= sum;
4244 }
4245 for (std::size_t j {step + 1}; j < n; ++j) { // the norms downdated as laqp2's; one that lost too much waits
4246 if (current[j] != 0.0) { // for the product (the blocks end 128 columns before the last)
4247 const double ratio {std::abs(w[(j * m) + step]) / current[j]};
4248 const double left {std::max(0.0, (1.0 + ratio) * (1.0 - ratio))};
4249 const double drift {left * (current[j] / reference[j]) * (current[j] / reference[j])};
4250 if (drift <= tolerance) {
4251 deferred.push_back(j);
4252 }
4253 else {
4254 current[j] *= std::sqrt(left);
4255 }
4256 }
4257 }
4258 head[step] = diagonal;
4259 reflectors[step] = std::move(v);
4260 taus[step] = tau;
4261 ++done;
4262 }
4263 const std::size_t below {start + done}; // the rows below the block's, the columns right of it: A −= V Fᴴ
4264 const std::size_t right {cols - done}; // each column on its own: by bands of columns on the threads
4265 detail::parallel_for((right + pivoted_qr_band - 1) / pivoted_qr_band, (m - below) * done * pivoted_qr_band, [&](std::size_t band) {
4266 std::array<T, pivoted_qr_block> terms {};
4267 const std::size_t first {done + (band * pivoted_qr_band)};
4268 const std::size_t last {std::min(cols, first + pivoted_qr_band)};
4269 for (std::size_t j {first}; j < last; ++j) {
4270 for (std::size_t t {0}; t < done; ++t) {
4271 terms[t] = detail::conj_of(row_of_f(j)[t]);
4272 }
4273 detail::subtract_products(w.data() + ((start + j) * m) + below, terms.data(), w.data() + (start * m) + below, m, done, m - below);
4274 }
4275 });
4276 for (std::size_t t {0}; t < done; ++t) { // R below the diagonal: the reflectors are kept apart
4277 T* const column {w.data() + ((start + t) * m)};
4278 std::fill(column + start + t + 1, column + m, T {});
4279 }
4280 for (const std::size_t j : deferred) {
4281 current[j] = detail::contiguous_norm(w.data() + (j * m) + below, m - below);
4282 reference[j] = current[j];
4283 }
4284 return done;
4285 }
4286
4289 std::vector<std::size_t> permutation_;
4290 };
4291
4299 template <typename T>
4300 [[nodiscard]] matrix<T> qr_r(matrix<T> a)
4301 {
4302 detail::require_finite(a, "QR decomposition");
4303 const std::size_t rows {a.rows()};
4304 const std::size_t cols {a.cols()};
4305 return matrix<T> {std::min(rows, cols), cols, detail::householder_r(std::move(a).elements(), rows, cols)};
4306 }
4307
4309 class convergence_error : public detail::tagged<std::runtime_error>
4310 {
4311 public:
4312
4314 explicit convergence_error(const std::string& message)
4315 : tagged(message)
4316 {}
4317 };
4318
4319 namespace detail {
4320
4323 inline constexpr std::size_t jacobi_sweeps {60};
4324
4326 template <typename T>
4327 requires std::is_floating_point_v<T>
4329 const T* x,
4330 const T& f,
4331 std::size_t count)
4332 {
4333 add_product(y, f, x, count);
4334 }
4335
4342 template <typename T>
4344 const T* x,
4345 const T& f,
4346 std::size_t count)
4347 {
4348 using R = typename T::value_type;
4349 const R c {f.real()};
4350 const R d {f.imag()};
4351 for (std::size_t i {0}; i < count; ++i) {
4352 const R a {x[i].real()};
4353 const R b {-x[i].imag()};
4354 const R ac {a * c};
4355 const R bd {b * d};
4356 const R ad {a * d};
4357 const R bc {b * c};
4358 const R re {ac - bd};
4359 const R im {ad + bc};
4360 y[i] = T {y[i].real() + re, y[i].imag() + im};
4361 }
4362 }
4363
4365 using symmetric_row_kernel = double (*)(const double*, const double*, double, double*, std::size_t);
4366
4368 using symmetric_rows_kernel = std::array<double, symmetric_row_group> (*)(const double*, std::size_t, const double*,
4369 double*, std::size_t);
4370
4375 template <typename T>
4376 [[nodiscard]] T hermitian_row(symmetric_row_kernel /*kernel*/,
4377 const T* row,
4378 const T* v,
4379 T* out,
4380 std::size_t i)
4381 {
4382 add_conjugate_product(out, row, v[i], i);
4383 return dot_product(row, v, i + 1);
4384 }
4385
4386#ifdef NUMERICAL_NEON_COMPILED
4393 inline std::complex<double> neon_hermitian_row(const double* row,
4394 const double* v,
4395 double vr,
4396 double vi,
4397 double* out,
4398 std::size_t count) noexcept
4399 {
4400 const float64x2_t real {vr, -vr};
4401 const float64x2_t imag {vi, vi};
4402 float64x2_t same[2] {vdupq_n_f64(0.0), vdupq_n_f64(0.0)};
4403 float64x2_t across[2] {vdupq_n_f64(0.0), vdupq_n_f64(0.0)};
4404 std::size_t j {0};
4405 for (; j + 2 <= count; j += 2) {
4406 for (std::size_t l {0}; l < 2; ++l) {
4407 const float64x2_t r {vld1q_f64(row + (2 * (j + l)))};
4408 const float64x2_t x {vld1q_f64(v + (2 * (j + l)))};
4409 const float64x2_t s {vextq_f64(r, r, 1)};
4410 float64x2_t o {vld1q_f64(out + (2 * (j + l)))};
4411 o = vfmaq_f64(o, real, r);
4412 o = vfmaq_f64(o, imag, s);
4413 same[l] = vfmaq_f64(same[l], r, x);
4414 across[l] = vfmaq_f64(across[l], s, x);
4415 vst1q_f64(out + (2 * (j + l)), o);
4416 }
4417 }
4418 const float64x2_t sum_same {vaddq_f64(same[0], same[1])};
4419 const float64x2_t sum_across {vaddq_f64(across[0], across[1])};
4420 double re {vgetq_lane_f64(sum_same, 0) - vgetq_lane_f64(sum_same, 1)};
4421 double im {vgetq_lane_f64(sum_across, 0) + vgetq_lane_f64(sum_across, 1)};
4422 for (; j < count; ++j) {
4423 const double rr {row[2 * j]};
4424 const double ri {row[(2 * j) + 1]};
4425 out[2 * j] = std::fma(vi, ri, std::fma(vr, rr, out[2 * j]));
4426 out[(2 * j) + 1] = std::fma(vi, rr, std::fma(-vr, ri, out[(2 * j) + 1]));
4427 re = std::fma(-ri, v[(2 * j) + 1], std::fma(rr, v[2 * j], re));
4428 im = std::fma(rr, v[(2 * j) + 1], std::fma(ri, v[2 * j], im));
4429 }
4430 return std::complex<double> {re, im};
4431 }
4432
4433#endif
4434
4439 [[nodiscard]] inline std::complex<double> hermitian_row(symmetric_row_kernel kernel,
4440 const std::complex<double>* row,
4441 const std::complex<double>* v,
4442 std::complex<double>* out,
4443 std::size_t i)
4444 {
4445#ifdef NUMERICAL_X86_COMPILED
4446 // A complex number's parts are an array of two (the standard's layout guarantee): addressed as doubles.
4447 const gemm_path path {gemm_path_current()};
4448 if (path == gemm_path::avx512 || path == gemm_path::avx2) {
4449 const auto* const values {reinterpret_cast<const double*>(v)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
4450 const auto* const parts {reinterpret_cast<const double*>(row)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
4451 auto* const into {reinterpret_cast<double*>(out)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
4452 const std::complex<double> lower {path == gemm_path::avx512
4453 ? avx512_hermitian_row(parts, values, v[i].real(), v[i].imag(), into, i)
4454 : avx2_hermitian_row(parts, values, v[i].real(), v[i].imag(), into, i)};
4455 return lower + (row[i] * v[i]);
4456 }
4457#endif
4458#ifdef NUMERICAL_NEON_COMPILED
4459 if (const gemm_path neon_path {gemm_path_current()}; neon_path == gemm_path::neon || neon_path == gemm_path::amx) {
4460 const auto* const values {reinterpret_cast<const double*>(v)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
4461 const auto* const parts {reinterpret_cast<const double*>(row)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
4462 auto* const into {reinterpret_cast<double*>(out)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the layout guarantee above
4463 const std::complex<double> lower {neon_hermitian_row(parts, values, v[i].real(), v[i].imag(), into, i)};
4464 return lower + (row[i] * v[i]);
4465 }
4466#endif
4467 return hermitian_row<std::complex<double>>(kernel, row, v, out, i);
4468 }
4469
4471 [[nodiscard]] inline double hermitian_row(symmetric_row_kernel kernel,
4472 const double* row,
4473 const double* v,
4474 double* out,
4475 std::size_t i)
4476 {
4477 return kernel(row, v, v[i], out, i) + (row[i] * v[i]);
4478 }
4479
4486 template <typename T>
4487 [[nodiscard]] std::size_t symmetric_row_speedup()
4488 {
4489#ifdef NUMERICAL_X86_COMPILED
4490 if constexpr (std::is_same_v<T, double>) {
4491 if (gemm_path_current() != gemm_path::portable) {
4492 return vector_speedup;
4493 }
4494 }
4495#endif
4496 return 1;
4497 }
4498
4514 template <typename T>
4515 void hermitian_product(const T* block,
4516 std::size_t n,
4517 std::size_t m,
4518 const T* v,
4519 T* w,
4520 buffer<T>& scratch)
4521 {
4522 constexpr std::size_t rows_per_task {32};
4523 const std::size_t tasks {(m + rows_per_task - 1) / rows_per_task};
4524 if (scratch.size() < tasks * m) {
4525 scratch.resize(tasks * m, T {});
4526 }
4527 symmetric_row_kernel kernel {[](const double* row, const double* x, double f, double* out, std::size_t count) {
4528 return symmetric_row(row, x, f, out, count);
4529 }};
4530 symmetric_rows_kernel group {[](const double* rows, std::size_t ld, const double* x, double* out, std::size_t i) {
4531 return symmetric_rows(rows, ld, x, out, i);
4532 }};
4533#ifdef NUMERICAL_X86_COMPILED
4534 if constexpr (std::is_same_v<T, double>) {
4535 const gemm_path path {gemm_path_current()};
4536 if (path == gemm_path::avx512) {
4537 kernel = &avx512_symmetric_row;
4538 group = &avx512_symmetric_rows;
4539 }
4540 else if (path == gemm_path::avx2) {
4541 kernel = &avx2_symmetric_row;
4542 group = &avx2_symmetric_rows;
4543 }
4544 }
4545#endif
4546 // The tasks of the longest rows first: the threads take the indices in order, and the last rows' task, of some
4547 // 32 m terms where the first's has 528, left one thread working alone at the end.
4548 parallel_for(tasks, rows_per_task * m * symmetric_row_speedup<T>(), [&](std::size_t index) {
4549 const std::size_t task {tasks - 1 - index};
4550 const std::size_t first {task * rows_per_task};
4551 const std::size_t last {std::min(first + rows_per_task, m)};
4552 T* own {scratch.data() + (task * m)};
4553 std::fill(own, own + last, T {}); // a task's rows reach w[0…last) alone
4554 std::size_t i {first};
4555 if constexpr (std::is_same_v<T, double>) { // rows by groups: v and own read once a group
4556 for (; i + symmetric_row_group <= last; i += symmetric_row_group) {
4557 const std::array<double, symmetric_row_group> sums {group(block + (i * n), n, v, own, i)};
4558 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
4559 own[i + r] += sums[r];
4560 }
4561 }
4562 }
4563 for (; i < last; ++i) {
4564 own[i] += hermitian_row(kernel, block + (i * n), v, own, i);
4565 }
4566 });
4567 std::fill(w, w + m, T {});
4568 for (std::size_t task {0}; task < tasks; ++task) { // the partials in order, each a contiguous update
4569 add_product(w, T {1}, scratch.data() + (task * m), std::min((task + 1) * rows_per_task, m));
4570 }
4571 }
4572
4576#ifdef NUMERICAL_X86_COMPILED
4577 inline constexpr std::size_t corrected_product_least {256};
4578#else
4579 inline constexpr std::size_t corrected_product_least {512};
4580#endif
4581
4583 template <typename T>
4584 requires std::is_floating_point_v<T>
4586 const T* f,
4587 const T* x,
4588 std::size_t ldx,
4589 std::size_t terms,
4590 std::size_t count)
4591 {
4592 subtract_products(y, f, x, ldx, terms, count);
4593 }
4594
4602 template <split_complex T>
4604 const T* f,
4605 const T* x,
4606 std::size_t ldx,
4607 std::size_t terms,
4608 std::size_t count)
4609 {
4610#ifdef NUMERICAL_X86_COMPILED
4611 if constexpr (std::is_same_v<T, std::complex<double>>) { // its own fused kernel on the x86-64 units, as fast
4612 if (const gemm_path path {gemm_path_current()}; path == gemm_path::avx2 || path == gemm_path::avx512) {
4613 subtract_products(y, f, x, ldx, terms, count);
4614 return;
4615 }
4616 }
4617#endif
4618 using R = typename T::value_type;
4619 for (std::size_t t {0}; t < terms; ++t) {
4620 const R c {f[t].real()};
4621 const R d {f[t].imag()};
4622 const T* const row {x + (t * ldx)};
4623 for (std::size_t i {0}; i < count; ++i) {
4624 const R a {row[i].real()};
4625 const R b {row[i].imag()};
4626 const R ac {c * a};
4627 const R bd {d * b};
4628 const R ad {c * b};
4629 const R bc {d * a};
4630 const R re {ac - bd};
4631 const R im {ad + bc};
4632 y[i] = T {y[i].real() - re, y[i].imag() - im};
4633 }
4634 }
4635 }
4636
4656 template <typename T>
4657 void corrected_hermitian_product(const T* block,
4658 std::size_t n,
4659 std::size_t m,
4660 const T* v,
4661 T* w,
4662 buffer<T>& scratch,
4663 const T* corrections,
4664 std::size_t ldc,
4665 std::size_t terms,
4666 T* factors,
4667 T* next = nullptr,
4668 const T* next_factors = nullptr)
4669 {
4670 constexpr std::size_t rows_per_task {32};
4671 const std::size_t tasks {(m + rows_per_task - 1) / rows_per_task};
4672 if (scratch.size() < tasks * m) {
4673 scratch.resize(tasks * m, T {});
4674 }
4675 symmetric_row_kernel kernel {[](const double* row, const double* x, double f, double* out, std::size_t count) {
4676 return symmetric_row(row, x, f, out, count);
4677 }};
4678 symmetric_rows_kernel group {[](const double* rows, std::size_t ld, const double* x, double* out, std::size_t i) {
4679 return symmetric_rows(rows, ld, x, out, i);
4680 }};
4681#ifdef NUMERICAL_X86_COMPILED
4682 if constexpr (std::is_same_v<T, double>) {
4683 const gemm_path path {gemm_path_current()};
4684 if (path == gemm_path::avx512) {
4685 kernel = &avx512_symmetric_row;
4686 group = &avx512_symmetric_rows;
4687 }
4688 else if (path == gemm_path::avx2) {
4689 kernel = &avx2_symmetric_row;
4690 group = &avx2_symmetric_rows;
4691 }
4692 }
4693#endif
4694 // Below corrected_product_least rows the coefficients and the bands run on the caller: woken threads cost more
4695 // than they take (an eigh of order 128, six x86-64 cores, +13 %).
4696 const bool share {m >= corrected_product_least};
4697 const std::size_t pairs {terms / 2};
4698 const auto coefficients = [&](std::size_t p) {
4699 factors[2 * p] = conjugate_dot(corrections + (((2 * p) + 1) * ldc), v, m);
4700 factors[(2 * p) + 1] = conjugate_dot(corrections + ((2 * p) * ldc), v, m);
4701 };
4702 parallel_for(tasks + (share ? pairs : 0), rows_per_task * m * symmetric_row_speedup<T>(), [&](std::size_t index) {
4703 if (index >= tasks) { // a pair of coefficients: an earlier step's w and v against v
4704 coefficients(index - tasks);
4705 return;
4706 }
4707 const std::size_t task {tasks - 1 - index}; // the longest rows first, as hermitian_product
4708 const std::size_t first {task * rows_per_task};
4709 const std::size_t last {std::min(first + rows_per_task, m)};
4710 T* own {scratch.data() + (task * m)};
4711 std::fill(own, own + last, T {});
4712 std::size_t i {first};
4713 if constexpr (std::is_same_v<T, double>) {
4714 for (; i + symmetric_row_group <= last; i += symmetric_row_group) {
4715 const std::array<double, symmetric_row_group> sums {group(block + (i * n), n, v, own, i)};
4716 for (std::size_t r {0}; r < symmetric_row_group; ++r) {
4717 own[i + r] += sums[r];
4718 }
4719 }
4720 }
4721 for (; i < last; ++i) {
4722 own[i] += hermitian_row(kernel, block + (i * n), v, own, i);
4723 }
4724 });
4725 for (std::size_t p {0}; !share && p < pairs; ++p) {
4726 coefficients(p);
4727 }
4728 constexpr std::size_t band {256};
4729 const auto bands = [&](std::size_t b) {
4730 const std::size_t first {b * band};
4731 const std::size_t count {std::min(band, m - first)};
4732 std::fill(w + first, w + first + count, T {});
4733 for (std::size_t task {0}; task < tasks; ++task) { // the partials in order, as hermitian_product
4734 const std::size_t reach {std::min((task + 1) * rows_per_task, m)};
4735 if (reach > first) {
4736 add_product(w + first, T {1}, scratch.data() + (task * m) + first, std::min(count, reach - first));
4737 }
4738 }
4739 subtract_corrections(w + first, factors, corrections + first, ldc, terms, count);
4740 if (next != nullptr) { // the next step's column: these earlier steps' terms (next implies an earlier step)
4741 subtract_corrections(next + first, next_factors, corrections + first, ldc, terms, count);
4742 }
4743 };
4744 if (share) {
4745 parallel_for((m + band - 1) / band, band * (tasks + terms) * 8, bands);
4746 return;
4747 }
4748 for (std::size_t b {0}; b < (m + band - 1) / band; ++b) {
4749 bands(b);
4750 }
4751 }
4752
4754 inline constexpr std::size_t tridiagonal_block {32};
4755
4759 inline constexpr std::size_t tridiagonal_band {512};
4760
4763 inline constexpr std::size_t ql_steps_per_value {30};
4764
4769 inline double real_subdiagonal(double t,
4770 double /*delta*/,
4771 double& /*next*/)
4772 {
4773 return t;
4774 }
4775
4783 inline double real_subdiagonal(std::complex<double> t,
4784 std::complex<double> delta,
4785 std::complex<double>& next)
4786 {
4787 next = delta * std::conj(phase_of(t));
4788 return std::abs(t);
4789 }
4790
4792 template <typename T>
4794 {
4795 std::vector<buffer<T>> reflectors;
4797 };
4798
4813 template <typename T>
4815 std::size_t n,
4816 buffer<double>& d,
4817 buffer<double>& e)
4818 {
4819 std::vector<buffer<T>> reflectors(n); // reflectors[k + 1] zeroes column k, acting on rows k + 1…
4820 buffer<T> symv_scratch; // hermitian_product's partials, kept from step to step
4821 buffer<T> sub(n, T {}); // T's subdiagonal before the phases
4822 // The trailing update's factors, for the first panel's size, the largest: one buffer each, mapped once.
4823 const auto left {std::make_unique_for_overwrite<T[]>(n * 2 * tridiagonal_block)}; // row i: −V's then −W's
4824 const auto right {std::make_unique_for_overwrite<T[]>(2 * tridiagonal_block * n)}; // Wᴴ's rows, then Vᴴ's
4825 // By reflection: row 2p holds step k0 + p's vector v over the rows (zero above), row 2p + 1 its w, so that a
4826 // step's terms of the panel's earlier ones are one update through the rows (subtract_products), in their order.
4827 buffer<T> by_row(2 * tridiagonal_block * n);
4828 buffer<T> columns(tridiagonal_block * n); // the panel's columns of A, gathered row by row
4829 buffer<T> factors(2 * tridiagonal_block); // a step's coefficients of those rows
4830 buffer<T> next_factors(2 * tridiagonal_block); // the next column's, applied ahead
4831 bool column_ahead {false}; // whether the step's column took its earlier terms ahead
4832 for (std::size_t k0 {0}; k0 + 1 < n; k0 += tridiagonal_block) {
4833 // A panel of reflections, as LAPACK's dlatrd: the matrix the panel's later steps see is
4834 // A − V Wᴴ − W Vᴴ, applied to the one column and the one product each step needs; the whole trailing
4835 // block takes it after the panel, through the matrix product kernel.
4836 const std::size_t k1 {std::min(k0 + tridiagonal_block, n - 1)};
4837 const std::size_t width {k1 - k0};
4838 std::fill_n(by_row.data(), 2 * width * n, T {});
4839 for (std::size_t i {k0}; i < n; ++i) { // column k from row k down, each row's part read once
4840 for (std::size_t k {k0}; k < std::min(k1, i + 1); ++k) {
4841 columns[((k - k0) * n) + i] = a[(i * n) + k];
4842 }
4843 }
4844 for (std::size_t k {k0}; k < k1; ++k) {
4845 const std::size_t j {k - k0};
4846 const std::size_t m {n - k - 1};
4847 T* const column {columns.data() + (j * n) + k}; // column k from the diagonal down, the panel's earlier steps applied
4848 // The steps before the last were applied during it (column_ahead); the last's two terms, or all, here.
4849 const std::size_t from {column_ahead ? j - 1 : 0};
4850 for (std::size_t p {from}; p < j; ++p) {
4851 factors[2 * (p - from)] = conj_of(by_row[(((2 * p) + 1) * n) + k]); // v's row by w's element
4852 factors[(2 * (p - from)) + 1] = conj_of(by_row[((2 * p) * n) + k]);
4853 }
4854 subtract_corrections(column, factors.data(), by_row.data() + ((2 * from) * n) + k, n, 2 * (j - from), m + 1);
4855 column_ahead = false;
4856 a[(k * n) + k] = column[0];
4857 buffer<T> v(column + 1, column + m + 1); // column k below the diagonal
4858 const carried_t<T> below {m > 1 ? scaled_norm(v.data() + 1, m - 1) : carried_t<T> {0}};
4859 if (below == 0.0) {
4860 sub[k] = v[0]; // already tridiagonal in this column: nothing to reflect
4861 continue;
4862 }
4863 const carried_t<T> up {rescale_tiny(v)}; // the reflector is the same at any scale; α is scaled back
4864 const T alpha {-phase_of(v[0]) * static_cast<T>(scaled_norm(v))}; // the sign that avoids cancellation
4865 sub[k] = alpha / up;
4866 v[0] -= alpha;
4867 const carried_t<T> length {scaled_norm(v)}; // real: each part divided, where a complex division scaled and divided both
4868 for (T& element : v) {
4869 element /= length;
4870 }
4871 const T* block {a.data() + ((k + 1) * n) + k + 1}; // the trailing block, row stride n
4872 buffer<T> w(m, T {});
4873 // w = (A − V Wᴴ − W Vᴴ) v on the trailing block, (Wᴴ v)ₚ and (Vᴴ v)ₚ beside A v; the next column of the
4874 // panel takes these earlier steps' terms in the same pass (its rows are w's).
4875 T* const next {k + 1 < k1 && j > 0 ? columns.data() + ((j + 1) * n) + k + 1 : nullptr};
4876 for (std::size_t p {0}; next != nullptr && p < j; ++p) {
4877 next_factors[2 * p] = conj_of(by_row[(((2 * p) + 1) * n) + k + 1]);
4878 next_factors[(2 * p) + 1] = conj_of(by_row[((2 * p) * n) + k + 1]);
4879 }
4880 corrected_hermitian_product(block, n, m, v.data(), w.data(), symv_scratch, by_row.data() + k + 1, n, 2 * j, factors.data(),
4881 next, next_factors.data());
4882 column_ahead = next != nullptr;
4883 const T c {conjugate_dot(v.data(), w.data(), m)};
4884 for (std::size_t i {0}; i < m; ++i) { // w = 2 B v − 2 (vᴴ B v) v, so H B H = B − v wᴴ − w vᴴ
4885 w[i] = (static_cast<T>(2) * w[i]) - (static_cast<T>(2) * c * v[i]);
4886 by_row[((2 * j) * n) + k + 1 + i] = v[i];
4887 by_row[(((2 * j) + 1) * n) + k + 1 + i] = w[i];
4888 }
4889 reflectors[k + 1] = std::move(v); // acting from row k + 1, as householder_qr's from row k
4890 }
4891 // The trailing block from row and column k1: A −= V Wᴴ + W Vᴴ, one product of depth 2 · width,
4892 // [−V −W] [Wᴴ; Vᴴ], on the lower triangle alone, as LAPACK's dsyr2k: the reduction reads nothing above the
4893 // diagonal. A real product the kernel packs runs on the triangle's tiles alone (\ref gemm_triangle, each
4894 // element the bands' sum); a smaller or complex one by bands of rows, each to the diagonal, past which a band
4895 // computes what is never read.
4896 const std::size_t rest {n - k1};
4897 const std::size_t depth {2 * width};
4898 for (std::size_t i {0}; i < rest; ++i) { // each row of left written whole, in order
4899 for (std::size_t p {0}; p < width; ++p) {
4900 left[(i * depth) + p] = -by_row[((2 * p) * n) + k1 + i];
4901 left[(i * depth) + width + p] = -by_row[(((2 * p) + 1) * n) + k1 + i];
4902 }
4903 }
4904 for (std::size_t p {0}; p < width; ++p) {
4905 for (std::size_t i {0}; i < rest; ++i) {
4906 right[(p * rest) + i] = conj_of(by_row[(((2 * p) + 1) * n) + k1 + i]);
4907 right[((width + p) * rest) + i] = conj_of(by_row[((2 * p) * n) + k1 + i]);
4908 }
4909 }
4910 T* trailing {a.data() + (k1 * n) + k1};
4911 if constexpr (std::is_same_v<T, double>) {
4912 if (depth >= gemm_min_depth && rest * rest >= ((gemm_packed_threshold + depth) - 1) / depth) {
4913 gemm_triangle<a_rows, b_rows>(rest, depth, left.get(), depth, right.get(), rest, trailing, n, true);
4914 continue;
4915 }
4916 }
4917 for (std::size_t r0 {0}; r0 < rest; r0 += tridiagonal_band) {
4918 const std::size_t rows {std::min(tridiagonal_band, rest - r0)};
4919 gemm(rows, r0 + rows, depth, left.get() + (r0 * depth), depth, right.get(), rest, trailing + (r0 * n), n);
4920 }
4921 }
4922 for (std::size_t i {0}; i < n; ++i) {
4923 d[i] = real_part(a[(i * n) + i]);
4924 }
4925 e[n - 1] = 0.0;
4926 // D T Dᴴ is real with D = diag(δ): δ₀ = 1, δ_{k+1} = δ_k · conj(phase(t_k)); then M = Q Dᴴ.
4927 buffer<T> delta(n, T {1});
4928 for (std::size_t k {0}; k + 1 < n; ++k) {
4929 e[k] = real_subdiagonal(sub[k], delta[k], delta[k + 1]);
4930 }
4931 return {.reflectors = std::move(reflectors), .delta = std::move(delta)};
4932 }
4933
4939 template <typename T>
4941 std::size_t n)
4942 {
4943 buffer<T> mt(n * n, T {});
4944 for (std::size_t i {0}; i < n; ++i) {
4945 mt[(i * n) + i] = T {1};
4946 }
4947 for (std::size_t k1 {n}; k1 > 1;) {
4948 const std::size_t k0 {k1 - std::min(k1 - 1, qr_block)};
4949 apply_transposed_reflector_block(gather_reflectors(reduction.reflectors, k0, k1, n), mt, n, k0, n);
4950 k1 = k0;
4951 }
4952 for (std::size_t j {0}; j < n; ++j) { // row j of Mᵀ = Dᴴ Qᵀ: row j of Qᵀ times conj(δⱼ)
4953 const T factor {conj_of(reduction.delta[j])};
4954 for (std::size_t i {0}; i < n; ++i) {
4955 mt[(j * n) + i] *= factor;
4956 }
4957 }
4958 return mt;
4959 }
4960
4967 template <typename T>
4969 buffer<T>& at,
4970 std::size_t n,
4971 std::size_t rows)
4972 {
4973 if constexpr (!std::is_floating_point_v<T>) { // a real reduction's phases are all 1
4974 parallel_for(rows, n, [&](std::size_t i) {
4975 for (std::size_t j {0}; j < n; ++j) {
4976 at[(i * n) + j] *= conj_of(reduction.delta[j]);
4977 }
4978 });
4979 }
4980 for (std::size_t k1 {n}; k1 > 1;) {
4981 const std::size_t k0 {k1 - std::min(k1 - 1, qr_block)};
4982 apply_transposed_reflector_block(gather_reflectors(reduction.reflectors, k0, k1, n), at, n, 0, rows);
4983 k1 = k0;
4984 }
4985 }
4986
4988 template <typename T>
4990 buffer<T>& at,
4991 std::size_t n)
4992 {
4993 to_matrix_basis(reduction, at, n, n);
4994 }
4995
4999 {
5000 enum class kind : std::uint8_t
5001 {
5002 rotate,
5003 negate,
5004 swap,
5005 };
5006
5008 std::size_t i;
5009 std::size_t j;
5010 double c;
5011 double s;
5012 };
5013
5017 {
5018 std::size_t first;
5019 std::size_t count;
5020 bool down;
5021 };
5022
5024 [[nodiscard]] inline std::vector<rotation_run> rotation_runs(const std::vector<row_operation>& operations)
5025 {
5026 std::vector<rotation_run> runs;
5027 for (std::size_t t {0}; t < operations.size();) {
5028 const row_operation& op {operations[t]};
5029 std::size_t count {1};
5030 bool down {false};
5031 if (op.what == row_operation::kind::rotate && t + 1 < operations.size()
5032 && operations[t + 1].what == row_operation::kind::rotate) {
5033 down = operations[t + 1].j == op.i;
5034 for (; t + count < operations.size(); ++count) {
5035 const row_operation& previous {operations[t + count - 1]};
5036 const row_operation& next {operations[t + count]};
5037 const bool chained {down ? next.j == previous.i : next.i == previous.j};
5038 if (next.what != row_operation::kind::rotate || !chained) {
5039 break;
5040 }
5041 }
5042 }
5043 runs.push_back({.first = t, .count = count, .down = down});
5044 t += count;
5045 }
5046 return runs;
5047 }
5048
5058 template <typename T>
5060 std::size_t n,
5061 const std::vector<row_operation>& operations)
5062 {
5063 constexpr std::size_t width {64};
5064 const std::vector<rotation_run> runs {rotation_runs(operations)};
5065 parallel_for((n + width - 1) / width, operations.size() * width, [&](std::size_t chunk) {
5066 const std::size_t first {chunk * width};
5067 const std::size_t count {std::min(first + width, n) - first};
5068 std::array<T, width> carry {};
5069 for (const rotation_run& run : runs) {
5070 const row_operation& start {operations[run.first]};
5071 if (run.count > 1) {
5072 // Down: rotation u's first row is rotation u+1's second; up: its second is the next's first.
5073 const T* source {m.data() + ((run.down ? start.j : start.i) * n) + first};
5074 std::copy(source, source + count, carry.begin());
5075 for (std::size_t u {run.first}; u < run.first + run.count; ++u) {
5076 const row_operation& op {operations[u]};
5077 const T c {detail::from_carried<T>(op.c)};
5078 const T s {detail::from_carried<T>(op.s)};
5079 if (run.down) { // the carried row is op.j: op.i is read, op.j written
5080 const T* row {m.data() + (op.i * n) + first};
5081 T* other {m.data() + (op.j * n) + first};
5082 for (std::size_t k {0}; k < count; ++k) {
5083 const T a {row[k]};
5084 const T b {carry[k]};
5085 other[k] = (c * b) - (s * a);
5086 carry[k] = (c * a) + (s * b);
5087 }
5088 }
5089 else { // the carried row is op.i: op.j is read, op.i written
5090 T* row {m.data() + (op.i * n) + first};
5091 const T* other {m.data() + (op.j * n) + first};
5092 for (std::size_t k {0}; k < count; ++k) {
5093 const T a {carry[k]};
5094 const T b {other[k]};
5095 row[k] = (c * a) + (s * b);
5096 carry[k] = (c * b) - (s * a);
5097 }
5098 }
5099 }
5100 const row_operation& end {operations[run.first + run.count - 1]};
5101 T* target {m.data() + ((run.down ? end.i : end.j) * n) + first};
5102 std::copy(carry.begin(), carry.begin() + static_cast<std::ptrdiff_t>(count), target);
5103 continue;
5104 }
5105 T* row {m.data() + (start.i * n) + first};
5106 T* other {m.data() + (start.j * n) + first};
5107 // The kind is tested outside the column loop, which then vectorises.
5108 if (start.what == row_operation::kind::rotate) {
5109 const T c {detail::from_carried<T>(start.c)};
5110 const T s {detail::from_carried<T>(start.s)};
5111 for (std::size_t k {0}; k < count; ++k) {
5112 const T a {row[k]};
5113 const T b {other[k]};
5114 row[k] = (c * a) + (s * b);
5115 other[k] = (c * b) - (s * a);
5116 }
5117 }
5118 else if (start.what == row_operation::kind::negate) {
5119 for (std::size_t k {0}; k < count; ++k) {
5120 row[k] = -row[k];
5121 }
5122 }
5123 else {
5124 std::swap_ranges(row, row + count, other);
5125 }
5126 }
5127 });
5128 }
5129
5132 {
5133 double c;
5134 double s;
5135 double r;
5136 };
5137
5140 [[nodiscard]] inline plane_rotation_real plane_rotation_to_zero(double f,
5141 double g) noexcept
5142 {
5143 if (g == 0.0) {
5144 return {1.0, 0.0, f};
5145 }
5146 const double length {rotation_length(f, g)};
5147 const double r {std::copysign(length, f)};
5148 return {std::abs(f) / length, g / r, r};
5149 }
5150
5153 {
5154 double larger;
5155 double smaller;
5156 double c;
5157 double s;
5158 };
5159
5165 [[nodiscard]] inline symmetric_two_by_two symmetric_eigen_2x2(double a,
5166 double b,
5167 double c) noexcept
5168 {
5169 const double sum {a + c};
5170 const double difference {a - c};
5171 const double spread {std::abs(difference)};
5172 const double twice_b {b + b};
5173 const double coupling {std::abs(twice_b)};
5174 const double larger_part {std::abs(a) > std::abs(c) ? a : c};
5175 const double other_part {std::abs(a) > std::abs(c) ? c : a};
5176 double root {coupling * std::sqrt(2.0)};
5177 if (spread > coupling) {
5178 root = spread * std::sqrt(1.0 + ((coupling / spread) * (coupling / spread)));
5179 }
5180 else if (spread < coupling) {
5181 root = coupling * std::sqrt(1.0 + ((spread / coupling) * (spread / coupling)));
5182 }
5183 double larger {0.5 * root};
5184 double smaller {-0.5 * root};
5185 int sign_of_sum {1};
5186 if (sum != 0.0) {
5187 larger = 0.5 * (sum < 0.0 ? sum - root : sum + root);
5188 sign_of_sum = sum < 0.0 ? -1 : 1;
5189 smaller = ((larger_part / larger) * other_part) - ((b / larger) * b);
5190 }
5191 const int sign_of_difference {difference >= 0.0 ? 1 : -1};
5192 const double towards {difference >= 0.0 ? difference + root : difference - root};
5193 double cs {1.0};
5194 double sn {0.0};
5195 if (std::abs(towards) > coupling) {
5196 const double t {-twice_b / towards};
5197 sn = 1.0 / std::sqrt(1.0 + (t * t));
5198 cs = t * sn;
5199 }
5200 else if (coupling != 0.0) {
5201 const double t {-towards / twice_b};
5202 cs = 1.0 / std::sqrt(1.0 + (t * t));
5203 sn = t * cs;
5204 }
5205 if (sign_of_sum == sign_of_difference) {
5206 const double swapped {cs};
5207 cs = -sn;
5208 sn = swapped;
5209 }
5210 return {larger, smaller, cs, sn};
5211 }
5212
5227 buffer<double>& e,
5228 std::size_t n,
5229 std::size_t max_steps)
5230 {
5231 constexpr double eps {std::numeric_limits<double>::epsilon() / 2.0}; // LAPACK's dlamch('E')
5232 constexpr double eps2 {eps * eps};
5233 constexpr double safe_least {std::numeric_limits<double>::min()};
5234 const double scaled_most {std::sqrt(1.0 / safe_least) / 3.0};
5235 const double scaled_least {std::sqrt(safe_least) / eps2};
5236 std::size_t steps {0};
5237 std::size_t first {0};
5238 while (first < n) {
5239 std::size_t last {first};
5240 while (last + 1 < n && std::abs(e[last]) > (std::sqrt(std::abs(d[last])) * std::sqrt(std::abs(d[last + 1]))) * eps) {
5241 ++last;
5242 }
5243 if (last + 1 < n) {
5244 e[last] = 0.0;
5245 }
5246 const std::size_t block_first {first};
5247 first = last + 1;
5248 if (last == block_first) {
5249 continue;
5250 }
5251 double largest {0.0};
5252 for (std::size_t i {block_first}; i <= last; ++i) {
5253 largest = std::max({largest, std::abs(d[i]), i < last ? std::abs(e[i]) : 0.0});
5254 }
5255 const bool scaled {largest > scaled_most || largest < scaled_least};
5256 const double scale {largest > scaled_most ? scaled_most : scaled_least};
5257 for (std::size_t i {block_first}; i <= last; ++i) {
5258 d[i] = scaled ? (d[i] / largest) * scale : d[i];
5259 if (i < last) { // the squares from here on
5260 const double entry {scaled ? (e[i] / largest) * scale : e[i]};
5261 e[i] = entry * entry;
5262 }
5263 }
5264 // The shift from the 2 × 2 at one end of [l, m], its off-diagonal's square `square`.
5265 const auto shift = [](double p, double next, double square) {
5266 const double root {std::sqrt(square)};
5267 const double sigma {(next - p) / (2.0 * root)};
5268 return p - (root / (sigma + std::copysign(rotation_length(sigma, 1.0), sigma)));
5269 };
5270 if (std::abs(d[last]) >= std::abs(d[block_first])) { // QL: from the top
5271 std::size_t l {block_first};
5272 while (l <= last) {
5273 std::size_t m {l};
5274 while (m < last && std::abs(e[m]) > eps2 * std::abs(d[m] * d[m + 1])) {
5275 ++m;
5276 }
5277 if (m < last) {
5278 e[m] = 0.0;
5279 }
5280 if (m == l) {
5281 ++l;
5282 continue;
5283 }
5284 if (m == l + 1) {
5285 const symmetric_two_by_two pair {symmetric_eigen_2x2(d[l], std::sqrt(e[l]), d[l + 1])};
5286 d[l] = pair.larger;
5287 d[l + 1] = pair.smaller;
5288 e[l] = 0.0;
5289 l += 2;
5290 continue;
5291 }
5292 if (steps == max_steps) {
5293 throw convergence_error("symmetric eigenvalues: QL did not converge within its steps");
5294 }
5295 ++steps;
5296 const double sigma {shift(d[l], d[l + 1], e[l])};
5297 double c {1.0};
5298 double s {0.0};
5299 double gamma {d[m] - sigma};
5300 double p {gamma * gamma};
5301 for (std::size_t i {m}; i-- > l;) {
5302 const double bb {e[i]};
5303 const double r {p + bb};
5304 if (i + 1 != m) {
5305 e[i + 1] = s * r;
5306 }
5307 const double old_c {c};
5308 c = p / r;
5309 s = bb / r;
5310 const double old_gamma {gamma};
5311 gamma = (c * (d[i] - sigma)) - (s * old_gamma);
5312 d[i + 1] = old_gamma + (d[i] - gamma);
5313 p = c != 0.0 ? (gamma * gamma) / c : old_c * bb;
5314 }
5315 e[l] = s * p;
5316 d[l] = sigma + gamma;
5317 }
5318 }
5319 else { // QR: from the bottom
5320 std::size_t l {last};
5321 for (;;) { // left once l reaches the block's first entry
5322 std::size_t m {l};
5323 while (m > block_first && std::abs(e[m - 1]) > eps2 * std::abs(d[m] * d[m - 1])) {
5324 --m;
5325 }
5326 if (m > block_first) {
5327 e[m - 1] = 0.0;
5328 }
5329 if (m == l) {
5330 if (l == block_first) {
5331 break;
5332 }
5333 --l;
5334 continue;
5335 }
5336 if (m + 1 == l) {
5337 const symmetric_two_by_two pair {symmetric_eigen_2x2(d[l], std::sqrt(e[l - 1]), d[l - 1])};
5338 d[l] = pair.larger;
5339 d[l - 1] = pair.smaller;
5340 e[l - 1] = 0.0;
5341 if (l < block_first + 2) {
5342 break;
5343 }
5344 l -= 2;
5345 continue;
5346 }
5347 if (steps == max_steps) {
5348 throw convergence_error("symmetric eigenvalues: QL did not converge within its steps");
5349 }
5350 ++steps;
5351 const double sigma {shift(d[l], d[l - 1], e[l - 1])};
5352 double c {1.0};
5353 double s {0.0};
5354 double gamma {d[m] - sigma};
5355 double p {gamma * gamma};
5356 for (std::size_t i {m}; i < l; ++i) {
5357 const double bb {e[i]};
5358 const double r {p + bb};
5359 if (i != m) {
5360 e[i - 1] = s * r;
5361 }
5362 const double old_c {c};
5363 c = p / r;
5364 s = bb / r;
5365 const double old_gamma {gamma};
5366 gamma = (c * (d[i + 1] - sigma)) - (s * old_gamma);
5367 d[i] = old_gamma + (d[i + 1] - gamma);
5368 p = c != 0.0 ? (gamma * gamma) / c : old_c * bb;
5369 }
5370 e[l - 1] = s * p;
5371 d[l] = sigma + gamma;
5372 }
5373 }
5374 for (std::size_t i {block_first}; scaled && i <= last; ++i) { // back to the matrix's scale
5375 d[i] = (d[i] / scale) * largest;
5376 }
5377 }
5378 }
5379
5397 template <typename T>
5399 buffer<double>& e,
5400 buffer<T>& zt,
5401 std::size_t n,
5402 std::size_t max_steps)
5403 {
5404 constexpr double eps {std::numeric_limits<double>::epsilon() / 2.0}; // LAPACK's dlamch('E')
5405 constexpr double eps2 {eps * eps};
5406 constexpr double safe_least {std::numeric_limits<double>::min()};
5407 const double scaled_most {std::sqrt(1.0 / safe_least) / 3.0};
5408 const double scaled_least {std::sqrt(safe_least) / eps2};
5409 if (zt.empty()) { // the eigenvalues alone: dsterf's root-free iteration
5410 tridiagonal_root_free_ql(d, e, n, max_steps);
5411 return;
5412 }
5413 // The iteration reads only d and e: its rotations of zt's rows are recorded and applied once at the
5414 // end, in parallel by columns.
5415 std::vector<row_operation> on_zt;
5416 std::size_t steps {0};
5417 const auto negligible = [&](std::size_t i) { // e[i] against d[i] and d[i + 1]
5418 return e[i] * e[i] <= ((eps2 * std::abs(d[i])) * std::abs(d[i + 1])) + safe_least;
5419 };
5420 std::size_t first {0};
5421 while (first < n) {
5422 // The next unreduced block [first, last].
5423 std::size_t last {first};
5424 while (last + 1 < n && std::abs(e[last]) > (std::sqrt(std::abs(d[last])) * std::sqrt(std::abs(d[last + 1]))) * eps) {
5425 ++last;
5426 }
5427 if (last + 1 < n) {
5428 e[last] = 0.0;
5429 }
5430 const std::size_t block_first {first};
5431 first = last + 1;
5432 if (last == block_first) {
5433 continue;
5434 }
5435 double largest {0.0};
5436 for (std::size_t i {block_first}; i <= last; ++i) {
5437 largest = std::max({largest, std::abs(d[i]), i < last ? std::abs(e[i]) : 0.0});
5438 }
5439 // Into a safe range, where it lies outside: the squares of the test neither overflow nor vanish.
5440 const bool scaled {largest > scaled_most || largest < scaled_least};
5441 const double scale {largest > scaled_most ? scaled_most : scaled_least};
5442 for (std::size_t i {block_first}; scaled && i <= last; ++i) {
5443 d[i] = (d[i] / largest) * scale;
5444 e[i] = i < last ? (e[i] / largest) * scale : e[i];
5445 }
5446 if (std::abs(d[last]) >= std::abs(d[block_first])) { // QL: the smaller end first, from the top
5447 std::size_t l {block_first};
5448 while (l <= last) {
5449 std::size_t m {l};
5450 while (m < last && !negligible(m)) {
5451 ++m;
5452 }
5453 if (m < last) {
5454 e[m] = 0.0;
5455 }
5456 if (m == l) {
5457 ++l;
5458 continue;
5459 }
5460 if (m == l + 1) {
5461 const symmetric_two_by_two pair {symmetric_eigen_2x2(d[l], e[l], d[l + 1])};
5462 on_zt.push_back({row_operation::kind::rotate, l, l + 1, pair.c, pair.s});
5463 d[l] = pair.larger;
5464 d[l + 1] = pair.smaller;
5465 e[l] = 0.0;
5466 l += 2;
5467 continue;
5468 }
5469 if (steps == max_steps) {
5470 throw convergence_error("symmetric eigendecomposition: QL did not converge within its steps");
5471 }
5472 ++steps;
5473 double g {(d[l + 1] - d[l]) / (2.0 * e[l])};
5474 double r {rotation_length(g, 1.0)};
5475 g = d[m] - d[l] + (e[l] / (g + std::copysign(r, g)));
5476 double s {1.0};
5477 double c {1.0};
5478 double p {0.0};
5479 for (std::size_t i {m}; i-- > l;) {
5480 const double f {s * e[i]};
5481 const double b {c * e[i]};
5483 c = turn.c;
5484 s = turn.s;
5485 if (i + 1 != m) {
5486 e[i + 1] = turn.r;
5487 }
5488 g = d[i + 1] - p;
5489 r = ((d[i] - g) * s) + (2.0 * c * b);
5490 p = s * r;
5491 d[i + 1] = g + p;
5492 g = (c * r) - b;
5493 on_zt.push_back({row_operation::kind::rotate, i, i + 1, c, -s});
5494 }
5495 d[l] -= p;
5496 e[l] = g;
5497 }
5498 }
5499 else { // QR: from the bottom
5500 std::size_t l {last};
5501 for (;;) { // left once l reaches the block's first entry (below)
5502 std::size_t m {l};
5503 while (m > block_first && !negligible(m - 1)) {
5504 --m;
5505 }
5506 if (m > block_first) {
5507 e[m - 1] = 0.0;
5508 }
5509 if (m == l) {
5510 if (l == block_first) {
5511 break;
5512 }
5513 --l;
5514 continue;
5515 }
5516 if (m + 1 == l) {
5517 const symmetric_two_by_two pair {symmetric_eigen_2x2(d[l - 1], e[l - 1], d[l])};
5518 on_zt.push_back({row_operation::kind::rotate, l - 1, l, pair.c, pair.s});
5519 d[l - 1] = pair.larger;
5520 d[l] = pair.smaller;
5521 e[l - 1] = 0.0;
5522 if (l < block_first + 2) {
5523 break;
5524 }
5525 l -= 2;
5526 continue;
5527 }
5528 if (steps == max_steps) {
5529 throw convergence_error("symmetric eigendecomposition: QL did not converge within its steps");
5530 }
5531 ++steps;
5532 double g {(d[l - 1] - d[l]) / (2.0 * e[l - 1])};
5533 double r {rotation_length(g, 1.0)};
5534 g = d[m] - d[l] + (e[l - 1] / (g + std::copysign(r, g)));
5535 double s {1.0};
5536 double c {1.0};
5537 double p {0.0};
5538 for (std::size_t i {m}; i < l; ++i) {
5539 const double f {s * e[i]};
5540 const double b {c * e[i]};
5542 c = turn.c;
5543 s = turn.s;
5544 if (i != m) {
5545 e[i - 1] = turn.r;
5546 }
5547 g = d[i] - p;
5548 r = ((d[i + 1] - g) * s) + (2.0 * c * b);
5549 p = s * r;
5550 d[i] = g + p;
5551 g = (c * r) - b;
5552 on_zt.push_back({row_operation::kind::rotate, i, i + 1, c, s});
5553 }
5554 d[l] -= p;
5555 e[l - 1] = g;
5556 }
5557 }
5558 for (std::size_t i {block_first}; scaled && i <= last; ++i) { // back to the matrix's scale
5559 d[i] = (d[i] / scale) * largest;
5560 e[i] = i < last ? (e[i] / scale) * largest : e[i];
5561 }
5562 }
5563 apply_row_operations(zt, n, on_zt);
5564 }
5565
5567 inline constexpr std::size_t divide_leaf {25};
5568
5571 inline constexpr std::size_t secular_iterations {100};
5572
5575 {
5576 std::size_t origin;
5577 double offset;
5578 };
5579
5588 [[nodiscard]] inline double quadratic_step(double a,
5589 double b,
5590 double c)
5591 {
5592 const double root {std::sqrt(std::abs((a * a) - (4.0 * b * c)))};
5593 return a <= 0.0 ? (a - root) / (2.0 * c) : (2.0 * b) / (a + root);
5594 }
5595
5614 template <typename Offset>
5616 double rho,
5617 std::size_t i,
5618 std::size_t max_iterations,
5619 double* delta,
5620 const Offset& offset)
5621 {
5622 const std::size_t k {z.size()};
5623 const double rhoinv {1.0 / rho};
5624 const bool last {i + 1 == k};
5625 double weight {0.0};
5626 for (const double element : z) {
5627 weight += element * element;
5628 }
5629 std::size_t origin {i};
5630 double lower {0.0};
5631 double upper {2.0 * rho * weight}; // the last root is at most ρ‖z‖² past its pole: the equation is positive here
5632 if (!last) {
5633 const double gap {offset(i, i + 1)};
5634 double f {rhoinv};
5635 for (std::size_t j {0}; j < k; ++j) {
5636 f += z[j] * z[j] / (offset(i, j) - (gap / 2.0));
5637 }
5638 upper = gap / 2.0;
5639 if (f < 0.0) { // the root is nearer the pole past it
5640 origin = i + 1;
5641 lower = -gap / 2.0;
5642 upper = 0.0;
5643 }
5644 }
5645 double tau {(lower + upper) / 2.0};
5646 for (std::size_t iteration {0}; iteration < max_iterations; ++iteration) {
5647 double psi {0.0}; // the terms of the poles up to i, and their derivative
5648 double dpsi {0.0};
5649 double phi {0.0}; // those of the poles past i
5650 double dphi {0.0};
5651 // The two sides apart, each in its order: a term chose its sum by a branch (an SVD of order 128 on one arm64
5652 // core: 1.87 → 1.68 ms, eigh 1.02 → 0.82).
5653 for (std::size_t j {0}; j <= i; ++j) {
5654 delta[j] = offset(origin, j) - tau;
5655 const double t {z[j] / delta[j]};
5656 psi += z[j] * t;
5657 dpsi += t * t;
5658 }
5659 for (std::size_t j {i + 1}; j < k; ++j) {
5660 delta[j] = offset(origin, j) - tau;
5661 const double t {z[j] / delta[j]};
5662 phi += z[j] * t;
5663 dphi += t * t;
5664 }
5665 const double w {rhoinv + psi + phi};
5666 if (std::abs(w) <= std::numeric_limits<double>::epsilon()
5667 * ((8.0 * (phi - psi + rhoinv)) + (3.0 * std::abs(tau) * (dpsi + dphi)))) {
5668 return {.origin = origin, .offset = tau};
5669 }
5670 (w < 0.0 ? lower : upper) = tau; // the equation increases with x
5671 // The fit c + s₁/(δᵢ − η) + s₂/(δᵢ₊₁ − η) matching the value and the slope of each side: C η² − A η + B = 0
5672 // (the last root has no pole to its right: its δᵢ₊₁ term is absent, A = δᵢ w − δᵢ² ψ' over C = w − δᵢ ψ').
5673 const double a1 {delta[i]};
5674 const double a2 {last ? 0.0 : delta[i + 1]};
5675 const double c {w - (a1 * dpsi) - (a2 * dphi)};
5676 const double a {last ? (a1 * c) + (a1 * a1 * dpsi) : ((a1 + a2) * w) - (a1 * a2 * (dpsi + dphi))};
5677 const double b {last ? 0.0 : a1 * a2 * w};
5678 const double next {tau + (last ? a / c : quadratic_step(a, b, c))};
5679 // A step out of the bracket (or not a number): bisect. The test above stops within a few units in the
5680 // last place of the root, before the bracket runs out of points.
5681 tau = next > lower && next < upper ? next : (lower + upper) / 2.0;
5682 }
5683 throw convergence_error("the secular equation did not converge");
5684 }
5685
5697 [[nodiscard]] inline double secular_root(const buffer<double>& d,
5698 const buffer<double>& z,
5699 double rho,
5700 std::size_t i,
5701 std::size_t max_iterations,
5702 double* delta)
5703 {
5704 const secular_solution root {solve_secular(z, rho, i, max_iterations, delta,
5705 [&d](std::size_t o, std::size_t j) { return d[j] - d[o]; })};
5706 return d[root.origin] + root.offset;
5707 }
5708
5710 inline constexpr std::size_t lowner_block {32};
5711
5714 {
5715 std::unique_ptr<double[]> data;
5716 std::size_t size;
5717 };
5718
5722 {
5724 workspace_buffer updated {};
5725 workspace_buffer columns {};
5727 };
5728
5731 inline constexpr std::size_t first_touch_chunk {std::size_t {1} << 15};
5732
5739 [[nodiscard]] inline double* workspace_of(workspace_buffer& buffer,
5740 std::size_t size)
5741 {
5742 if (buffer.size < size) {
5743 buffer.data = std::make_unique_for_overwrite<double[]>(size);
5744 buffer.size = size;
5745 double* const elements {buffer.data.get()};
5746 parallel_for((size + first_touch_chunk - 1) / first_touch_chunk, first_touch_chunk, [&](std::size_t chunk) {
5747 const std::size_t first {chunk * first_touch_chunk};
5748 std::fill(elements + first, elements + std::min(first + first_touch_chunk, size), 0.0);
5749 });
5750 }
5751 return buffer.data.get();
5752 }
5753
5756 inline constexpr std::size_t kept_workspace {std::size_t {1} << 18};
5757
5763 [[nodiscard]] inline merge_workspace& thread_workspace()
5764 {
5765 thread_local merge_workspace work;
5766 [[maybe_unused]] thread_local const bool registered {on_release([] { work = merge_workspace {}; })};
5767 return work;
5768 }
5769
5772 {
5773 for (workspace_buffer* buffer : {&work.gaps, &work.updated, &work.columns, &work.factor}) {
5774 if (buffer->size > kept_workspace) {
5775 buffer->data.reset();
5776 buffer->size = 0;
5777 }
5778 }
5779 }
5780
5795 double rho,
5796 double* q,
5797 std::size_t ldq,
5798 std::size_t n,
5799 std::size_t split,
5800 merge_workspace& work)
5801 {
5802 // Which halves of the rows each column reaches: 1 the upper, 2 the lower.
5803 std::vector<unsigned> reach(n);
5804 for (std::size_t j {0}; j < n; ++j) {
5805 reach[j] = j < split ? 1U : 2U;
5806 }
5807 for (double& element : z) { // ‖z‖ = √2: the rows of two orthogonal matrices
5808 element /= std::numbers::sqrt2;
5809 }
5810 rho *= 2.0;
5811 std::vector<std::size_t> order(n);
5812 for (std::size_t j {0}; j < n; ++j) {
5813 order[j] = j;
5814 }
5815 std::sort(order.begin(), order.end(), [&d](std::size_t x, std::size_t y) { return d[x] < d[y]; });
5816 double largest_d {0.0};
5817 double largest_z {0.0};
5818 for (std::size_t j {0}; j < n; ++j) {
5819 largest_d = std::max(largest_d, std::abs(d[j]));
5820 largest_z = std::max(largest_z, std::abs(z[j]));
5821 }
5822 const double tolerance {8.0 * std::numeric_limits<double>::epsilon() * std::max(largest_d, largest_z)};
5823 std::vector<std::size_t> kept; // the columns the secular equation updates, by ascending d
5824 std::vector<std::size_t> deflated; // the columns that keep their eigenvalue
5825 std::size_t pending {n}; // the last column kept so far, still open to a rotation with the next
5826 for (const std::size_t j : order) {
5827 if (rho * std::abs(z[j]) <= tolerance) {
5828 deflated.push_back(j);
5829 continue;
5830 }
5831 if (pending == n) {
5832 pending = j;
5833 continue;
5834 }
5835 const double length {std::hypot(z[j], z[pending])};
5836 const double c {z[j] / length};
5837 const double s {-z[pending] / length};
5838 if (std::abs((d[j] - d[pending]) * c * s) <= tolerance) {
5839 // Close poles: a rotation of the two columns zeroes the pending one's weight, which then deflates.
5840 z[j] = length;
5841 z[pending] = 0.0;
5842 for (std::size_t r {0}; r < n; ++r) {
5843 const double x {q[(r * ldq) + pending]};
5844 const double y {q[(r * ldq) + j]};
5845 q[(r * ldq) + pending] = (c * x) + (s * y);
5846 q[(r * ldq) + j] = (c * y) - (s * x);
5847 }
5848 reach[j] = reach[pending] = reach[j] | reach[pending];
5849 const double rotated {(d[pending] * c * c) + (d[j] * s * s)};
5850 d[j] = (d[pending] * s * s) + (d[j] * c * c);
5851 d[pending] = rotated;
5852 deflated.push_back(pending);
5853 }
5854 else {
5855 kept.push_back(pending);
5856 }
5857 pending = j;
5858 }
5859 if (pending != n) {
5860 kept.push_back(pending);
5861 }
5862 const std::size_t k {kept.size()};
5863 buffer<double> poles(k, double {});
5864 buffer<double> weights(k, double {});
5865 double norm {0.0};
5866 for (std::size_t i {0}; i < k; ++i) {
5867 poles[i] = d[kept[i]];
5868 weights[i] = z[kept[i]];
5869 norm += weights[i] * weights[i];
5870 }
5871 norm = std::sqrt(norm);
5872 for (double& weight : weights) {
5873 weight /= norm;
5874 }
5875 const double scaled_rho {rho * norm * norm};
5876 buffer<double> roots(k, double {});
5877 // gaps[i * k + j] = poles[j] − roots[i], every element written by its root's solve.
5878 double* const gaps {workspace_of(work.gaps, k * k)};
5879 parallel_for(k, 8 * k, [&](std::size_t i) {
5880 roots[i] = secular_root(poles, weights, scaled_rho, i, secular_iterations, gaps + (i * k));
5881 });
5882 // Löwner: ẑⱼ² = −Πᵢ (dⱼ − λᵢ) / Πᵢ≠ⱼ (dⱼ − dᵢ), each pole paired with its root, by blocks of poles, each pole's
5883 // product in ascending order of the roots (a row of gaps at a time, along the block): the same operations in
5884 // the same order on any number of threads.
5885 buffer<double> signed_weights(k, double {});
5886 parallel_for((k + lowner_block - 1) / lowner_block, lowner_block * k, [&](std::size_t block) {
5887 const std::size_t first {block * lowner_block};
5888 const std::size_t last {std::min(first + lowner_block, k)};
5889 std::array<double, lowner_block> product {};
5890 for (std::size_t j {first}; j < last; ++j) {
5891 product[j - first] = gaps[(j * k) + j];
5892 }
5893 for (std::size_t i {0}; i < k; ++i) {
5894 for (std::size_t j {first}; j < last; ++j) {
5895 if (i != j) {
5896 product[j - first] *= gaps[(i * k) + j] / (poles[j] - poles[i]);
5897 }
5898 }
5899 }
5900 for (std::size_t j {first}; j < last; ++j) {
5901 signed_weights[j] = std::copysign(std::sqrt(-product[j - first]), weights[j]);
5902 }
5903 });
5904 // u[j][i] = ẑⱼ / (dⱼ − λᵢ), column i roots[i]'s eigenvector in the kept basis, in gaps' place: gaps[i][j] and
5905 // gaps[j][i] are read, then each written as the other's quotient, tile by tile across the diagonal.
5906 double* const u {gaps};
5907 const std::size_t tiles {(k + lowner_block - 1) / lowner_block};
5908 parallel_for(tiles * tiles, lowner_block * lowner_block, [&](std::size_t tile) {
5909 const std::size_t ti {tile / tiles};
5910 const std::size_t tj {tile % tiles};
5911 if (ti > tj) {
5912 return; // the pair of tiles is (tj, ti)'s
5913 }
5914 for (std::size_t i {ti * lowner_block}; i < std::min((ti + 1) * lowner_block, k); ++i) {
5915 for (std::size_t j {std::max(tj * lowner_block, ti == tj ? i : 0)}; j < std::min((tj + 1) * lowner_block, k); ++j) {
5916 const double above {gaps[(i * k) + j]};
5917 const double below {gaps[(j * k) + i]};
5918 u[(j * k) + i] = signed_weights[j] / above;
5919 u[(i * k) + j] = signed_weights[i] / below;
5920 }
5921 }
5922 });
5923 buffer<double> length(k, 0.0);
5924 for (std::size_t j {0}; j < k; ++j) { // by rows of u, each a contiguous update of every column's sum
5925 for (std::size_t i {0}; i < k; ++i) {
5926 length[i] += u[(j * k) + i] * u[(j * k) + i];
5927 }
5928 }
5929 for (double& l : length) {
5930 l = std::sqrt(l);
5931 }
5932 parallel_for(k, k, [&](std::size_t j) {
5933 for (std::size_t i {0}; i < k; ++i) {
5934 u[(j * k) + i] /= length[i];
5935 }
5936 });
5937 // updated = q(:, kept) · u, by halves of the rows: a half's rows take the kept columns that reach it, and
5938 // u's rows for those, the others being zero there.
5939 double* const updated {workspace_of(work.updated, n * k)};
5940 for (const auto& [half, first, rows] : {std::tuple {1U, std::size_t {0}, split}, std::tuple {2U, split, n - split}}) {
5941 std::vector<std::size_t> reaching;
5942 for (std::size_t i {0}; i < k; ++i) {
5943 if ((reach[kept[i]] & half) != 0U) {
5944 reaching.push_back(i);
5945 }
5946 }
5947 const std::size_t width {reaching.size()};
5948 double* const columns {workspace_of(work.columns, rows * width)};
5949 double* const factor {workspace_of(work.factor, width * k)};
5950 parallel_for(rows, width + k, [&](std::size_t r) {
5951 for (std::size_t c {0}; c < width; ++c) {
5952 columns[(r * width) + c] = q[((first + r) * ldq) + kept[reaching[c]]];
5953 }
5954 std::fill_n(updated + ((first + r) * k), k, 0.0); // the product adds to it
5955 });
5956 parallel_for(width, k, [&](std::size_t c) { std::copy_n(u + (reaching[c] * k), k, factor + (c * k)); });
5957 gemm(rows, k, width, columns, width, factor, k, updated + (first * k), k);
5958 }
5959 // Every eigenpair, then by ascending eigenvalue: column j of the result is column source[j] of the
5960 // updated ones (below k) or deflated column source[j] − k of q, written row by row.
5961 buffer<double> values(n, double {});
5962 for (std::size_t i {0}; i < k; ++i) {
5963 values[i] = roots[i];
5964 }
5965 for (std::size_t i {0}; i < deflated.size(); ++i) {
5966 values[k + i] = d[deflated[i]];
5967 }
5968 for (std::size_t j {0}; j < n; ++j) {
5969 order[j] = j;
5970 }
5971 std::sort(order.begin(), order.end(), [&values](std::size_t x, std::size_t y) { return values[x] < values[y]; });
5972 for (std::size_t j {0}; j < n; ++j) {
5973 d[j] = values[order[j]];
5974 }
5975 parallel_for(n, n, [&](std::size_t r) {
5976 const buffer<double> before(q + (r * ldq), q + (r * ldq) + n);
5977 for (std::size_t j {0}; j < n; ++j) {
5978 const std::size_t source {order[j]};
5979 q[(r * ldq) + j] = source < k ? updated[(r * k) + source] : before[deflated[source - k]];
5980 }
5981 });
5982 }
5983
5990 {
5991 double* d {};
5992 double* work_d {};
5993 const double* work_e {};
5994 std::size_t n {};
5995 double* q {};
5996 std::size_t ldq {};
5997 int exponent {0};
5998 buffer<double> scaled_d {};
5999 buffer<double> scaled_e {};
6000 std::size_t n1 {0};
6001 double beta {};
6002 std::size_t depth {};
6003 std::size_t first {};
6004 std::size_t second {};
6005 };
6006
6015 NUMERICAL_OUT_OF_LINE inline void plan_divide(std::vector<divide_node>& nodes,
6016 double* d,
6017 const double* e,
6018 std::size_t n,
6019 double* q,
6020 std::size_t ldq,
6021 std::size_t depth)
6022 {
6023 double largest {0.0};
6024 for (std::size_t i {0}; i < n; ++i) {
6025 largest = std::max({largest, std::abs(d[i]), i + 1 < n ? std::abs(e[i]) : 0.0});
6026 }
6027 divide_node node {.d = d, .work_d = d, .work_e = e, .n = n, .q = q, .ldq = ldq, .depth = depth};
6028 (void)std::frexp(largest, &node.exponent);
6029 if (node.exponent != 0) {
6030 node.scaled_d.assign(n, double {});
6031 node.scaled_e.assign(n, 0.0);
6032 for (std::size_t i {0}; i < n; ++i) {
6033 node.scaled_d[i] = std::ldexp(d[i], -node.exponent);
6034 node.scaled_e[i] = i + 1 < n ? std::ldexp(e[i], -node.exponent) : 0.0;
6035 }
6036 node.work_d = node.scaled_d.data(); // the buffers' storage stays put as the node moves into the vector
6037 node.work_e = node.scaled_e.data();
6038 }
6039 if (n <= divide_leaf) {
6040 nodes.push_back(std::move(node));
6041 return;
6042 }
6043 node.n1 = n / 2;
6044 node.beta = node.work_e[node.n1 - 1];
6045 node.work_d[node.n1 - 1] -= std::abs(node.beta);
6046 node.work_d[node.n1] -= std::abs(node.beta);
6047 double* const work_d {node.work_d};
6048 const double* const work_e {node.work_e};
6049 const std::size_t n1 {node.n1};
6050 const std::size_t index {nodes.size()};
6051 nodes.push_back(std::move(node));
6052 nodes[index].first = nodes.size();
6053 plan_divide(nodes, work_d, work_e, n1, q, ldq, depth + 1);
6054 nodes[index].second = nodes.size();
6055 plan_divide(nodes, work_d + n1, work_e + n1, n - n1, q + (n1 * ldq) + n1, ldq, depth + 1);
6056 }
6057
6061 {
6062 const std::size_t n {node.n};
6063 buffer<double> diagonal(node.work_d, node.work_d + n);
6064 buffer<double> sub(n, 0.0);
6065 std::copy(node.work_e, node.work_e + (n - 1), sub.begin());
6066 buffer<double> zt(n * n, 0.0);
6067 for (std::size_t i {0}; i < n; ++i) {
6068 zt[(i * n) + i] = 1.0;
6069 }
6070 tridiagonal_ql(diagonal, sub, zt, n, ql_steps_per_value * n);
6071 std::vector<std::size_t> order(n);
6072 for (std::size_t i {0}; i < n; ++i) {
6073 order[i] = i;
6074 }
6075 std::sort(order.begin(), order.end(), [&diagonal](std::size_t x, std::size_t y) { return diagonal[x] < diagonal[y]; });
6076 for (std::size_t j {0}; j < n; ++j) {
6077 node.work_d[j] = diagonal[order[j]];
6078 for (std::size_t r {0}; r < n; ++r) {
6079 node.q[(r * node.ldq) + j] = zt[(order[j] * n) + r];
6080 }
6081 }
6082 }
6083
6086 merge_workspace& work)
6087 {
6088 const std::size_t n {node.n};
6089 const std::size_t n1 {node.n1};
6090 const std::size_t n2 {n - n1};
6091 double* const q {node.q};
6092 buffer<double> z(n, double {});
6093 for (std::size_t i {0}; i < n1; ++i) {
6094 z[i] = q[((n1 - 1) * node.ldq) + i];
6095 }
6096 for (std::size_t i {0}; i < n2; ++i) {
6097 z[n1 + i] = std::copysign(1.0, node.beta) * q[(n1 * node.ldq) + n1 + i];
6098 }
6099 buffer<double> values(node.work_d, node.work_d + n);
6100 merge_rank_one(values, std::move(z), std::abs(node.beta), q, node.ldq, n, n1, work);
6101 std::copy(values.begin(), values.end(), node.work_d);
6102 }
6103
6106 {
6107 if (node.exponent != 0) {
6108 for (std::size_t i {0}; i < node.n; ++i) {
6109 node.d[i] = std::ldexp(node.work_d[i], node.exponent);
6110 }
6111 }
6112 }
6113
6115 NUMERICAL_OUT_OF_LINE inline void solve_divide_subtree(std::vector<divide_node>& nodes,
6116 std::size_t index,
6117 merge_workspace& work)
6118 {
6119 const divide_node& node {nodes[index]};
6120 if (node.n1 == 0) {
6121 solve_divide_leaf(node);
6122 }
6123 else {
6124 solve_divide_subtree(nodes, node.first, work);
6125 solve_divide_subtree(nodes, node.second, work);
6126 join_divide_node(node, work);
6127 }
6128 unscale_divide_node(node);
6129 }
6130
6147 inline void divide_and_conquer(double* d,
6148 const double* e,
6149 std::size_t n,
6150 buffer<double>& q)
6151 {
6152 q.assign(n * n, 0.0);
6153 std::vector<divide_node> nodes;
6154 plan_divide(nodes, d, e, n, q.data(), n, 0);
6155 const auto cut {static_cast<std::size_t>(std::bit_width(max_threads() - 1))};
6156 std::vector<std::size_t> tasks;
6157 for (std::size_t i {0}; i < nodes.size(); ++i) {
6158 if (nodes[i].depth == cut || (nodes[i].depth < cut && nodes[i].n1 == 0)) {
6159 tasks.push_back(i);
6160 }
6161 }
6162 const std::size_t task_order {nodes[tasks[0]].n};
6163 parallel_for(tasks.size(), tasks.size() > 1 ? task_order * task_order * task_order : 0, [&](std::size_t t) {
6164 solve_divide_subtree(nodes, tasks[t], thread_workspace());
6166 });
6167 for (std::size_t level {cut}; level-- > 0;) {
6168 std::vector<std::size_t> joins;
6169 for (std::size_t i {0}; i < nodes.size(); ++i) {
6170 if (nodes[i].n1 != 0 && nodes[i].depth == level) {
6171 joins.push_back(i);
6172 }
6173 }
6174 if (joins.empty()) { // a tree shallower than the cut
6175 continue;
6176 }
6177 if (joins.size() == 1) {
6178 join_divide_node(nodes[joins[0]], thread_workspace());
6180 unscale_divide_node(nodes[joins[0]]);
6181 continue;
6182 }
6183 const std::size_t order {nodes[joins[0]].n};
6184 parallel_for(joins.size(), order * order * order / 4, [&](std::size_t j) {
6185 join_divide_node(nodes[joins[j]], thread_workspace());
6186 release_large_workspace(thread_workspace());
6187 unscale_divide_node(nodes[joins[j]]);
6188 });
6189 }
6190 }
6191
6193 template <typename T>
6195 buffer<T>& zt,
6196 std::size_t n)
6197 {
6198 buffer<T> converted(n * n, T {});
6199 for (std::size_t i {0}; i < n * n; ++i) {
6200 converted[i] = from_carried<T>(left[i]);
6201 }
6202 buffer<T> product(n * n, T {});
6203 gemm(n, n, n, converted.data(), n, zt.data(), n, product.data(), n);
6204 zt = std::move(product);
6205 }
6206
6208 inline void multiply_real_left(const buffer<double>& left,
6209 buffer<double>& zt,
6210 std::size_t n)
6211 {
6212 buffer<double> product(n * n, 0.0);
6213 gemm(n, n, n, left.data(), n, zt.data(), n, product.data(), n);
6214 zt = std::move(product);
6215 }
6216
6218 inline void multiply_real_left(const buffer<double>& left,
6219 buffer<std::complex<double>>& zt,
6220 std::size_t n)
6221 {
6222 buffer<double> re(n * n, double {});
6223 buffer<double> im(n * n, double {});
6224 for (std::size_t i {0}; i < n * n; ++i) {
6225 re[i] = zt[i].real();
6226 im[i] = zt[i].imag();
6227 }
6228 multiply_real_left(left, re, n);
6229 multiply_real_left(left, im, n);
6230 for (std::size_t i {0}; i < n * n; ++i) {
6231 zt[i] = {re[i], im[i]};
6232 }
6233 }
6234
6241 template <typename T>
6243 buffer<double>& e,
6244 const tridiagonal_reduction<T>& reduction,
6245 std::size_t n)
6246 {
6247 if (n <= divide_leaf) {
6248 buffer<T> zt {tridiagonal_basis(reduction, n)};
6249 tridiagonal_ql(d, e, zt, n, ql_steps_per_value * n);
6250 return zt;
6251 }
6253 divide_and_conquer(d.data(), e.data(), n, q);
6254 buffer<T> at(n * n, T {}); // row j: the eigenvector of d[j], in the tridiagonal's basis, then the matrix's
6255 parallel_for(n, n, [&](std::size_t j) {
6256 for (std::size_t r {0}; r < n; ++r) {
6257 at[(j * n) + r] = from_carried<T>(q[(r * n) + j]);
6258 }
6259 });
6260 to_matrix_basis(reduction, at, n);
6261 return at;
6262 }
6263
6267 buffer<double>& v,
6268 std::size_t n,
6269 std::size_t p,
6270 std::size_t q)
6271 {
6272 const double apq {a[(p * n) + q]};
6273 const double app {a[(p * n) + p]};
6274 const double aqq {a[(q * n) + q]};
6275 const double theta {(aqq - app) / (2.0 * apq)};
6276 const double t {(theta >= 0.0 ? 1.0 : -1.0) / (std::abs(theta) + std::sqrt((theta * theta) + 1.0))};
6277 const double c {1.0 / std::sqrt((t * t) + 1.0)};
6278 const double s {t * c};
6279 a[(p * n) + p] = (c * c * app) - (2.0 * s * c * apq) + (s * s * aqq);
6280 a[(q * n) + q] = (s * s * app) + (2.0 * s * c * apq) + (c * c * aqq);
6281 a[(p * n) + q] = 0.0;
6282 a[(q * n) + p] = 0.0;
6283 for (std::size_t i {0}; i < n; ++i) {
6284 if (i != p && i != q) {
6285 const double aip {a[(i * n) + p]};
6286 const double aiq {a[(i * n) + q]};
6287 a[(i * n) + p] = (c * aip) - (s * aiq);
6288 a[(p * n) + i] = a[(i * n) + p];
6289 a[(i * n) + q] = (s * aip) + (c * aiq);
6290 a[(q * n) + i] = a[(i * n) + q];
6291 }
6292 }
6293 for (std::size_t i {0}; i < n; ++i) {
6294 const double vip {v[(i * n) + p]};
6295 const double viq {v[(i * n) + q]};
6296 v[(i * n) + p] = (c * vip) - (s * viq);
6297 v[(i * n) + q] = (s * vip) + (c * viq);
6298 }
6299 }
6300
6311 buffer<double>& v,
6312 std::size_t n,
6313 std::size_t max_sweeps)
6314 {
6315 // A pair is rotated while its element exceeds ε √|a_pp a_qq| (Demmel and Veselić's bound, which keeps each
6316 // eigenvalue's relative accuracy; a fixed fraction of the norm stopped four digits short), DBL_MIN where a
6317 // diagonal element is 0; the sweeps end when none is.
6318 for (std::size_t sweeps {0};; ++sweeps) {
6319 bool rotated {false};
6320 for (std::size_t p {0}; p < n; ++p) {
6321 for (std::size_t q {p + 1}; q < n; ++q) {
6322 const double bound {std::numeric_limits<double>::epsilon() * std::sqrt(std::abs(a[(p * n) + p])) * std::sqrt(std::abs(a[(q * n) + q]))};
6323 if (std::abs(a[(p * n) + q]) > std::max(bound, std::numeric_limits<double>::min())) {
6324 if (sweeps == max_sweeps) {
6325 throw convergence_error("symmetric eigendecomposition: Jacobi did not converge within its sweeps");
6326 }
6327 jacobi_rotate(a, v, n, p, q);
6328 rotated = true;
6329 }
6330 }
6331 }
6332 if (!rotated) {
6333 return;
6334 }
6335 }
6336 }
6337
6339 [[nodiscard]] inline std::vector<std::size_t> ascending_order(const buffer<double>& diagonal)
6340 {
6341 std::vector<std::size_t> order(diagonal.size());
6342 for (std::size_t i {0}; i < order.size(); ++i) {
6343 order[i] = i;
6344 }
6345 std::sort(order.begin(), order.end(), [&diagonal](std::size_t x, std::size_t y) { return diagonal[x] < diagonal[y]; });
6346 return order;
6347 }
6348
6354 template <typename T>
6355 [[nodiscard]] buffer<T> columns_by_value(const buffer<T>& zt,
6356 const std::vector<std::size_t>& order,
6357 std::size_t n)
6358 {
6359 buffer<T> out(n * n, T {});
6360 parallel_for((n + transpose_tile - 1) / transpose_tile, transpose_tile * n, [&](std::size_t block) {
6361 const std::size_t r0 {block * transpose_tile};
6362 const std::size_t r1 {std::min(r0 + transpose_tile, n)};
6363 for (std::size_t c0 {0}; c0 < n; c0 += transpose_tile) {
6364 for (std::size_t c {c0}; c < std::min(c0 + transpose_tile, n); ++c) {
6365 const T* const source {zt.data() + (order[c] * n)};
6366 for (std::size_t r {r0}; r < r1; ++r) {
6367 out[(r * n) + c] = source[r];
6368 }
6369 }
6370 }
6371 });
6372 return out;
6373 }
6374 } // namespace detail
6375
6376 namespace detail {
6377
6379 inline constexpr std::size_t pencil_block {64};
6380
6381
6388 template <typename T>
6390 std::size_t ldc,
6391 const T* l,
6392 std::size_t ldl,
6393 std::size_t n)
6394 {
6395 buffer<T> x(n); // the column below the diagonal, conjugated, then solved
6396 buffer<T> y(n); // L's, conjugated
6397 for (std::size_t k {0}; k < n; ++k) {
6398 const auto bkk {real_part(l[(k * ldl) + k])};
6399 const auto akk {real_part(c[(k * ldc) + k]) / (bkk * bkk)};
6400 c[(k * ldc) + k] = T {akk};
6401 const auto inverse {static_cast<carried_t<T>>(1) / bkk};
6402 const T half_akk {static_cast<carried_t<T>>(-0.5) * akk};
6403 for (std::size_t i {k + 1}; i < n; ++i) {
6404 c[(i * ldc) + k] = (c[(i * ldc) + k] * T {inverse}) + (half_akk * l[(i * ldl) + k]);
6405 x[i] = conj_of(c[(i * ldc) + k]);
6406 y[i] = conj_of(l[(i * ldl) + k]);
6407 }
6408 for (std::size_t i {k + 1}; i < n; ++i) { // C₂₂ −= x yᴴ + y xᴴ, x the column, y L's, row by row to the diagonal
6409 T* const row {c + (i * ldc)};
6410 subtract_product(row + k + 1, c[(i * ldc) + k], y.data() + k + 1, i - k);
6411 subtract_product(row + k + 1, l[(i * ldl) + k], x.data() + k + 1, i - k);
6412 }
6413 for (std::size_t i {k + 1}; i < n; ++i) { // L₂₂ z = x, by forward substitution, the column gathered
6414 x[i] = c[(i * ldc) + k] + (half_akk * l[(i * ldl) + k]);
6415 x[i] = (x[i] - dot_product(l + (i * ldl) + k + 1, x.data() + k + 1, i - k - 1)) / l[(i * ldl) + i];
6416 c[(i * ldc) + k] = x[i];
6417 }
6418 }
6419 }
6420
6427 template <typename T>
6428 [[nodiscard]] bool lower_triangle_product(std::size_t /*m*/,
6429 std::size_t /*depth*/,
6430 const T* /*a*/,
6431 const T* /*b*/,
6432 T* /*c*/,
6433 std::size_t /*ldc*/)
6434 {
6435 return false;
6436 }
6437
6439 [[nodiscard]] inline bool lower_triangle_product(std::size_t m,
6440 std::size_t depth,
6441 const double* a,
6442 const double* b,
6443 double* c,
6444 std::size_t ldc)
6445 {
6446 static_assert(2 * pencil_block >= gemm_min_depth);
6447 if (m * m < ((gemm_packed_threshold + depth) - 1) / depth) {
6448 return false;
6449 }
6450 gemm_triangle<a_rows, b_rows>(m, depth, a, depth, b, m, c, ldc, true);
6451 return true;
6452 }
6453
6463 template <typename T>
6464 void reduce_pencil(T* c,
6465 const T* l,
6466 std::size_t n)
6467 {
6470 buffer<T> turned;
6471 buffer<T> left;
6472 buffer<T> right;
6473 for (std::size_t k0 {0};; k0 += pencil_block) { // to the last block, whose rows below are none
6474 const std::size_t kb {std::min(pencil_block, n - k0)};
6475 const std::size_t k1 {k0 + kb};
6476 const std::size_t m {n - k1};
6477 reduce_pencil_columns(c + (k0 * n) + k0, n, l + (k0 * n) + k0, n, kb);
6478 if (m == 0) {
6479 break;
6480 }
6481 T* const c21 {c + (k1 * n) + k0};
6482 const T* const l21 {l + (k1 * n) + k0};
6483 // C₂₁ L₁₁⁻ᴴ: the rows of C₂₁ conjugated as columns, L₁₁ X = C₂₁ᴴ, turned back.
6484 for (std::size_t i {0}; i < kb; ++i) {
6485 for (std::size_t j {0}; j <= i; ++j) {
6486 diagonal[(i * kb) + j] = l[((k0 + i) * n) + k0 + j];
6487 }
6488 std::fill(diagonal.begin() + static_cast<std::ptrdiff_t>((i * kb) + i + 1), diagonal.begin() + static_cast<std::ptrdiff_t>((i + 1) * kb), T {});
6489 }
6490 turned.resize(kb * m);
6491 for (std::size_t r {0}; r < m; ++r) {
6492 for (std::size_t q {0}; q < kb; ++q) {
6493 turned[(q * m) + r] = conj_of(c21[(r * n) + q]);
6494 }
6495 }
6496 triangular_rows(diagonal.data(), kb, true, false, false, turned.data(), m);
6497 for (std::size_t r {0}; r < m; ++r) {
6498 for (std::size_t q {0}; q < kb; ++q) {
6499 c21[(r * n) + q] = conj_of(turned[(q * m) + r]);
6500 }
6501 }
6502 // −½ C₁₁, whole from its lower triangle.
6503 for (std::size_t i {0}; i < kb; ++i) {
6504 for (std::size_t j {0}; j < i; ++j) {
6505 const T value {static_cast<carried_t<T>>(-0.5) * c[((k0 + i) * n) + k0 + j]};
6506 half[(i * kb) + j] = value;
6507 half[(j * kb) + i] = conj_of(value);
6508 }
6509 half[(i * kb) + i] = T {static_cast<carried_t<T>>(-0.5) * real_part(c[((k0 + i) * n) + k0 + i])};
6510 }
6511 gemm(m, kb, kb, l21, n, half.data(), kb, c21, n);
6512 // C₂₂ −= C₂₁ L₂₁ᴴ + L₂₁ C₂₁ᴴ on the lower triangle: [−C₂₁ −L₂₁] [L₂₁ᴴ; C₂₁ᴴ].
6513 const std::size_t depth {2 * kb};
6514 left.resize(m * depth);
6515 right.resize(depth * m);
6516 for (std::size_t r {0}; r < m; ++r) {
6517 for (std::size_t q {0}; q < kb; ++q) {
6518 left[(r * depth) + q] = -c21[(r * n) + q];
6519 left[(r * depth) + kb + q] = -l21[(r * n) + q];
6520 right[(q * m) + r] = conj_of(l21[(r * n) + q]);
6521 right[((kb + q) * m) + r] = conj_of(c21[(r * n) + q]);
6522 }
6523 }
6524 T* const c22 {c + (k1 * n) + k1};
6525 const bool done {lower_triangle_product(m, depth, left.data(), right.data(), c22, n)};
6526 for (std::size_t r0 {0}; !done && r0 < m; r0 += tridiagonal_band) {
6527 const std::size_t rows {std::min(tridiagonal_band, m - r0)};
6528 gemm(rows, r0 + rows, depth, left.data() + (r0 * depth), depth, right.data(), m, c22 + (r0 * n), n);
6529 }
6530 gemm(m, kb, kb, l21, n, half.data(), kb, c21, n);
6531 }
6532 // Each block's L₂₂⁻¹ C₂₁, L₂₂ read where it lies: a block's rows below it are its own once the rank-2k update
6533 // has read them, so the blocks are solved together, the largest first, each on one thread.
6534 const std::size_t below {(std::max(n, std::size_t {1}) - 1) / pencil_block}; // the blocks with rows below them
6535 parallel_for(below, n * n * pencil_block / 2, [&](std::size_t index) {
6536 const std::size_t k0 {index * pencil_block};
6537 const std::size_t k1 {k0 + pencil_block};
6538 const std::size_t m {n - k1};
6539 buffer<T> minus((((m + 1) / 2) + triangular_block) * pencil_block);
6540 triangular_halves(l + (k1 * n) + k1, n, true, false, false, c + (k1 * n) + k0, n, pencil_block, 0, m, minus.data());
6541 });
6542 }
6543
6553 template <typename T>
6554 inline constexpr std::size_t pencil_reduction_least {is_complex_number<T> ? 3072 : 2048};
6555
6561 template <typename T>
6563 const T* lower,
6564 std::size_t n,
6565 bool blocks)
6566 {
6567 if (blocks) {
6568 reduce_pencil(y.data(), lower, n);
6569 for (std::size_t i {0}; i < n; ++i) {
6570 for (std::size_t j {0}; j < i; ++j) {
6571 y[(j * n) + i] = conj_of(y[(i * n) + j]);
6572 }
6573 }
6574 return y;
6575 }
6576 for (std::size_t i {0}; i < n; ++i) { // A whole
6577 for (std::size_t j {0}; j < i; ++j) {
6578 y[(j * n) + i] = conj_of(y[(i * n) + j]);
6579 }
6580 }
6581 solve_triangular_in_place(lower, n, true, triangular_op::none, false, y.data(), n); // L⁻¹ A
6582 buffer<T> c(n * n, T {});
6583 for (std::size_t i {0}; i < n; ++i) { // (L⁻¹ A)ᴴ = A L⁻ᴴ
6584 for (std::size_t j {0}; j < n; ++j) {
6585 c[(i * n) + j] = conj_of(y[(j * n) + i]);
6586 }
6587 }
6588 solve_triangular_in_place(lower, n, true, triangular_op::none, false, c.data(), n); // L⁻¹ A L⁻ᴴ
6589 return c;
6590 }
6591
6599 template <typename T>
6601 {
6608 const matrix<T>& b)
6609 : factor(checked_factor(a, b)),
6610 reduced(standard_form(a, factor))
6611 {}
6612
6614 [[nodiscard]] matrix<T> vectors_of(matrix<T>&& v) const
6615 {
6616 const std::size_t n {factor.rows()};
6617 const std::size_t k {v.cols()};
6618 buffer<T> x {std::move(v).elements()};
6619 solve_triangular_in_place(std::to_address(factor.begin()), n, true, triangular_op::adjoint, false, x.data(), k);
6620 return matrix<T> {n, k, std::move(x)};
6621 }
6622
6625
6626 private:
6627
6628 [[nodiscard]] static matrix<T> checked_factor(const matrix<T>& a,
6629 const matrix<T>& b)
6630 {
6631 if (a.rows() != a.cols() || b.rows() != b.cols() || b.rows() != a.rows()) {
6632 throw dimension_error("a definite pencil requires two square matrices of one order");
6633 }
6634 require_finite_lower(a, "a definite pencil");
6635 return cholesky_decomposition<T> {b}.lower();
6636 }
6637
6638 [[nodiscard]] static matrix<T> standard_form(const matrix<T>& a,
6639 const matrix<T>& l)
6640 {
6641 const std::size_t n {a.rows()};
6642 buffer<T> y(n * n, T {});
6643 for (std::size_t i {0}; i < n; ++i) { // A's lower triangle, a real diagonal
6644 for (std::size_t j {0}; j < i; ++j) {
6645 y[(i * n) + j] = a(i, j);
6646 }
6647 y[(i * n) + i] = T {real_part(a(i, i))};
6648 }
6649 return matrix<T> {n, n, pencil_standard_form(std::move(y), std::to_address(l.begin()), n, n >= pencil_reduction_least<T>)};
6650 }
6651 };
6652 } // namespace detail
6653
6655 enum class eigen_method
6656 {
6657 tridiagonal_ql,
6658 jacobi,
6659 divide_and_conquer
6660 };
6661
6675 {
6676 public:
6677
6685 explicit symmetric_eigen(const matrix<double>& input,
6686 eigen_method method = eigen_method::divide_and_conquer)
6687 : symmetric_eigen(matrix<double> {input},
6688 method)
6689 {}
6690
6700 eigen_method method = eigen_method::divide_and_conquer)
6701 : n_(input.rows())
6702 {
6703 if (input.cols() != n_) {
6704 throw dimension_error("symmetric eigendecomposition requires a square matrix");
6705 }
6706 detail::require_finite_lower(input, "symmetric eigendecomposition");
6707 // Work on the matrix divided by the power of two of its largest entry: a sum of squares of entries near 1e170
6708 // overflows and of entries near 1e-170 underflows to 0, and either stops the sweeps before a
6709 // single rotation. The eigenvalues are scaled back; the eigenvectors do not depend on it.
6710 double scale {0.0};
6711 for (std::size_t i {0}; i < n_; ++i) {
6712 for (std::size_t j {0}; j <= i; ++j) {
6713 scale = std::max(scale, std::abs(input(i, j)));
6714 }
6715 }
6716 scale = detail::power_of_two_scale(scale);
6717 buffer<double> a {std::move(input).elements()};
6718 buffer<double> diagonal(n_, double {});
6719 for (std::size_t i {0}; i < n_; ++i) { // the lower triangle divided; above it, whatever the input held
6720 for (std::size_t j {0}; j <= i; ++j) {
6721 a[(i * n_) + j] /= scale;
6722 }
6723 }
6724 if (method == eigen_method::jacobi) {
6725 buffer<double> v(n_ * n_, 0.0);
6726 for (std::size_t i {0}; i < n_; ++i) {
6727 v[(i * n_) + i] = 1.0;
6728 for (std::size_t j {i + 1}; j < n_; ++j) {
6729 a[(i * n_) + j] = a[(j * n_) + i]; // the lower triangle, mirrored
6730 }
6731 }
6732 detail::cyclic_jacobi(a, v, n_, detail::jacobi_sweeps);
6733 for (std::size_t i {0}; i < n_; ++i) {
6734 diagonal[i] = a[(i * n_) + i] * scale;
6735 }
6736 extract_sorted(diagonal, v);
6737 return;
6738 }
6739 for (std::size_t i {0}; i < n_; ++i) { // the lower triangle alone: the reduction reads nothing above it
6740 std::fill(a.begin() + static_cast<std::ptrdiff_t>((i * n_) + i + 1), a.begin() + static_cast<std::ptrdiff_t>((i + 1) * n_), 0.0);
6741 }
6742 buffer<double> zt;
6743 if (n_ > 0) {
6744 buffer<double> off(n_, double {});
6745 const detail::tridiagonal_reduction<double> reduction {detail::tridiagonalize(a, n_, diagonal, off)};
6746 if (method == eigen_method::tridiagonal_ql) {
6747 zt = detail::tridiagonal_basis(reduction, n_);
6748 detail::tridiagonal_ql(diagonal, off, zt, n_, detail::ql_steps_per_value * n_);
6749 }
6750 else {
6751 zt = detail::tridiagonal_divide_and_conquer(diagonal, off, reduction, n_);
6752 }
6753 }
6754 const std::vector<std::size_t> order {detail::ascending_order(diagonal)};
6755 buffer<double> values(n_, double {});
6756 for (std::size_t c {0}; c < n_; ++c) {
6757 values[c] = diagonal[order[c]] * scale;
6758 }
6759 values_ = vector<double> {std::move(values)};
6760 vectors_ = matrix<double> {n_, n_, detail::columns_by_value(zt, order, n_)};
6761 }
6762
6776 const matrix<double>& b,
6777 eigen_method method = eigen_method::divide_and_conquer)
6778 : symmetric_eigen(detail::definite_pencil<double> {a, b},
6779 method)
6780 {}
6781
6783 [[nodiscard]] const vector<double>& values() const noexcept
6784 {
6785 return values_;
6786 }
6787
6789 [[nodiscard]] const matrix<double>& vectors() const& noexcept
6790 {
6791 return vectors_;
6792 }
6793
6795 [[nodiscard]] matrix<double> vectors() &&
6796 {
6797 return std::move(vectors_);
6798 }
6799
6800 private:
6801
6804 eigen_method method)
6805 : symmetric_eigen(std::move(pencil.reduced),
6806 method)
6807 {
6808 vectors_ = pencil.vectors_of(std::move(vectors_));
6809 }
6810
6812 void extract_sorted(const buffer<double>& diagonal,
6813 const buffer<double>& v)
6814 {
6815 std::vector<std::size_t> order(n_);
6816 for (std::size_t i {0}; i < n_; ++i) {
6817 order[i] = i;
6818 }
6819 std::sort(order.begin(), order.end(),
6820 [&diagonal](std::size_t x, std::size_t y) {
6821 return diagonal[x] < diagonal[y];
6822 });
6823 buffer<double> values(n_, double {});
6824 buffer<double> vectors(n_ * n_, double {});
6825 for (std::size_t col {0}; col < n_; ++col) {
6826 values[col] = diagonal[order[col]];
6827 for (std::size_t row {0}; row < n_; ++row) {
6828 vectors[(row * n_) + col] = v[(row * n_) + order[col]];
6829 }
6830 }
6831 values_ = vector<double> {std::move(values)};
6832 vectors_ = matrix<double> {n_, n_, std::move(vectors)};
6833 }
6834
6835 std::size_t n_;
6838 };
6839
6851 {
6852 public:
6853
6865 explicit hermitian_eigen(const matrix<std::complex<double>>& input,
6866 eigen_method method = eigen_method::divide_and_conquer)
6867 : n_(input.rows())
6868 {
6869 if (method == eigen_method::jacobi) {
6870 throw argument_error("Hermitian eigendecomposition: the Jacobi method is for real symmetric matrices");
6871 }
6872 if (input.cols() != n_) {
6873 throw dimension_error("Hermitian eigendecomposition requires a square matrix");
6874 }
6875 detail::require_finite_lower(input, "Hermitian eigendecomposition");
6876 // Work on the matrix divided by the power of two of its largest entry, as symmetric_eigen does; the vectors do not depend
6877 // on it. The entries read: the lower triangle, and the diagonal's real part (its imaginary part is ignored,
6878 // so it scales nothing: an infinite one made every value NaN, a huge one wrong).
6879 double scale {0.0};
6880 for (std::size_t i {0}; i < n_; ++i) {
6881 for (std::size_t j {0}; j < i; ++j) {
6882 scale = std::max(scale, std::abs(input(i, j)));
6883 }
6884 scale = std::max(scale, std::abs(input(i, i).real()));
6885 }
6886 scale = detail::power_of_two_scale(scale);
6887 buffer<std::complex<double>> a(n_ * n_, std::complex<double> {});
6888 for (std::size_t i {0}; i < n_; ++i) { // the lower triangle alone, a real diagonal: the reduction reads no more
6889 for (std::size_t j {0}; j <= i; ++j) {
6890 a[(i * n_) + j] = (i == j ? std::complex<double> {input(i, j).real(), 0.0} : input(i, j)) / scale;
6891 }
6892 }
6893 buffer<double> diagonal(n_, double {});
6895 if (n_ > 0) {
6896 buffer<double> off(n_, double {});
6897 const detail::tridiagonal_reduction<std::complex<double>> reduction {detail::tridiagonalize(a, n_, diagonal, off)};
6898 if (method == eigen_method::tridiagonal_ql) {
6899 zt = detail::tridiagonal_basis(reduction, n_);
6900 detail::tridiagonal_ql(diagonal, off, zt, n_, detail::ql_steps_per_value * n_);
6901 }
6902 else {
6903 zt = detail::tridiagonal_divide_and_conquer(diagonal, off, reduction, n_);
6904 }
6905 }
6906 const std::vector<std::size_t> order {detail::ascending_order(diagonal)};
6907 buffer<double> values(n_, double {});
6908 for (std::size_t c {0}; c < n_; ++c) {
6909 values[c] = diagonal[order[c]] * scale;
6910 }
6911 values_ = vector<double> {std::move(values)};
6912 vectors_ = matrix<std::complex<double>> {n_, n_, detail::columns_by_value(zt, order, n_)};
6913 }
6914
6928 hermitian_eigen(const matrix<std::complex<double>>& a,
6929 const matrix<std::complex<double>>& b,
6930 eigen_method method = eigen_method::divide_and_conquer)
6931 : hermitian_eigen(detail::definite_pencil<std::complex<double>> {a, b},
6932 method)
6933 {}
6934
6936 [[nodiscard]] const vector<double>& values() const noexcept
6937 {
6938 return values_;
6939 }
6940
6942 [[nodiscard]] const matrix<std::complex<double>>& vectors() const& noexcept
6943 {
6944 return vectors_;
6945 }
6946
6949 {
6950 return std::move(vectors_);
6951 }
6952
6953 private:
6954
6956 hermitian_eigen(const detail::definite_pencil<std::complex<double>>& pencil,
6957 eigen_method method)
6958 : hermitian_eigen(pencil.reduced,
6959 method)
6960 {
6961 vectors_ = pencil.vectors_of(std::move(vectors_));
6962 }
6963
6964 std::size_t n_;
6967 };
6968} // namespace numerical
6969
6970#endif // NUMERICAL_FACTORIZATIONS_HPP
#define NUMERICAL_OUT_OF_LINE
Definition buffer.hpp:30
Thrown when an argument is outside what an operation takes (a tolerance below 0, an FFT type past 4).
Definition vector.hpp:87
A Cholesky factorisation A = L Lᴴ of a Hermitian positive-definite matrix.
matrix< T > solve(const matrix< T > &b) const
Solves A X = B for every column of b at once: L Y = B, then Lᴴ X = Y, row by row, each row of the sol...
cholesky_decomposition(matrix< T > &&a)
Factorises a in its own storage, an expiring matrix's elements taken over: no copy of them.
vector< T > solve(const vector< T > &b) const
Solves A x = b via L y = b then Lᴴ x = y.
std::size_t n_
The matrix order.
cholesky_decomposition(const matrix< T > &a)
Factorises a.
double reciprocal_condition(double one_norm) const
The reciprocal condition number 1 / (‖A‖₁ ‖A⁻¹‖₁) of the factorised matrix, as LAPACK's pocon: ‖A⁻¹‖₁...
matrix< T > lower() &&
The lower-triangular factor L, moved out of an expiring decomposition.
void factor_rows_together(std::size_t i, std::size_t k0)
The columns [k0, i) of the detail::cholesky_diagonal_rows rows from i, together: each element depends...
void factor_row(std::size_t i, std::size_t k0, std::size_t first, std::size_t last)
Row i of L in the columns [first, last), the terms of the columns before k0 already subtracted: the u...
T determinant() const
The determinant — the product of the squared diagonal of L (real, positive), its partial products kep...
void factor_rows_below(std::size_t k0, std::size_t end, buffer< T > &minus_adjoint)
The rows below the panel [k0, end), L₂₁ = A₂₁ L₁₁⁻ᴴ: chunks of detail::cholesky_rows rows in parallel...
matrix< T > lower() const &
The lower-triangular factor L (with L Lᴴ == A).
void update_trailing(std::size_t k0, std::size_t end, const buffer< T > &minus_adjoint, buffer< T > &packing)
A22 += L21 (−L21ᴴ), by tiles of detail::cholesky_update_columns columns and detail::cholesky_update_r...
buffer< T > lower_
The lower-triangular factor L, row-major (upper triangle zero).
Thrown when an iterative algorithm has not converged within its bound of sweeps.
convergence_error(const std::string &message)
Builds the error.
A standard exception Std that numerical::error tags too, its one what answering for both bases.
Definition vector.hpp:61
Thrown when an operation's operands have incompatible shapes.
Definition vector.hpp:76
The eigenvalues and eigenvectors of a complex Hermitian matrix.
hermitian_eigen(const detail::definite_pencil< std::complex< double > > &pencil, eigen_method method)
The pencil's eigenpairs: its standard form's, the vectors carried back.
matrix< std::complex< double > > vectors_
Eigenvectors as columns.
hermitian_eigen(const matrix< std::complex< double > > &input, eigen_method method=eigen_method::divide_and_conquer)
Diagonalises input.
const matrix< std::complex< double > > & vectors() const &noexcept
The eigenvectors as columns, ordered to match values.
hermitian_eigen(const matrix< std::complex< double > > &a, const matrix< std::complex< double > > &b, eigen_method method=eigen_method::divide_and_conquer)
Diagonalises the pencil A − λ B, A x = λ B x, as scipy.linalg.eigh(a, b): B = L Lᴴ and the Hermitian ...
std::size_t n_
The matrix order.
matrix< std::complex< double > > vectors() &&
The eigenvectors, moved out of an expiring decomposition.
vector< double > values_
Eigenvalues, ascending.
const vector< double > & values() const noexcept
The eigenvalues, ascending.
An LU factorisation with partial pivoting of a square matrix: each column's pivot the first entry of ...
T determinant() const
The determinant (the pivot sign times the product of U's diagonal; 0 if singular).
void swap_rows_outside(std::size_t a, std::size_t b, std::size_t k0, std::size_t end)
Swaps rows a and b of the factor outside the panel's columns [k0, end): right of the panel up to swap...
std::vector< std::pair< std::size_t, std::size_t > > deferred_swaps_
Swaps left for the columns past it.
matrix< T > factors() const &
L and U in one matrix, as LAPACK's dgetrf leaves them: U on and above the diagonal,...
void update_columns(const detail::packed_left< T > &left, std::size_t k0, std::size_t c0, std::size_t c1)
A₂₂ −= L₂₁ U₁₂ on the trailing columns [c0, c1), as A₂₂ += (−L₂₁) U₁₂ through the matrix product kern...
void solve_rows(buffer< T > &y, std::size_t cols, bool lower=false) const
Performs the pivoted elimination, filling lu_ and pivots_.
std::size_t column_slices(std::size_t cols) const
The column slices solve_rows splits cols right-hand sides into, one a thread, each of detail::slice_c...
const std::vector< std::size_t > & pivots() const noexcept
The row permutation: row i of P A = L U is row pivots()[i] of A.
void factorize_columns(buffer< T > &panel, std::size_t rows, std::size_t first, std::size_t last, std::size_t k0, std::size_t end)
Factorises the columns [first, last) of the panel (column-major, rows rows), every row from first dow...
lu_decomposition(const matrix< T > &a)
Factorises a (in place, into combined L\U with a row permutation).
std::size_t panel_of(std::size_t i) const noexcept
The first row of the panel that holds row i.
vector< T > solve(const vector< T > &b) const
Solves A x = b using the stored factors.
matrix< T > factors() &&
factors, moved out of an expiring decomposition: no copy.
double reciprocal_condition(double one_norm) const
The reciprocal condition number 1 / (‖A‖₁ ‖A⁻¹‖₁) of the factorised matrix, as LAPACK's gecon: ‖A⁻¹‖₁...
matrix< T > solve(matrix< T > &&b) const
solve on an expiring b, in its own storage: its rows permuted in place (cycle by cycle of the pivots,...
buffer< T > lu_
Combined L (below diag, unit) and U (on/above), row-major.
void solve_columns(std::size_t k0, std::size_t end, std::size_t first, std::size_t last)
The rows [k0, end) of U₁₂ on the columns [first, last), against the panel's unit-lower L: each column...
void negated_block_into(buffer< T > &out, std::size_t r0, std::size_t r1, std::size_t c0, std::size_t c1) const
negated_block into out, resized to it.
std::size_t lower_slice_start(std::size_t s, std::size_t slices, std::size_t cols) const
The first column of slice s of slices (slices itself: cols) where the forward pass is lower: column c...
matrix< T > pivoted_inverse(buffer< T > storage) const
(P A)⁻¹ = U⁻¹ L⁻¹, the inverse of the rows in their pivoted order, in storage (its elements discarded...
void transpose_panel(buffer< T > &panel, std::size_t rows, std::size_t k0, std::size_t end, bool in)
Copies the columns [k0, end) of the factor, every row from k0, into panel (column-major,...
std::vector< std::size_t > pivots_
Row permutation: row now at i came from pivots_[i].
void solve_panel_rows(std::size_t k0, std::size_t end)
U₁₂ = L₁₁⁻¹ A₁₂: the panel's rows right of it, by forward substitution with its unit-lower L....
void swap_left_columns()
Applies left_swaps_ to the columns left of each swap's panel: a panel's columns take,...
void subtract_block(buffer< T > &y, std::size_t cols, std::size_t r0, std::size_t r1, std::size_t c0, std::size_t c1, std::size_t first, std::size_t last) const
Y[r0, r1) −= F[r0, r1) × [c0, c1) · Y[c0, c1), F the factor, through the matrix product kernel (nothi...
std::size_t block_
The panel width.
matrix< T > inverse(buffer< T > storage) const
inverse in storage, its elements discarded: no fresh n × n mapped and faulted in where it holds that ...
int sign_
Permutation sign, for the determinant.
void solve_columns(buffer< T > &y, std::size_t cols, std::size_t c0, std::size_t c1, bool lower) const
solve_rows on the columns [c0, c1) of y (n × cols, row-major): the blocked passes,...
bool singular() const noexcept
Whether a pivot was zero: U is singular, and solve and inverse throw.
void factorize_panel(buffer< T > &panel, std::size_t k0, std::size_t end)
Unblocked LU with partial pivoting of the columns [k0, end), every row below k0.
std::pair< T, double > log_determinant() const
The determinant as its unit-modulus sign and the natural logarithm of its magnitude,...
std::size_t n_
The matrix order.
bool singular_
Set if a zero pivot was met.
std::size_t swap_limit_
The columns a panel's row swaps reach now.
void factorize_trailing(buffer< T > &panel, std::size_t first_end)
The panels after the first (whose columns end at first_end), each with its trailing update....
matrix< T > solve(const matrix< T > &b) const
Solves A X = B for every column of b at once.
std::vector< std::pair< std::size_t, std::size_t > > left_swaps_
Swaps left for the columns before their panel.
matrix< T > inverse() const
The inverse, U⁻¹ L⁻¹ P, by blocks of rows through the matrix product kernel: L⁻¹ from the identity,...
lu_decomposition(matrix< T > &&a)
Factorises a in its own storage, an expiring matrix's elements taken over: no copy of them.
void solve_narrow(buffer< T > &y, std::size_t cols) const
solve_rows for a few right-hand sides: row by row, each one's terms in the order the blocked passes g...
buffer< T > negated_block(std::size_t r0, std::size_t r1, std::size_t c0, std::size_t c1) const
The rows [r0, r1) and columns [c0, c1) of the factor, negated, row-major and contiguous.
A dense, row-major matrix of T with value semantics.
Definition matrix.hpp:36
buffer< T >::const_iterator end() const noexcept
Const iterator past the last element.
Definition matrix.hpp:139
buffer< T >::const_iterator begin() const noexcept
Const iterator to the first element (row-major).
Definition matrix.hpp:133
std::size_t rows() const noexcept
The number of rows.
Definition matrix.hpp:108
std::size_t cols() const noexcept
The number of columns.
Definition matrix.hpp:114
Thrown when a decomposition's input holds an infinite or NaN entry: no factorisation of it means anyt...
non_finite_error(const std::string &message)
Builds the error.
Thrown when a Cholesky factorisation finds the matrix not positive-definite.
std::size_t order() const noexcept
The order of the first leading minor found not positive definite (LAPACK's info), 0 where unknown.
std::size_t order_
The order of the failing leading minor, or 0.
not_positive_definite_error(const std::string &message, std::size_t order=0)
Builds the error.
A QR factorisation with column pivoting A P = Q R of an m × n matrix, as scipy.linalg....
pivoted_qr_decomposition(const matrix< T > &a, qr_mode mode=qr_mode::reduced)
Factorises a: Q m × k and R k × n (k = min(m, n)), or Q m × m and R m × n (mode complete).
std::vector< std::size_t > permutation_
P, by its columns' origins.
const matrix< T > & r() const noexcept
R, upper trapezoidal, its diagonal real, non-negative and not increasing: k × n, or m × n.
const matrix< T > & q() const noexcept
Q, orthonormal columns: m × k, or m × m complete.
std::size_t factor_block(buffer< T > &w, std::size_t m, std::size_t n, std::size_t start, std::size_t width, buffer< double > &current, buffer< double > &reference, std::vector< buffer< T > > &reflectors, std::vector< T > &taus)
Up to width steps from column start of the column-major m × n w, as LAPACK's laqps: the reflections k...
const std::vector< std::size_t > & permutation() const noexcept
The column permutation: column j of A P is column permutation()[j] of A, as SciPy's P.
A QR factorisation A = Q R of an m × n matrix, as NumPy's qr, reduced by default.
vector< T > solve(const vector< T > &b) const
The least-squares solution of A x ≈ b (x = R⁻¹ Qᴴ b) for a full-rank A.
std::size_t depth_
min(m, n).
qr_decomposition(matrix< T > &&a, qr_mode mode=qr_mode::reduced)
Factorises a in its own storage, an expiring matrix's elements taken over: no copy of them.
std::size_t cols_
The number of columns (n).
qr_decomposition(const matrix< T > &a, qr_mode mode=qr_mode::reduced)
Factorises a.
std::size_t width_
Q's columns and R's rows: k, or m complete.
std::size_t rows_
The number of rows (m).
matrix< T > q() &&
Q, moved out of an expiring decomposition (r stays).
matrix< T > q() const &
The factor Q (orthonormal columns): m × k, k = min(m, n), or m × m complete.
buffer< T > q_
The factor Q, row-major, m × width_.
buffer< T > r_
The upper-trapezoidal factor R, row-major, width_ × n.
matrix< T > r() &&
R, moved out of an expiring decomposition (q stays).
matrix< T > solve(const matrix< T > &b) const
The least-squares solutions of A X ≈ B (X = R⁻¹ Qᴴ B) for a full-rank A, every column of b at once: Q...
matrix< T > r() const &
The upper-trapezoidal factor R: k × n, k = min(m, n), or m × n complete.
The magnitude at or under which a singular value counts as zero, in a rank (matrix_rank),...
constexpr rank_cutoff() noexcept=default
NumPy's default: the largest singular value times the machine epsilon times the larger dimension.
static rank_cutoff absolute(double magnitude)
magnitude itself, whatever the largest singular value.
static double checked(double value)
constexpr rank_cutoff(double value, bool absolute) noexcept
rank_cutoff scaled(int exponent) const noexcept
The same cutoff for the matrix divided by 2^exponent.
double magnitude(double largest, std::size_t rows, std::size_t cols) const noexcept
The cutoff for a rows × cols matrix whose largest singular value is largest.
Thrown when a matrix has no unique solution (it is singular).
singular_matrix_error(const std::string &message)
Builds the error.
The eigenvalues and eigenvectors of a real symmetric matrix.
matrix< double > vectors_
Eigenvectors as columns.
const matrix< double > & vectors() const &noexcept
The eigenvectors as columns, ordered to match values.
matrix< double > vectors() &&
The eigenvectors, moved out of an expiring decomposition.
void extract_sorted(const buffer< double > &diagonal, const buffer< double > &v)
Sorts the eigenvalues diagonal ascending and reorders v's columns to match.
symmetric_eigen(detail::definite_pencil< double > pencil, eigen_method method)
The pencil's eigenpairs: its standard form's, the vectors carried back.
symmetric_eigen(matrix< double > &&input, eigen_method method=eigen_method::divide_and_conquer)
Diagonalises input in its own storage, an expiring matrix's elements taken over: its scaled copy is m...
symmetric_eigen(const matrix< double > &input, eigen_method method=eigen_method::divide_and_conquer)
Diagonalises input.
vector< double > values_
Eigenvalues, ascending.
std::size_t n_
The matrix order.
symmetric_eigen(const matrix< double > &a, const matrix< double > &b, eigen_method method=eigen_method::divide_and_conquer)
Diagonalises the pencil A − λ B, A x = λ B x, as scipy.linalg.eigh(a, b): B = L Lᴴ and the symmetric ...
const vector< double > & values() const noexcept
The eigenvalues, ascending.
A dense vector of T with value semantics.
Definition vector.hpp:171
buffer< T >::const_iterator end() const noexcept
Const iterator past the last element.
Definition vector.hpp:212
std::size_t size() const noexcept
The number of elements.
Definition vector.hpp:194
buffer< T >::const_iterator begin() const noexcept
Const iterator to the first element.
Definition vector.hpp:206
The scalar types NumericAL's factorisations and solvers take: a field of floating-point numbers,...
#define NUMERICAL_UNROLL_BLOCK
Definition gemm.hpp:72
A dense, row-major, value-semantic matrix over a scalar type.
constexpr std::size_t gemm_packed_threshold
Below this many multiply-adds the plain loop (gemm_rows, by blocks of rows held in registers) runs in...
Definition gemm.hpp:909
T symmetric_row(const T *row, const T *v, const T &f, T *out, std::size_t count)
Σ row[i] · v[i] and out[i] += row[i] · f, in order, for a type no native vector holds.
Definition simd.hpp:1048
constexpr std::size_t lu_tall_ratio
The ratio of rows to columns up to which lu_factors factorises a tall matrix as the square of it besi...
constexpr std::size_t lu_lookahead_columns
The trailing columns at most past which the LU factorises its next panel beside the update of the res...
constexpr std::size_t qr_transpose_tile
The side of the tiles in which the QR transposes (transpose_into): a tile's rows and columns both sta...
constexpr std::size_t qr_update_slice
The fewest columns a task of reflect_in_columns' look-ahead updates: each task repacks the block,...
constexpr carried_t< T > rescue_below
The magnitude under which rescue_factor scales a T up: about the square root of the smallest normal n...
constexpr std::size_t lu_block_for(std::size_t n) noexcept
The panel width of the LU of order n: a power of two, so a row's panel is a mask away,...
bool on_release(void(*step)()) noexcept
Registers step among the calling thread's thread_releases, while a place is left (a workspace past th...
Definition buffer.hpp:65
double reciprocal_condition(std::size_t n, double one_norm, const std::function< void(T *, bool)> &apply)
The reciprocal condition number 1 / (‖A‖₁ ‖A⁻¹‖₁) of A from one_norm, ‖A‖₁, and an estimate of ‖A⁻¹‖₁...
void factor_qr_panel(T *panel, std::size_t ld, std::size_t top, std::vector< buffer< T > > &reflectors, std::vector< T > &taus, std::size_t k0, std::size_t k1)
Householder reflections of the columns k0…k1-1 of a column-major matrix, rows from k0,...
constexpr std::size_t cholesky_update_rows
The rows of one tile of a Cholesky update: the tiles of a panel's update, each one product on one thr...
double contiguous_norm(const T *x, std::size_t count)
The 2-norm of the count entries at x, scaled by the largest so no square overflows or underflows.
T scaled_product(T start, std::size_t count, const Factor &factor)
The product of count factors, factor(i) for each i, times start, as a determinant multiplies its pivo...
R part_size(R x)
The magnitude of a real x, x's exponent read from it.
bool all_finite_shared(const double *x, std::size_t count)
all_finite on the threads by chunks of finite_chunk, past two of them: a pass bound by the memory,...
double rotation_length(double a, double b) noexcept
√(a² + b²) for a plane rotation of the QR and QL iterations, as LAPACK's dlartg takes it: the plain s...
constexpr std::size_t cholesky_block
The columns of one panel of the blocked Cholesky factorisation (as lu_block).
void parallel_for_beside(std::size_t count, std::size_t work_per_index, const Body &body, const Side &side)
Runs side on the caller while the pool's workers run body(i) for every i in [0, count),...
Definition parallel.hpp:827
bool same_sign(R x, int sign) noexcept
Whether a real entry's sign is sign (sign_entry's), as dlacn2 finds a sign vector repeated.
bool triangle_is_finite(const T *a, std::size_t n, bool lower)
Whether the triangle of the row-major n × n a a solve reads (the diagonal with it) is finite,...
R sign_entry(R x, int &sign) noexcept
A real entry's sign for one_norm_estimate's sign vectors, dlacn2's: 1 from zero up,...
bool lower_is_finite(const matrix< T > &a)
Whether the lower triangle of the square a, and its diagonal's real part, are finite: entry by entry,...
void tridiagonal_root_free_ql(buffer< double > &d, buffer< double > &e, std::size_t n, std::size_t max_steps)
The eigenvalues of the symmetric tridiagonal (d, e), no vector, as LAPACK's dsterf: the Pal–Walker–Ka...
void form_q_into(buffer< T > &out, const std::vector< reflector_block< T > > &blocks, std::size_t rows, std::size_t first, std::size_t count, bool transposed)
The columns [first, first + count) of Q = H₀ H₁ …, the blocks applied (the last first) to those of th...
constexpr std::size_t lu_update_speedup
The multiply-adds of an LU's trailing update one of the tile's dependent ones stands for,...
void subtract_corrections(T *y, const T *f, const T *x, std::size_t ldx, std::size_t terms, std::size_t count)
subtract_products of a real T, where the reduction's corrections are those of its type.
void apply_q(buffer< T > &p, const std::vector< reflector_block< T > > &blocks, std::size_t rows, std::size_t count)
P ← Q P for the rows × count row-major p, Q = H₀ H₁ … the product of the blocks' reflectors,...
void add_conjugate_product(T *y, const T *x, const T &f, std::size_t count)
y[i] += conj(x[i]) · f for a real T, where the conjugate is the value: add_product.
void subtract_rows(double *y, std::size_t ldy, const double *f, std::size_t ldf, const double *x, std::size_t ldx, std::size_t terms, std::size_t count)
subtract_rows for double, the units chosen as subtract_products chooses them: each element the value ...
Definition gemm.hpp:551
@ plain
gemm_rows, or a scalar whose products are not shared: A is read in place.
T conjugate_dot_product(const T *x, const T *y, std::size_t count)
Σ conj(x[i]) · y[i], in order, where no native vector holds the parts.
Definition simd.hpp:1089
std::optional< matrix< double > > mixed_precision_solve(const matrix< double > &a, const matrix< double > &b, double *reciprocal_condition=nullptr)
X of A X = B (A square, B of its rows) as LAPACK's dsgesv: A factorised in single precision,...
constexpr std::size_t cholesky_together_width
The narrowest Cholesky diagonal block whose rows go together: in a block of 8, the groups' loops cost...
void reduce_pencil(T *c, const T *l, std::size_t n)
The lower triangle of the Hermitian n × n c (row-major) taken to L⁻¹ C L⁻ᴴ for the lower-triangular l...
double one_norm(const buffer< T > &a, std::size_t n)
The 1-norm of the n × n row-major a (n from 1): its largest column sum of moduli.
constexpr double plain_squares_low
The smallest plain sum of squares plain_or_at_scale keeps: a term that underflowed is below 2⁻¹⁰²²,...
constexpr std::size_t gemm_kernel_speedup
The multiply-adds the register tile does in the time of one in work_per_thread's dependent chain (abo...
Definition gemm.hpp:893
constexpr std::size_t symmetric_row_group
The rows of a lower triangle symmetric_rows takes at once.
Definition simd.hpp:158
symmetric_two_by_two symmetric_eigen_2x2(double a, double b, double c) noexcept
The eigensystem of [a b; b c] as LAPACK's dlaev2 computes it: the larger eigenvalue in magnitude from...
std::size_t parallel_threads(std::size_t count, std::size_t work_per_index)
The threads parallel_for shares count indices of work_per_index out to (threads_for): the hardware's,...
Definition parallel.hpp:766
std::complex< R > scaled(R c, const std::complex< R > &v) noexcept
c · v of a real c and a complex v: each part's product, rounded.
Definition fft.hpp:1290
constexpr std::size_t divide_leaf
The order up to which tridiagonal_divide_and_conquer diagonalises by implicit QL.
carried_t< T > real_part(const T &x)
The real part of a real (or general) scalar, in carried_t.
void gemm_at(std::size_t m, std::size_t n, std::size_t k, const T *at, std::size_t ldat, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
gemm_layouts with A given as Aᵀ (at, k × m, ldat apart).
Definition gemm.hpp:4787
std::pair< matrix< T >, matrix< T > > split_lu(const T *lu, std::size_t rows, std::size_t cols, std::size_t k)
The rows of lu (rows × cols row-major, L below its diagonal, U on and above) as L,...
T square(const T &x)
x · x, by the type's own product (a complex one's C Annex G, as the library's *).
T operator_times(const T &a, const T &b)
a · b, as std::operator* computes it, in line. libc++'s complex product (the four products and their ...
Definition simd.hpp:1122
constexpr std::size_t vector_speedup
The multiply-adds a vectorised inner loop does in the time of one in work_per_thread's dependent chai...
Definition parallel.hpp:739
void merge_rank_one(buffer< double > &d, buffer< double > z, double rho, double *q, std::size_t ldq, std::size_t n, std::size_t split, merge_workspace &work)
The eigenvalues of diag(d) + ρ z zᵀ, ascending, into d, and its eigenvectors times q (the row-major n...
void require_finite_triangle(const T *a, std::size_t n, bool lower, const T *x, std::size_t count)
Refuses an infinite or NaN entry of the triangle of a a triangular solve reads, or of its right-hand ...
constexpr std::size_t slice_columns
The fewest columns an LU solve gives a thread of its own: a narrower slice starves the product kernel...
constexpr std::size_t cholesky_update_columns
The fewest columns of the trailing block one Cholesky update's product takes: each product packs ever...
void add_product(double *y, const double &f, const double *x, std::size_t count)
y[i] += f · x[i] for double, as subtract_product.
Definition gemm.hpp:648
constexpr std::size_t jacobi_sweeps
The sweeps a Jacobi method may take: it converges quadratically, in under 15 on every matrix measured...
R times_power_of_two(R x, int e)
x times 2ᵉ, rounded once (a power of 2 scales a normal number exactly).
std::array< double, symmetric_row_group >(*)(const double *, std::size_t, const double *, double *, std::size_t) symmetric_rows_kernel
A kernel of symmetric_rows' form: symmetric_rows itself, or an x86-64 one.
void subtract_products(double *y, const double *f, const double *x, std::size_t ldx, std::size_t terms, std::size_t count)
y[i] -= Σₜ f[t] · x[(t · ldx) + i] for double, the units chosen as subtract_product chooses them,...
Definition gemm.hpp:522
void require_finite(const matrix< T > &a, const char *what)
The one non-finite policy, checked where every decomposition starts.
T hermitian_row(symmetric_row_kernel, const T *row, const T *v, T *out, std::size_t i)
Row i's terms of a Hermitian product from the lower triangle: out[j] += conj(row[j]) v[i] for j < i,...
constexpr std::size_t tridiagonal_block
The reflections of one panel of the blocked tridiagonal reduction.
void tridiagonal_ql(buffer< double > &d, buffer< double > &e, buffer< T > &zt, std::size_t n, std::size_t max_steps)
The eigenvalues of the symmetric tridiagonal (d, e) by implicit QL or QR as LAPACK's dsteqr,...
carried_t< T > rescue_factor(carried_t< T > largest)
The exact power of two that brings a magnitude below about the square root of the smallest normal num...
constexpr std::size_t work_per_thread
The work one thread must have to pay for starting it: about 60 µs of multiply-adds,...
Definition parallel.hpp:729
double secular_root(const buffer< double > &d, const buffer< double > &z, double rho, std::size_t i, std::size_t max_iterations, double *delta)
Root i of 1/ρ + Σⱼ zⱼ²/(dⱼ − λ) = 0 (d strictly ascending, ‖z‖ = 1), the i-th eigenvalue of diag(d) +...
void parallel_copy(const T *in, std::size_t count, T *out)
Copies the count elements at in to out, split across the threads the work pays for,...
Definition parallel.hpp:930
buffer< T > tridiagonal_basis(const tridiagonal_reduction< T > &reduction, std::size_t n)
Mᵀ of a reduction, row-major (its rows are the columns of M): Qᵀ by blocks of reflectors from the las...
void reflect_columns(T *panel, std::size_t height, const buffer< T > &v, std::size_t first, std::size_t from, std::size_t to, const T &s=T {2})
Applies I − s v vᴴ from the left (s 2 for a unit v) to the columns from…to-1 of the column-major pane...
constexpr std::size_t ql_steps_per_value
The implicit QL steps the tridiagonal eigensolver may take per eigenvalue, as LAPACK's dsteqr bounds ...
void gemm_with(const packed_left< T > &left, std::size_t n, const T *b, std::size_t ldb, T *c, std::size_t ldc)
C += A·B for the packed left (pack_left) and the k × n b, as gemm computes it, A read in place here,...
Definition gemm.hpp:4992
T conj_of(const T &x)
The conjugate of a real (or general) scalar — the value itself.
std::vector< rotation_run > rotation_runs(const std::vector< row_operation > &operations)
The operations split into runs of chained rotations, every other operation a run of one.
constexpr std::size_t kept_workspace
The most elements a buffer of a thread's merge_workspace keeps from one decomposition to the next: 2 ...
constexpr std::size_t cholesky_block_for(std::size_t n) noexcept
The panel width of the Cholesky factorisation of order n: narrower up to order 256,...
double cholesky_reciprocal_condition(const T *factor, std::size_t n, bool lower, double one_norm)
reciprocal_condition of A = L Lᴴ or Uᴴ U from its Cholesky factor, L in the lower triangle of the n ×...
buffer< T > pencil_standard_form(buffer< T > y, const T *lower, std::size_t n, bool blocks)
C = L⁻¹ A L⁻ᴴ whole, y holding A's lower triangle (n × n, row-major, the diagonal real) and lower L: ...
void factor_row_major_panel(buffer< T > &a, std::size_t rows, std::size_t cols, std::vector< buffer< T > > &reflectors, std::vector< T > &taus, std::size_t k0, std::size_t k1)
factor_qr_panel on the columns k0…k1-1 of the row-major a, rows from k0, in a column-major copy of th...
constexpr std::size_t lu_wide_block
The columns of one panel of the LU from lu_wide_order.
constexpr std::size_t tridiagonal_band
The rows of each band of the tridiagonal reduction's trailing update: a band reaches its diagonal,...
double triangular_reciprocal_condition(const T *a, std::size_t n, bool lower)
reciprocal_condition of the triangular A, the lower triangle of the n × n row-major a or its upper on...
void tall_lu_columns(T *panel, std::size_t m, std::size_t n, std::size_t first, std::size_t last, std::vector< std::size_t > &rows)
Factorises the columns [first, last) of the m × n column-major panel (m ≥ n; column j at panel[j m]) ...
constexpr std::size_t narrow_solve
The most right-hand sides an LU solve takes row by row rather than by blocks through the product kern...
constexpr std::size_t lu_small_order
The order up to which the LU takes panels of lu_small_block columns: there the panels,...
void transpose_into(const T *from, std::size_t rows, std::size_t cols, T *to)
The row-major rows × cols from into to as its transpose, cols × rows row-major, by tiles,...
tridiagonal_reduction< T > tridiagonalize(buffer< T > &a, std::size_t n, buffer< double > &d, buffer< double > &e)
Reduces the Hermitian (real: symmetric) n × n row-major a, of which the lower triangle is read,...
@ tiles
The x86-64 complex tiles': each part's product from zero, re·re − im·im and re·im + im·re,...
@ split
arm64's real kernel's (gemm_split): Re a·Re b from zero, −Im a·Im b fused into it; Re a·Im b,...
constexpr std::size_t pencil_block
The columns reduce_pencil reduces a block at a time, as LAPACK's sygst by its block size.
void subtract_product(double *y, const double &f, const double *x, std::size_t count)
y[i] −= f · x[i] for double on the widest x86-64 units the matrix product's kernel may use (gemm_path...
Definition gemm.hpp:485
void require_finite_lower(const matrix< T > &a, const char *what)
require_finite over what a Hermitian eigensolver reads: the lower triangle, and the diagonal's real p...
qr_reflections< T > householder_reflect(buffer< T > a, std::size_t rows, std::size_t cols, bool reflectors=true)
The reflections of householder_qr, without forming Q: A = Q R with Q the product of the blocks' refle...
void solve_divide_subtree(std::vector< divide_node > &nodes, std::size_t index, merge_workspace &work)
Solves the block index of nodes and everything below it, each block right after its halves.
packed_left< T > pack_left(std::size_t m, std::size_t, std::size_t k, const T *a, std::size_t lda, buffer< T > &)
The m × k a packed once for products with it of about n columns each (gemm_with), in storage (grown a...
Definition gemm.hpp:4938
std::pair< matrix< float >, int > scaled_to_float(const matrix< double > &m)
The finite m times 2⁻ᵉ in single precision, 2ᵉ the least power of two above its largest magnitude: ev...
magnitude_t< T > pivot_magnitude(const T &x)
The magnitude partial pivoting compares: |x| of a real x.
constexpr std::size_t finite_chunk
The elements all_finite_shared gives a thread at a time.
std::array< T, symmetric_row_group > symmetric_rows(const T *rows, std::size_t n, const T *v, T *out, std::size_t i)
symmetric_rows one column at a time, for a type no native vector holds.
Definition simd.hpp:1065
double rank_tolerance(double largest, std::size_t rows, std::size_t cols)
The magnitude under which a diagonal entry of R or a singular value counts as zero,...
double quadratic_step(double a, double b, double c)
The smaller root η of C η² − A η + B = 0, in the form that does not cancel: a step of solve_secular's...
void triangular_halves(const T *a, std::size_t n, bool lower, bool transposed, bool unit, T *x, std::size_t ldx, std::size_t k, std::size_t first, std::size_t last, T *minus)
A X = B in place in x (n × k, row-major), for the n × n row-major triangle of a, its lower one or its...
std::conditional_t< std::is_same_v< T, float >||std::is_same_v< T, std::complex< float > >, float, carried_t< T > > value_range_t
The real type whose range a T's values take: float for a single-precision scalar, real or complex; ca...
double * workspace_of(workspace_buffer &buffer, std::size_t size)
buffer at least size elements long, its storage kept for the next join. A larger one is mapped fresh,...
reflector_block< T > gather_reflectors(const std::vector< buffer< T > > &reflectors, std::size_t k0, std::size_t k1, std::size_t rows, const std::vector< T > *taus=nullptr)
Gathers the reflectors reflectors[k0…k1-1] (each I − τ_k v vᴴ: τ_k from taus, else 2 for a unit v; an...
void subtract_term(T &sum, const T &a, const T &b)
sum −= a · b in one statement, which the compiler may fuse (as it fuses the matrix solve's).
void apply_reflector_block(const reflector_block< T > &block, bool adjoint, buffer< T > &a, std::size_t cols, std::size_t from, std::size_t to)
Applies I − V T Vᴴ (or its adjoint I − V Tᴴ Vᴴ) from the left to the block's rows of a row-major arra...
double plain_or_at_scale(double plain, const T *x, std::size_t count)
√plain, the plain sum of the squares of the count elements at x, where it neither overflowed nor unde...
buffer< T > columns_by_value(const buffer< T > &zt, const std::vector< std::size_t > &order, std::size_t n)
The n × n row-major matrix whose column c is row order[c] of zt (the eigenvectors as rows): by tiles,...
void parallel_for(std::size_t count, std::size_t work_per_index, const Body &body)
Runs body(i) for every i in [0, count).
Definition parallel.hpp:790
constexpr std::size_t lu_fused_columns
The trailing columns at most past which each block of the update beside the panel solves its own rows...
bool climb_ends(R last, R largest) noexcept
Whether the climb's last column last is as large as the new largest entry largest,...
void reflect_rows(buffer< T > &a, std::size_t cols, const buffer< T > &v, std::size_t first, std::size_t from, const T &s=T {2})
Applies I − s v vᴴ from the left (s 2 for a unit v) to the rows first…first+|v|-1 of a row-major arra...
void hermitian_product(const T *block, std::size_t n, std::size_t m, const T *v, T *w, buffer< T > &scratch)
w = B v for the Hermitian m × m block (row stride n) from its lower triangle alone,...
decltype(std::abs(std::declval< const T & >())) magnitude_t
The type std::abs gives for a T: the real type of a complex one, T itself otherwise.
constexpr std::size_t lu_wide_order
The order from which the LU takes panels of lu_wide_block columns: each trailing update rereads the w...
constexpr bool real_rows
Whether a solve of T goes by rows of real terms (lu_decomposition's solve_narrow): double and float.
T over_pivot(const T &x, const T &divisor)
x over the pivot whose pivot_divisor is divisor: the same quotient wherever the LU takes it.
gemm_path gemm_path_current() noexcept
The path the next double product takes: the most capable one under the cap that runs here (each path'...
Definition gemm.hpp:148
std::vector< std::size_t > ascending_order(const buffer< double > &diagonal)
The indices of diagonal by ascending value.
void solve_divide_leaf(const divide_node &node)
Solves the leaf node by implicit QL: its eigenvalues ascending into its diagonal, their eigenvectors ...
void reduce_pencil_columns(T *c, std::size_t ldc, const T *l, std::size_t ldl, std::size_t n)
The lower triangle of the n × n c (row stride ldc) taken to L⁻¹ C L⁻ᴴ for the lower-triangular l (row...
void release_large_workspace(merge_workspace &work)
Releases the buffers of work longer than kept_workspace.
bool elements_finite(const T *x, std::size_t count)
Whether the count elements at x are finite, one at a time: a type no vectorised pass takes.
constexpr std::size_t qr_look_ahead_threads
The fewest threads for which reflect_in_columns looks ahead, a panel factorised while the threads upd...
plane_rotation_real plane_rotation_to_zero(double f, double g) noexcept
The rotation that takes (f, g) to (r, 0), as LAPACK's dlartg: r of f's sign, c 0 or more; none for g ...
constexpr std::size_t lu_small_block
The columns of one panel of the LU of an order up to lu_small_order.
double real_subdiagonal(double t, double, double &)
A real subdiagonal entry of the tridiagonal: kept with its sign (the rotations take any sign).
std::pair< T, T > larfg_in_place(buffer< T > &w)
The reflector LAPACK's dlarfg (zlarfg) forms of w, in place: H = I − τ w wᴴ with w₀ = 1 and Hᴴ x = β ...
void subtract_panel_columns(T *column, const T *columns, std::size_t height, const T *l, std::size_t terms)
column −= Σₜ x_t · conj(l[t]) over the terms columns columns (height elements each,...
constexpr std::size_t cholesky_one_panel
The order below which the Cholesky factorisation is one panel: no trailing product,...
carried_t< T > rescale_tiny(buffer< T > &x)
Scales x by rescue_factor of its largest magnitude, exactly, so that a Householder vector built from ...
bool is_finite_entry(const T &x)
Whether x is finite.
constexpr std::size_t first_touch_chunk
The elements a task of workspace_of touches first: 64 pages of 4 KiB, so a buffer of a few megabytes ...
tensor< To > converted(const tensor< From > &t)
A copy of t, each element converted to To (between a signed integer type and its unsigned one,...
Definition tensor.hpp:2413
void gemm(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
C += A·B, every element of C summed in ascending order of the depth.
Definition gemm.hpp:4583
void apply_row_operations(buffer< T > &m, std::size_t n, const std::vector< row_operation > &operations)
Applies the recorded operations, in order, to the row-major m (n per row). Each column takes them ind...
constexpr std::size_t solve_leaf
The columns of one leaf of the recursive LU panel.
bool all_finite(const R *x, std::size_t count) noexcept
Whether the count values at x are all finite: a NaN or an infinity has every exponent bit set,...
Definition simd.hpp:1392
void triangular_rows(const T *a, std::size_t n, bool lower, bool transposed, bool unit, T *x, std::size_t k)
M X = B in place in x (n × k, row-major) for M the n × n row-major a or, transposed,...
matrix< T > parallel_copy_of(const matrix< T > &a)
A copy of a whose pages the threads fault in (parallel_copy), a decomposition's own storage: the matr...
constexpr std::size_t qr_block
The reflectors of one panel of the blocked QR.
constexpr std::size_t lu_block
The columns of one panel of the blocked LU: the trailing update, a product of this depth,...
double dot_product(const double *x, const double *y, std::size_t count)
Σ x[i] · y[i] for double, as subtract_product chooses its units (in another order of summation there,...
Definition gemm.hpp:715
void apply_transposed_reflector_block(const reflector_block< T > &block, buffer< T > &at, std::size_t cols, std::size_t from, std::size_t to, bool adjoint=false)
apply_reflector_block to A held transposed: at is Aᵀ (row-major, cols columns), and A ← (I − V T Vᴴ) ...
T phase_of(const T &x)
The unit-modulus factor of x: x / |x| (its sign, for a real scalar), and 1 at 0. A subnormal x is sca...
void divide_and_conquer(double *d, const double *e, std::size_t n, buffer< double > &q)
The eigenvalues, ascending, of the symmetric tridiagonal with diagonal d[0…n-1] and subdiagonal e[0…n...
constexpr std::size_t lowner_block
The poles a task of merge_rank_one's Löwner products takes, and the side of its tiles of u.
constexpr std::size_t secular_iterations
The iterations one root of the secular equation may take: it converges in a few; the bound turns a no...
constexpr std::size_t corrected_product_least
The rows from which corrected_hermitian_product shares its coefficients and bands with the threads: o...
void solve_triangular_in_place(const T *a, std::size_t n, bool lower, triangular_op op, bool unit_diagonal, T *x, std::size_t k)
numerical::solve_triangular on the n × n row-major a, in place in x (n × k, row-major): the triangle ...
void join_divide_node(const divide_node &node, merge_workspace &work)
Joins the two solved halves of node by merge_rank_one, in work.
T root(const T &x)
The square root of a scalar, as std::sqrt gives it.
bool lower_triangle_product(std::size_t, std::size_t, const T *, const T *, T *, std::size_t)
C += A·B on the lower triangle of the m × m c (row stride ldc), a m × depth and b depth × m row-major...
std::vector< reflector_block< T > > reflect_in_rows(buffer< T > &a, std::size_t rows, std::size_t cols, buffer< T > *r=nullptr, bool reflectors=true)
The blocks of reflectors of a Householder QR of the row-major a (rows × cols), factorised in place as...
constexpr std::size_t product_rows
The rows of y subtract_rows takes at once.
Definition simd.hpp:731
constexpr std::size_t svd_band
The rows of an SVD's input one thread scales and copies at a time.
T over_real_diagonal(const T &x, const T &diagonal)
x over a Cholesky factor's diagonal entry diagonal, real and positive whatever the scalar.
constexpr std::size_t inverse_band
The rows an LU inverse writes its identity into, and permutes the columns of, on one thread at a time...
void plan_divide(std::vector< divide_node > &nodes, double *d, const double *e, std::size_t n, double *q, std::size_t ldq, std::size_t depth)
Appends the block (d, e, n) and, before it is solved, everything below it to nodes,...
constexpr std::size_t gemm_min_depth
Below this depth the plain loop runs too: each panel reads and writes its tiles of C once,...
Definition gemm.hpp:915
constexpr std::size_t pencil_reduction_least
The order from which a pencil's standard form is reduced by blocks (reduce_pencil): half the operatio...
matrix< T > adjoint(const matrix< T > &a)
The conjugate transpose of a, as numerical::transpose copies (by blocks, on the threads).
Definition svd.hpp:860
carried_t< T > norm_at_scale(const T *x, std::size_t count)
The Euclidean norm of the count elements at x, summed at the scale of their largest magnitude: a plai...
void unscale_divide_node(const divide_node &node)
The eigenvalues of a solved node, scaled back, where its parent reads them.
double adjoint_dot(const double *x, const double *y, std::size_t count)
Σ x[i] y[i], vectorised (dot_product): xᴴ y for real numbers.
carried_t< T > scaled_norm(const T *x, std::size_t count)
The Euclidean norm of the count elements at x, by norm_at_scale for this type.
constexpr double plain_squares_high
The largest plain sum of squares plain_or_at_scale keeps: no term of it overflowed.
double lu_reciprocal_condition(const T *lu, std::size_t n, double one_norm)
reciprocal_condition of A = P⁻¹ L U from the n × n row-major lu (L below the diagonal,...
double power_of_two_scale(double largest) noexcept
The power of two a decomposition divides a matrix by whose largest magnitude is largest: its leading ...
secular_solution solve_secular(const buffer< double > &z, double rho, std::size_t i, std::size_t max_iterations, double *delta, const Offset &offset)
Root i of 1/ρ + Σⱼ zⱼ²/(Pⱼ − x) = 0, poles P strictly ascending, no zⱼ zero, ρ > 0,...
double one_norm_estimate(std::size_t n, const std::function< void(T *, bool)> &apply)
An estimate of ‖B‖₁ for the n × n operator apply gives, as LAPACK's dlacn2 and zlacn2 (Hager's method...
void rotate(T *x, T *y, std::size_t count, std::size_t stride, double cs, double sn)
(x, y) ← (cs x + sn y, cs y − sn x) over count pairs stride apart: a plane rotation.
Definition eigen.hpp:791
merge_workspace & thread_workspace()
The calling thread's merge_workspace, kept from one decomposition to the next: mapping its pages anew...
void jacobi_rotate(buffer< double > &a, buffer< double > &v, std::size_t n, std::size_t p, std::size_t q)
One Jacobi rotation of the n × n a zeroing a[p][q], accumulated into v.
buffer< T > householder_r(buffer< T > a, std::size_t rows, std::size_t cols)
householder_qr's R alone, k × cols, k = min(rows, cols): the same reflections and the same rotation o...
std::conditional_t< std::is_same_v< magnitude_t< T >, long double >, long double, double > carried_t
The real type a T's norms, reflector lengths and pivot magnitudes are carried in: long double for a l...
T from_carried(carried_t< T > x)
x, a magnitude carried in carried_t, as the scalar T: converted to T's real type first,...
buffer< T > tridiagonal_divide_and_conquer(buffer< double > &d, buffer< double > &e, const tridiagonal_reduction< T > &reduction, std::size_t n)
The eigenvalues of the tridiagonal (d, e) of reduction into d, and the matrix's eigenvectors as the r...
void solve_finite_triangular_in_place(const T *a, std::size_t n, bool lower, triangular_op op, bool unit_diagonal, T *x, std::size_t k)
solve_triangular_in_place of a triangle and a right-hand side known finite (require_finite_triangle,...
std::vector< reflector_block< T > > reflect_in_columns(buffer< T > &columns, std::size_t rows, std::size_t cols, bool reflectors=true)
The blocks of reflectors of a Householder QR of the column-major columns (rows per column,...
thin_qr< T > householder_qr(buffer< T > a, std::size_t rows, std::size_t cols, bool complete=false)
Factorises the rows × cols row-major a by Householder reflections, A = Q R with k = min(rows,...
void multiply_real_left(const buffer< double > &left, buffer< T > &zt, std::size_t n)
zt ← left · zt, n × n, row-major, for any other element type: the real factor converted.
T pivot_divisor(const T &pivot)
The divisor over_pivot takes for the pivot pivot. A complex pivot's reciprocal, once,...
constexpr std::size_t zeroed_rows
The rows of a Cholesky factor whose upper part one thread zeroes at a time.
constexpr std::size_t triangular_block
The rows triangular_halves solves one by one; past them, by halves of a multiple of them.
constexpr std::size_t cholesky_rows
The rows of one chunk of a Cholesky panel's rows below it, solved together along each column.
double residual_ratio(const matrix< double > &x, const matrix< double > &r)
The largest of ‖r‖∞ / ‖x‖∞ over the columns of the residual r and the solution x (LAPACK's dsgesv com...
void to_matrix_basis(const tridiagonal_reduction< T > &reduction, buffer< T > &at, std::size_t n, std::size_t rows)
The rows rows of at (n columns, every row by default), vectors in the tridiagonal's basis,...
double(*)(const double *, const double *, double, double *, std::size_t) symmetric_row_kernel
A kernel of symmetric_row's form: symmetric_row itself, or an x86-64 one.
constexpr std::size_t cholesky_diagonal_rows
The rows of a Cholesky diagonal block factorised together, one chain of sums each (one x86-64 AVX2 co...
void gemm_rows(std::size_t m, std::size_t n, std::size_t k, const T *a, std::size_t lda, const T *b, std::size_t ldb, T *c, std::size_t ldc, c_start start=c_start::kept)
C += A·B by the plain i-k-j loop (gemm_rows_rounded), each element rounded as the tile of the path in...
Definition gemm.hpp:2336
T conjugate_dot(const T *x, const T *y, std::size_t count)
Σ conj(x[i]) · y[i] for a real T, where the conjugate is the value: dot_product.
constexpr std::size_t memory_element_weight
The weight of an element a pass bound by the memory writes or moves, against a multiply-add,...
Definition parallel.hpp:760
void corrected_hermitian_product(const T *block, std::size_t n, std::size_t m, const T *v, T *w, buffer< T > &scratch, const T *corrections, std::size_t ldc, std::size_t terms, T *factors, T *next=nullptr, const T *next_factors=nullptr)
hermitian_product corrected by a panel's earlier steps: w = B v − Σₜ fₜ cₜ, the coefficients f[2p] = ...
constexpr std::size_t triangular_slice
The columns of the right-hand sides a thread takes in triangular_halves' rows one by one.
std::size_t symmetric_row_speedup()
The weight of a symmetric row's multiply-adds in hermitian_product: on the x86-64 units,...
std::size_t largest_magnitude_vectors(const T *, std::size_t, Magnitude &, std::size_t &)
Only x[0], the one best and at come in with, for a type no native vector holds.
Definition simd.hpp:1000
void cyclic_jacobi(buffer< double > &a, buffer< double > &v, std::size_t n, std::size_t max_sweeps)
Cyclic Jacobi on the full symmetric a, the rotations accumulated into v.
The NumericAL public API.
Definition amx.hpp:44
std::vector< T, detail::default_init_allocator< T > > buffer
The storage of vector, matrix and tensor, namely a std::vector whose buffer<T>(n) and resize(n) leave...
Definition buffer.hpp:443
std::size_t max_threads()
The threads a kernel may use: the cap if one is set, else the hardware threads (starting none).
Definition parallel.hpp:995
cholesky_decomposition< T > cholesky(matrix< T > a)
Cholesky-factorises a Hermitian positive-definite matrix.
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
vector< T > solve(matrix< T > a, const vector< T > &b)
Solves the linear system A x = b for a square A.
T determinant(matrix< T > a)
The determinant of a square matrix.
rectangular_lu< T > lu_factors(const matrix< T > &a)
The pivoted LU factorisation A = P L U of an m × n matrix, as scipy.linalg.lu: L m × k lower trapezoi...
matrix< T > transpose(const matrix< T > &m)
The transpose, by blocks of transpose_tile (walked by detail::transpose_piece's pieces),...
Definition matrix.hpp:251
gemm_path
The kernels a double matrix product can run on (a float one too on arm64), from the most portable....
Definition gemm.hpp:83
@ portable
The compiler's vectors (16 bytes on x86-64 wheels), as the compiler rounds c + a·b (see gemm).
@ amx
The Apple AMX coprocessor, fused multiply-adds; the products it does not suit on the NEON path's tile...
qr_mode
The factors a numerical::qr_decomposition keeps, as numpy.linalg.qr's modes name them.
@ reduced
Q is m × k, R k × n, k = min(m, n): NumPy's default.
@ complete
Q is m × m, its columns past k a basis of the complement of A's column space; R is m × n.
constexpr fft_norm swapped(fft_norm norm) noexcept
norm with its forward and backward exchanged, as NumPy's hfft and ihfft take it.
Definition fft.hpp:3175
triangular_op
How solve_triangular reads its matrix: as it lies, transposed, or conjugate-transposed (SciPy's trans...
matrix< T > solve_triangular(const matrix< T > &a, matrix< T > b, bool lower, triangular_op op=triangular_op::none, bool unit_diagonal=false)
Solves A X = B, Aᵀ X = B or Aᴴ X = B (op) for a triangular A, as scipy.linalg.solve_triangular: its l...
eigen_method
How numerical::symmetric_eigen diagonalises.
@ jacobi
Cyclic Jacobi rotations: several times slower, kept as an independent second method.
tensor< T > diagonal(const tensor< T > &t, std::ptrdiff_t offset)
The offset diagonal of each matrix of t (its last two axes), as numpy.linalg.diagonal: above the main...
qr_decomposition< T > qr(const matrix< T > &a)
QR-factorises an m × n matrix.
T min(const tensor< T > &t)
The smallest element (a NaN, if there is one), of a transpose's elements as they lie (in any order th...
matrix< T > inverse(matrix< T > a)
The inverse of a square matrix.
matrix< T > qr_r(matrix< T > a)
The factor R of the QR decomposition of an m × n matrix, k × n, k = min(m, n): qr_decomposition's R t...
A minimal std::thread parallel-for for the hot kernels.
The native vector of float and double the kernels hold their lanes in (GNU vector extensions,...
A complex pivot as over_pivot takes it: its reciprocal, or, where that would overflow (a subnormal pi...
std::complex< R > reciprocal
1 / pivot.
std::complex< R > pivot
The pivot.
bool divides
Whether over_pivot divides by the pivot: its parts under the smallest normal.
The pencil A − λ B of a Hermitian A and a Hermitian positive-definite B (each read in its lower trian...
matrix< T > vectors_of(matrix< T > &&v) const
The pencil's eigenvectors from v, columns of C's (all or some): L⁻ᴴ V.
matrix< T > reduced
C = L⁻¹ A L⁻ᴴ, whole.
matrix< T > factor
L, lower-triangular, B = L Lᴴ.
static matrix< T > standard_form(const matrix< T > &a, const matrix< T > &l)
definite_pencil(const matrix< T > &a, const matrix< T > &b)
static matrix< T > checked_factor(const matrix< T > &a, const matrix< T > &b)
One block of divide_and_conquer's tree: its rows [0, n) of the diagonal d its parent reads,...
double * d
The diagonal its parent reads, the eigenvalues once solved.
const double * work_e
The subdiagonal it is solved on.
std::size_t ldq
The distance between the rows of q.
double * q
Its diagonal block of the eigenvectors.
double * work_d
The diagonal it is solved on.
std::size_t n1
The order of its first half; 0 for a leaf.
int exponent
The power of two it is scaled down by.
double beta
The coupling of its halves.
The buffers of merge_rank_one, kept from one join to the next: a fresh buffer per join is mapped anew...
workspace_buffer updated
The kept columns of q times u.
workspace_buffer factor
The rows of u for those columns.
workspace_buffer gaps
poles[j] − roots[i], row by root, then the eigenvectors in its place.
workspace_buffer columns
One half's rows of the kept columns that reach it.
A left operand packed once for several products by it (an LU's −L₂₁ against blocks of its U₁₂): the t...
Definition gemm.hpp:4832
A plane rotation [c s; −s c] and what it leaves of its first argument, as LAPACK's dlartg makes it.
The reflections of a Householder QR, Q left as its blocks of reflectors.
buffer< T > r
R, k × cols upper trapezoidal, its diagonal as it comes.
std::vector< reflector_block< T > > blocks
Q = H₀ H₁ …, block by block of qr_block reflectors.
The reflectors of one panel of the blocked QR, I − V T Vᴴ as LAPACK's dlarft forms it.
std::size_t height
The rows it acts on, first…first+height-1.
buffer< T > v
V, height × width row-major, reflector j from row j (zeros above).
buffer< T > t
T, width × width upper triangular.
std::size_t width
Its reflector count.
std::size_t first
The first row the block acts on.
A run of recorded rotations in which each shares a row with the next, as a sweep of the QL or QR iter...
std::size_t first
Its first operation.
bool down
Whether each next rotation's second row is this one's first.
std::size_t count
Its operations (one for an operation outside any run).
One operation of tridiagonal_ql or bidiagonal_qr on the rows of a vector matrix, recorded to be appli...
std::size_t j
The second row (unused by negate).
double c
The cosine of a rotation.
double s
The sine of a rotation.
A root of a secular equation: the pole it was sought from, and its offset from that pole.
double offset
The root minus the pole.
The eigenvalues and the rotation of a symmetric 2 × 2 [a b; b c], as LAPACK's dlaev2.
double c
[c s; −s c] takes the matrix to diag(larger, smaller).
double larger
The eigenvalue of the larger magnitude.
A thin QR factorisation, row-major: q is rows × cols with orthonormal columns, r is cols × cols upper...
buffer< T > q
The orthonormal factor.
buffer< T > r
The triangular factor.
The unitary M = Q Dᴴ of a reduction to real tridiagonal form, A = M T Mᴴ, kept as its factors.
std::vector< buffer< T > > reflectors
Q = H₁ H₂ …: reflectors[k + 1] zeroes column k, from row k + 1.
buffer< T > delta
D = diag(δ), the unit phases that make T real.
One buffer of merge_workspace: its storage and how many elements it holds.
std::unique_ptr< double[]> data
The elements.
The pivoted LU factorisation of an m × n matrix (lu_factors): A = P L U.
matrix< T > l
m × k (k = min(m, n)), lower trapezoidal with a unit diagonal.
matrix< T > u
k × n, upper trapezoidal.
std::vector< std::size_t > rows
Row i of P⁻¹ A = L U is row rows[i] of A.
A dense, value-semantic vector over a scalar type.