NumericAL
A header-only C++20 lexer built on REAL
Loading...
Searching...
No Matches
tensor_ops.hpp
Go to the documentation of this file.
1
11#ifndef NUMERICAL_TENSOR_OPS_HPP
12#define NUMERICAL_TENSOR_OPS_HPP
13
14#include <algorithm>
15#include <array>
16#include <bit>
17#include <cmath>
18#include <complex>
19#include <concepts>
20#include <cstddef>
21#include <cstdint>
22#include <cstring>
23#include <functional>
24#include <limits>
25#include <numeric>
26#include <stdexcept>
27#include <string>
28#include <type_traits>
29#include <utility>
30#include <vector>
31
32#include "complex_math.hpp"
33#include "elementary.hpp"
34#include "simd.hpp"
35#include "tensor.hpp"
36
37namespace numerical {
38
39 namespace detail {
40
41
44 struct axis_walk
45 {
46 std::size_t outer;
47 std::size_t extent;
48 std::size_t inner;
49 };
50
53 [[nodiscard]] inline axis_walk walk_along(const std::vector<std::size_t>& shape,
54 std::size_t axis)
55 {
56 if (axis >= shape.size()) {
57 throw dimension_error("axis " + std::to_string(axis) + " is out of bounds for a tensor of rank "
58 + std::to_string(shape.size()));
59 }
60 const auto before {shape.begin() + static_cast<std::ptrdiff_t>(axis)};
61 axis_walk walk {.outer = 1, .extent = shape[axis], .inner = 1};
62 // The result's elements, the axis taken out: none (and no line walked) where a kept extent is 0, whatever
63 // the others; else their product, checked, which the outer and inner parts then divide.
64 if (std::find(shape.begin(), before, std::size_t {0}) != before || std::find(before + 1, shape.end(), std::size_t {0}) != shape.end()) {
65 walk.outer = 0;
66 walk.inner = 0;
67 return walk;
68 }
69 walk.outer = detail::element_count(shape.begin(), before);
70 walk.inner = detail::element_count(before + 1, shape.end());
71 static_cast<void>(detail::checked_mul(walk.outer, walk.inner));
72 return walk;
73 }
74
77 template <std::floating_point R>
79 R y,
80 R z,
81 bool fused)
82 {
83 if (fused) {
84 return std::fma(x, y, z);
85 }
86 const R product {x * y};
87 return product + z;
88 }
89
97 template <split_complex C>
98 [[nodiscard]] C numpy_multiply(const C& a,
99 const C& b,
100 bool fused)
101 {
102 using R = typename C::value_type;
103 const R ii {a.imag() * b.imag()};
104 const R ir {a.imag() * b.real()};
105 return C {numpy_muladd(a.real(), b.real(), -ii, fused), numpy_muladd(a.real(), b.imag(), ir, fused)};
106 }
107
111 template <split_complex C>
113 const C& b)
114 {
115 using R = typename C::value_type;
116 const R ii {a.imag() * b.imag()};
117 const R ri {a.real() * b.imag()};
118 return C {std::fma(a.real(), b.real(), -ii), std::fma(a.imag(), b.real(), ri)};
119 }
120
123 template <std::floating_point R>
124 [[nodiscard]] R divided_by_zero(R x) noexcept
125 {
126 if (x == R {0} || std::isnan(x)) {
127 return std::numeric_limits<R>::quiet_NaN();
128 }
129 return std::copysign(std::numeric_limits<R>::infinity(), x);
130 }
131
137 template <split_complex C>
138 [[nodiscard]] C numpy_divide(const C& a,
139 const C& b,
140 bool fused)
141 {
142 using R = typename C::value_type;
143 const R real_part {std::abs(b.real())};
144 const R imaginary_part {std::abs(b.imag())};
145 if (real_part >= imaginary_part) {
146 if (real_part == R {0}) { // the imaginary part's magnitude no larger: a zero too
147 return C {divided_by_zero(a.real()), divided_by_zero(a.imag())};
148 }
149 const R ratio {b.imag() / b.real()};
150 const R scale {R {1} / numpy_muladd(b.imag(), ratio, b.real(), fused)};
151 return C {numpy_muladd(a.imag(), ratio, a.real(), fused) * scale, numpy_muladd(-a.real(), ratio, a.imag(), fused) * scale};
152 }
153 const R ratio {b.real() / b.imag()};
154 const R scale {R {1} / numpy_muladd(b.real(), ratio, b.imag(), fused)};
155 return C {numpy_muladd(a.real(), ratio, a.imag(), fused) * scale, numpy_muladd(a.imag(), ratio, -a.real(), fused) * scale};
156 }
157
163 template <split_complex C>
164 [[nodiscard]] C numpy_reciprocal(const C& x,
165 bool fused)
166 {
167 using R = typename C::value_type;
168 if (std::abs(x.imag()) <= std::abs(x.real())) {
169 const R ratio {x.imag() / x.real()};
170 const R d {numpy_muladd(x.imag(), ratio, x.real(), fused)};
171 return C {R {1} / d, -ratio / d};
172 }
173 const R ratio {x.real() / x.imag()};
174 const R d {numpy_muladd(x.real(), ratio, x.imag(), fused)};
175 return C {ratio / d, R {-1} / d};
176 }
177
178#if defined(__APPLE__)
180 struct c_complex_double {
181 double re;
182 double im;
183 };
184
186 struct c_complex_float {
187 float re;
188 float im;
189 };
190
191 // The C library's `cpow` and `cpowf` under names of their own (C++ has no `_Complex`), their symbols bound by label.
193 c_complex_double exponent) __asm__ ("_cpow");
195 c_complex_float exponent) __asm__ ("_cpowf");
196
199 [[nodiscard]] inline std::complex<double> library_power(const std::complex<double>& a,
200 const std::complex<double>& b) noexcept
201 {
202 const c_complex_double z {c_library_cpow({a.real(), a.imag()}, {b.real(), b.imag()})};
203 return {z.re, z.im};
204 }
205
207 [[nodiscard]] inline std::complex<float> library_power(const std::complex<float>& a,
208 const std::complex<float>& b) noexcept
209 {
210 const c_complex_float z {c_library_cpowf({a.real(), a.imag()}, {b.real(), b.imag()})};
211 return {z.re, z.im};
212 }
213
214#else
217 template <split_complex C>
218 [[nodiscard]] C library_power(const C& a,
219 const C& b)
220 {
221 return std::pow(a, b);
222 }
223
224#endif
225
232 template <split_complex C>
233 [[nodiscard]] C numpy_power(const C& a,
234 const C& b,
235 bool fused)
236 {
237 using R = typename C::value_type;
238 if (b.real() == R {0} && b.imag() == R {0}) {
239 return C {R {1}, R {0}};
240 }
241 if (a.real() == R {0} && a.imag() == R {0}) {
242 if (b.real() > R {0}) {
243 return C {R {0}, R {0}};
244 }
245 return C {std::numeric_limits<R>::quiet_NaN(), std::numeric_limits<R>::quiet_NaN()};
246 }
247 if (b.imag() != R {0} || !(b.real() > R {-100} && b.real() < R {100}) || std::trunc(b.real()) != b.real()) {
248 return library_power(a, b);
249 }
250 const auto n {static_cast<int>(b.real())};
251 if (n == 1) {
252 return a;
253 }
254 if (n == 2) {
255 return numpy_multiply(a, a, fused);
256 }
257 if (n == 3) {
258 return numpy_multiply(a, numpy_multiply(a, a, fused), fused);
259 }
260 const auto magnitude {static_cast<unsigned>(n < 0 ? -n : n)};
261 C product {R {1}, R {0}};
262 C power {a};
263 for (unsigned bit {1U};; bit <<= 1U) { // the binary powers of a, those of the exponent's bits multiplied in
264 if ((magnitude & bit) != 0U) {
266 }
267 if (magnitude < (bit << 1U)) {
268 break;
269 }
271 }
272 if (n < 0) {
273 return numpy_divide(C {R {1}, R {0}}, product, fused);
274 }
275 return product;
276 }
277
279 template <typename T>
280 [[nodiscard]] T root(const T& x)
281 {
282 return static_cast<T>(std::sqrt(x));
283 }
284
286 template <split_complex T>
287 [[nodiscard]] T root(const T& x)
288 {
289 return complex_sqrt(x);
290 }
291
299 template <typename Into>
301 const std::vector<std::size_t>& shape,
302 Into into)
303 {
304 const std::size_t count {element_count(shape.begin(), shape.end())};
305 buffer<float> out(count);
306 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
307 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
308 constexpr std::size_t piece {512};
309 std::array<double, piece> wide {};
310 std::array<double, piece> image {};
311 const std::size_t last {std::min(count, (c + 1) * elementwise_chunk)};
312 for (std::size_t first {c * elementwise_chunk}; first < last; first += piece) {
313 const std::size_t size {std::min(piece, last - first)};
314 std::copy_n(data + first, size, wide.begin());
315 into(wide.data(), image.data(), size);
316 std::transform(image.begin(), image.begin() + static_cast<std::ptrdiff_t>(size), out.data() + first,
317 [](double y) { return static_cast<float>(y); });
318 }
319 });
320 return tensor<float> {shape, std::move(out)};
321 }
322
324 template <typename T>
326 {
327 return static_cast<T>(std::exp(x));
328 }
329
330#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
336 [[nodiscard]] inline bool exp_by_table() noexcept
337 {
338# if defined(_MSC_VER) && !defined(__clang__)
339 return false;
340# else
341 return cpu_features().avx2_fma;
342# endif
343 }
344
345#endif
346
349 [[nodiscard]] inline double elementary_exp(double x)
350 {
351#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
352 if (!exp_by_table()) {
353 return std::exp(x);
354 }
355#endif
356 return x < exp_normal_bound ? std::exp(x) : table_exp(x);
357 }
358
365 inline void exp_into(const double* in,
366 double* out,
367 std::size_t count)
368 {
369#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
370 if (!exp_by_table()) {
371 std::transform(in, in + count, out, [](double x) { return std::exp(x); });
372 return;
373 }
374#endif
375 // std::exp only where a value is below the bound, behind a branch: where it sets no errno (macOS), a compiler
376 // may else evaluate it on every element and keep the few it needs (Apple's Clang 16: 10⁶ values 1.9 → 7.6 ms)
377#ifdef NUMERICAL_EXP_NEON
378 std::size_t i {0};
379 for (; i + 2 <= count; i += 2) {
380 const float64x2_t x {vld1q_f64(in + i)};
381 vst1q_f64(out + i, table_exp(x));
382 if (vminnmvq_f64(x) < exp_normal_bound) { // the lesser number, past a NaN
383 for (std::size_t k {i}; k < i + 2; ++k) {
384 if (in[k] < exp_normal_bound) {
385 out[k] = std::exp(in[k]);
386 }
387 }
388 }
389 }
390 if (i < count) {
391 out[i] = elementary_exp(in[i]);
392 }
393#else
394 run_fused([&](std::size_t) NUMERICAL_INLINE_LAMBDA {
395 for (std::size_t i {0}; i < count; ++i) {
396 out[i] = table_exp(in[i]);
397 }
398 });
399# if defined(__clang__)
400# pragma clang loop vectorize(disable)
401# endif
402 for (std::size_t i {0}; i < count; ++i) {
403 if (in[i] < exp_normal_bound) {
404 out[i] = std::exp(in[i]);
405 }
406 }
407#endif
408 }
409
411 template <typename T>
413 {
414 return static_cast<T>(std::expm1(x));
415 }
416
419 [[nodiscard]] inline double elementary_expm1(double x)
420 {
421#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
422 if (!exp_by_table()) {
423 return std::expm1(x);
424 }
425#endif
426 return table_expm1(x);
427 }
428
432 inline void expm1_into(const double* in,
433 double* out,
434 std::size_t count)
435 {
436#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
437 if (!exp_by_table()) {
438 std::transform(in, in + count, out, [](double x) { return std::expm1(x); });
439 return;
440 }
441#endif
442#ifdef NUMERICAL_EXP_NEON
443 std::size_t i {0};
444 for (; i + 2 <= count; i += 2) {
445 vst1q_f64(out + i, table_expm1(vld1q_f64(in + i)));
446 }
447 if (i < count) {
448 out[i] = table_expm1(in[i]);
449 }
450#else
451 run_fused([&](std::size_t) NUMERICAL_INLINE_LAMBDA {
452 for (std::size_t i {0}; i < count; ++i) {
453 out[i] = table_expm1(in[i]);
454 }
455 });
456#endif
457 }
458
460 template <typename T>
461 [[nodiscard]] tensor<T> expm1_of(const T* data,
462 const std::vector<std::size_t>& shape)
463 {
464 return parallel_map(data, shape, [](const T& x) { return elementary_expm1(x); });
465 }
466
469 [[nodiscard]] inline tensor<double> expm1_of(const double* data,
470 const std::vector<std::size_t>& shape)
471 {
472 const std::size_t count {element_count(shape.begin(), shape.end())};
473 buffer<double> out(count);
474 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
475 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
476 const std::size_t first {c * elementwise_chunk};
477 expm1_into(data + first, out.data() + first, std::min(elementwise_chunk, count - first));
478 });
479 return tensor<double> {shape, std::move(out)};
480 }
481
483 [[nodiscard]] inline float elementary_expm1(float x)
484 {
485 return static_cast<float>(elementary_expm1(static_cast<double>(x)));
486 }
487
490 [[nodiscard]] inline tensor<float> expm1_of(const float* data,
491 const std::vector<std::size_t>& shape)
492 {
493 return through_doubles(data, shape, &expm1_into);
494 }
495
497 template <typename T>
499 {
500 return static_cast<T>(std::atan(x));
501 }
502
505 [[nodiscard]] inline double elementary_atan(double x)
506 {
507#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
508 if (!exp_by_table()) {
509 return std::atan(x);
510 }
511#endif
512 return table_atan(x);
513 }
514
518 inline void atan_into(const double* in,
519 double* out,
520 std::size_t count)
521 {
522#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
523 if (!exp_by_table()) {
524 std::transform(in, in + count, out, [](double x) { return std::atan(x); });
525 return;
526 }
527#endif
528#ifdef NUMERICAL_EXP_NEON
529 std::size_t i {0};
530 for (; i + 2 <= count; i += 2) {
531 vst1q_f64(out + i, table_atan(vld1q_f64(in + i)));
532 }
533 if (i < count) {
534 out[i] = table_atan(in[i]);
535 }
536#else
537 run_fused([&](std::size_t) NUMERICAL_INLINE_LAMBDA {
538 for (std::size_t i {0}; i < count; ++i) {
539 out[i] = table_atan(in[i]);
540 }
541 });
542#endif
543 }
544
546 template <typename T>
547 [[nodiscard]] tensor<T> atan_of(const T* data,
548 const std::vector<std::size_t>& shape)
549 {
550 return parallel_map(data, shape, [](const T& x) { return elementary_atan(x); });
551 }
552
555 [[nodiscard]] inline tensor<double> atan_of(const double* data,
556 const std::vector<std::size_t>& shape)
557 {
558 const std::size_t count {element_count(shape.begin(), shape.end())};
559 buffer<double> out(count);
560 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
561 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
562 const std::size_t first {c * elementwise_chunk};
563 atan_into(data + first, out.data() + first, std::min(elementwise_chunk, count - first));
564 });
565 return tensor<double> {shape, std::move(out)};
566 }
567
569 [[nodiscard]] inline float elementary_atan(float x)
570 {
571 return static_cast<float>(elementary_atan(static_cast<double>(x)));
572 }
573
576 [[nodiscard]] inline tensor<float> atan_of(const float* data,
577 const std::vector<std::size_t>& shape)
578 {
579 return through_doubles(data, shape, &atan_into);
580 }
581
583 template <typename T>
585 {
586 return static_cast<T>(std::sinh(x));
587 }
588
591 [[nodiscard]] inline double elementary_sinh(double x)
592 {
593#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
594 if (!exp_by_table()) {
595 return std::sinh(x);
596 }
597#endif
598 return table_sinh(x);
599 }
600
604 inline void sinh_into(const double* in,
605 double* out,
606 std::size_t count)
607 {
608#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
609 if (!exp_by_table()) {
610 std::transform(in, in + count, out, [](double x) { return std::sinh(x); });
611 return;
612 }
613#endif
614#ifdef NUMERICAL_EXP_NEON
615 std::size_t i {0};
616 for (; i + 2 <= count; i += 2) {
617 vst1q_f64(out + i, table_sinh(vld1q_f64(in + i)));
618 }
619 if (i < count) {
620 out[i] = table_sinh(in[i]);
621 }
622#else
623 run_fused([&](std::size_t) NUMERICAL_INLINE_LAMBDA {
624 for (std::size_t i {0}; i < count; ++i) {
625 out[i] = table_sinh(in[i]);
626 }
627 });
628#endif
629 }
630
632 template <typename T>
633 [[nodiscard]] tensor<T> sinh_of(const T* data,
634 const std::vector<std::size_t>& shape)
635 {
636 return parallel_map(data, shape, [](const T& x) { return elementary_sinh(x); });
637 }
638
641 [[nodiscard]] inline tensor<double> sinh_of(const double* data,
642 const std::vector<std::size_t>& shape)
643 {
644 const std::size_t count {element_count(shape.begin(), shape.end())};
645 buffer<double> out(count);
646 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
647 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
648 const std::size_t first {c * elementwise_chunk};
649 sinh_into(data + first, out.data() + first, std::min(elementwise_chunk, count - first));
650 });
651 return tensor<double> {shape, std::move(out)};
652 }
653
655 [[nodiscard]] inline float elementary_sinh(float x)
656 {
657 return static_cast<float>(elementary_sinh(static_cast<double>(x)));
658 }
659
662 [[nodiscard]] inline tensor<float> sinh_of(const float* data,
663 const std::vector<std::size_t>& shape)
664 {
665 return through_doubles(data, shape, &sinh_into);
666 }
667
669 template <typename T>
671 {
672 return static_cast<T>(std::cosh(x));
673 }
674
677 [[nodiscard]] inline double elementary_cosh(double x)
678 {
679#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
680 if (!exp_by_table()) {
681 return std::cosh(x);
682 }
683#endif
684 return table_cosh(x);
685 }
686
690 inline void cosh_into(const double* in,
691 double* out,
692 std::size_t count)
693 {
694#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
695 if (!exp_by_table()) {
696 std::transform(in, in + count, out, [](double x) { return std::cosh(x); });
697 return;
698 }
699#endif
700#ifdef NUMERICAL_EXP_NEON
701 std::size_t i {0};
702 for (; i + 2 <= count; i += 2) {
703 vst1q_f64(out + i, table_cosh(vld1q_f64(in + i)));
704 }
705 if (i < count) {
706 out[i] = table_cosh(in[i]);
707 }
708#else
709 run_fused([&](std::size_t) NUMERICAL_INLINE_LAMBDA {
710 for (std::size_t i {0}; i < count; ++i) {
711 out[i] = table_cosh(in[i]);
712 }
713 });
714#endif
715 }
716
718 template <typename T>
719 [[nodiscard]] tensor<T> cosh_of(const T* data,
720 const std::vector<std::size_t>& shape)
721 {
722 return parallel_map(data, shape, [](const T& x) { return elementary_cosh(x); });
723 }
724
727 [[nodiscard]] inline tensor<double> cosh_of(const double* data,
728 const std::vector<std::size_t>& shape)
729 {
730 const std::size_t count {element_count(shape.begin(), shape.end())};
731 buffer<double> out(count);
732 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
733 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
734 const std::size_t first {c * elementwise_chunk};
735 cosh_into(data + first, out.data() + first, std::min(elementwise_chunk, count - first));
736 });
737 return tensor<double> {shape, std::move(out)};
738 }
739
741 [[nodiscard]] inline float elementary_cosh(float x)
742 {
743 return static_cast<float>(elementary_cosh(static_cast<double>(x)));
744 }
745
748 [[nodiscard]] inline tensor<float> cosh_of(const float* data,
749 const std::vector<std::size_t>& shape)
750 {
751 return through_doubles(data, shape, &cosh_into);
752 }
753
755 inline constexpr double exp2_normal_bound {-1022.0};
756
758 template <typename T>
760 {
761 return static_cast<T>(std::exp2(x));
762 }
763
766 [[nodiscard]] inline double elementary_exp2(double x)
767 {
768#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
769 if (!exp_by_table()) {
770 return std::exp2(x);
771 }
772#endif
773 return x < exp2_normal_bound ? std::exp2(x) : table_exp2(x);
774 }
775
781 inline void exp2_into(const double* in,
782 double* out,
783 std::size_t count)
784 {
785#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
786 if (!exp_by_table()) {
787 std::transform(in, in + count, out, [](double x) { return std::exp2(x); });
788 return;
789 }
790#endif
791#ifdef NUMERICAL_EXP_NEON
792 std::size_t i {0};
793 for (; i + 2 <= count; i += 2) {
794 const float64x2_t x {vld1q_f64(in + i)};
795 vst1q_f64(out + i, table_exp2(x));
796 if (vminnmvq_f64(x) < exp2_normal_bound) { // the lesser number, past a NaN
797 for (std::size_t k {i}; k < i + 2; ++k) {
798 if (in[k] < exp2_normal_bound) {
799 out[k] = std::exp2(in[k]);
800 }
801 }
802 }
803 }
804 if (i < count) {
805 out[i] = elementary_exp2(in[i]);
806 }
807#else
808 run_fused([&](std::size_t) NUMERICAL_INLINE_LAMBDA {
809 for (std::size_t i {0}; i < count; ++i) {
810 out[i] = table_exp2(in[i]);
811 }
812 });
813# if defined(__clang__)
814# pragma clang loop vectorize(disable)
815# endif
816 for (std::size_t i {0}; i < count; ++i) {
817 if (in[i] < exp2_normal_bound) {
818 out[i] = std::exp2(in[i]);
819 }
820 }
821#endif
822 }
823
825 template <typename T>
826 [[nodiscard]] tensor<T> exp2_of(const T* data,
827 const std::vector<std::size_t>& shape)
828 {
829 return parallel_map(data, shape, [](const T& x) { return elementary_exp2(x); });
830 }
831
834 [[nodiscard]] inline tensor<double> exp2_of(const double* data,
835 const std::vector<std::size_t>& shape)
836 {
837 const std::size_t count {element_count(shape.begin(), shape.end())};
838 buffer<double> out(count);
839 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
840 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
841 const std::size_t first {c * elementwise_chunk};
842 exp2_into(data + first, out.data() + first, std::min(elementwise_chunk, count - first));
843 });
844 return tensor<double> {shape, std::move(out)};
845 }
846
848 template <split_complex T>
849 [[nodiscard]] T elementary_exp(const T& x)
850 {
851 return complex_exp(x);
852 }
853
855 template <typename T>
857 {
858 return static_cast<T>(std::log(x));
859 }
860
862 template <split_complex T>
863 [[nodiscard]] T elementary_log(const T& x)
864 {
865 return complex_log(x);
866 }
867
869 template <typename T>
871 {
872 return static_cast<T>(std::log1p(x));
873 }
874
877 [[nodiscard]] inline double elementary_log1p(double x)
878 {
879#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
880 if (!exp_by_table()) {
881 return std::log1p(x);
882 }
883#endif
884 return table_log1p(x);
885 }
886
890 inline void log1p_into(const double* in,
891 double* out,
892 std::size_t count)
893 {
894#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
895 if (!exp_by_table()) {
896 std::transform(in, in + count, out, [](double x) { return std::log1p(x); });
897 return;
898 }
899#endif
900#ifdef NUMERICAL_EXP_NEON
901 std::size_t i {0};
902 for (; i + 2 <= count; i += 2) {
903 vst1q_f64(out + i, table_log1p(vld1q_f64(in + i)));
904 }
905 if (i < count) {
906 out[i] = table_log1p(in[i]);
907 }
908#else
909 run_fused([&](std::size_t) NUMERICAL_INLINE_LAMBDA {
910 for (std::size_t i {0}; i < count; ++i) {
911 out[i] = table_log1p(in[i]);
912 }
913 });
914#endif
915 }
916
918 template <typename T>
919 [[nodiscard]] tensor<T> log1p_of(const T* data,
920 const std::vector<std::size_t>& shape)
921 {
922 return parallel_map(data, shape, [](const T& x) { return elementary_log1p(x); });
923 }
924
927 [[nodiscard]] inline tensor<double> log1p_of(const double* data,
928 const std::vector<std::size_t>& shape)
929 {
930 const std::size_t count {element_count(shape.begin(), shape.end())};
931 buffer<double> out(count);
932 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
933 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
934 const std::size_t first {c * elementwise_chunk};
935 log1p_into(data + first, out.data() + first, std::min(elementwise_chunk, count - first));
936 });
937 return tensor<double> {shape, std::move(out)};
938 }
939
941 [[nodiscard]] inline float elementary_log1p(float x)
942 {
943 return static_cast<float>(elementary_log1p(static_cast<double>(x)));
944 }
945
948 [[nodiscard]] inline tensor<float> log1p_of(const float* data,
949 const std::vector<std::size_t>& shape)
950 {
951 return through_doubles(data, shape, &log1p_into);
952 }
953
955 template <typename T>
957 {
958 return static_cast<T>(std::sin(x));
959 }
960
962 template <split_complex T>
963 [[nodiscard]] T elementary_sin(const T& x)
964 {
965 return complex_sin(x);
966 }
967
969 template <typename T>
971 {
972 return static_cast<T>(std::cos(x));
973 }
974
976 template <split_complex T>
977 [[nodiscard]] T elementary_cos(const T& x)
978 {
979 return complex_cos(x);
980 }
981
983 template <bool Cosine>
985
987 template <>
989 {
991 [[nodiscard]] static double call(double x)
992 {
993 return std::sin(x);
994 }
995 };
996
998 template <>
1000 {
1002 [[nodiscard]] static double call(double x)
1003 {
1004 return std::cos(x);
1005 }
1006 };
1007
1010 template <bool Cosine>
1011 [[nodiscard]] double float_sine(double x)
1012 {
1013 if (std::fabs(x) <= sine_reduction_bound) {
1014 return reduced_sine<Cosine>(x);
1015 }
1017 }
1018
1025 template <bool Cosine>
1026 void float_sines_into(const double* in,
1027 double* out,
1028 std::size_t count)
1029 {
1030#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
1031 if (!exp_by_table()) {
1032 std::transform(in, in + count, out, [](double x) { return library_sine<Cosine>::call(x); });
1033 return;
1034 }
1035#endif
1036 unsigned far {0};
1037 run_fused([&](std::size_t) NUMERICAL_INLINE_LAMBDA {
1038 for (std::size_t i {0}; i < count; ++i) {
1039 out[i] = reduced_sine<Cosine>(in[i]);
1040 far |= static_cast<unsigned>(!(std::fabs(in[i]) <= sine_reduction_bound));
1041 }
1042 });
1043 if (far != 0U) {
1044 for (std::size_t i {0}; i < count; ++i) {
1045 if (!(std::fabs(in[i]) <= sine_reduction_bound)) {
1046 out[i] = library_sine<Cosine>::call(in[i]);
1047 }
1048 }
1049 }
1050 }
1051
1055 template <bool Cosine>
1056 [[nodiscard]] double double_sine(double x)
1057 {
1058#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
1059 if (!exp_by_table()) {
1061 }
1062#endif
1063 if (std::fabs(x) <= sine_reduction_bound) {
1064 return table_sine<Cosine>(x);
1065 }
1067 }
1068
1076 template <bool Cosine>
1077 void double_sines_into(const double* in,
1078 double* out,
1079 std::size_t count)
1080 {
1081#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
1082 if (!exp_by_table()) {
1083 std::transform(in, in + count, out, [](double x) { return library_sine<Cosine>::call(x); });
1084 return;
1085 }
1086#endif
1087 run_fused([&](std::size_t) NUMERICAL_INLINE_LAMBDA {
1088 for (std::size_t i {0}; i < count; ++i) {
1089 out[i] = std::fabs(in[i]) <= sine_reduction_bound ? table_sine<Cosine>(in[i]) : library_sine<Cosine>::call(in[i]);
1090 }
1091 });
1092 }
1093
1095 [[nodiscard]] inline double elementary_sin(double x)
1096 {
1097 return double_sine<false>(x);
1098 }
1099
1101 [[nodiscard]] inline double elementary_cos(double x)
1102 {
1103 return double_sine<true>(x);
1104 }
1105
1107 [[nodiscard]] inline float elementary_sin(float x)
1108 {
1109 return static_cast<float>(float_sine<false>(static_cast<double>(x)));
1110 }
1111
1113 [[nodiscard]] inline float elementary_cos(float x)
1114 {
1115 return static_cast<float>(float_sine<true>(static_cast<double>(x)));
1116 }
1117
1119 template <typename T>
1120 [[nodiscard]] tensor<T> sin_of(const T* data,
1121 const std::vector<std::size_t>& shape)
1122 {
1123 return parallel_map(data, shape, [](const T& x) { return elementary_sin(x); });
1124 }
1125
1128 [[nodiscard]] inline tensor<float> sin_of(const float* data,
1129 const std::vector<std::size_t>& shape)
1130 {
1131 return through_doubles(data, shape, &float_sines_into<false>);
1132 }
1133
1136 [[nodiscard]] inline tensor<double> sin_of(const double* data,
1137 const std::vector<std::size_t>& shape)
1138 {
1139 const std::size_t count {element_count(shape.begin(), shape.end())};
1140 buffer<double> out(count);
1141 parallel_for((count + elementwise_chunk - 1) / elementwise_chunk, elementwise_chunk, [&](std::size_t c) {
1142 const std::size_t first {c * elementwise_chunk};
1143 double_sines_into<false>(data + first, out.data() + first, std::min(elementwise_chunk, count - first));
1144 });
1145 return tensor<double> {shape, std::move(out)};
1146 }
1147
1149 template <typename T>
1150 [[nodiscard]] tensor<T> cos_of(const T* data,
1151 const std::vector<std::size_t>& shape)
1152 {
1153 return parallel_map(data, shape, [](const T& x) { return elementary_cos(x); });
1154 }
1155
1158 [[nodiscard]] inline tensor<double> cos_of(const double* data,
1159 const std::vector<std::size_t>& shape)
1160 {
1161 const std::size_t count {element_count(shape.begin(), shape.end())};
1162 buffer<double> out(count);
1163 parallel_for((count + elementwise_chunk - 1) / elementwise_chunk, elementwise_chunk, [&](std::size_t c) {
1164 const std::size_t first {c * elementwise_chunk};
1165 double_sines_into<true>(data + first, out.data() + first, std::min(elementwise_chunk, count - first));
1166 });
1167 return tensor<double> {shape, std::move(out)};
1168 }
1169
1172 [[nodiscard]] inline tensor<float> cos_of(const float* data,
1173 const std::vector<std::size_t>& shape)
1174 {
1175 return through_doubles(data, shape, &float_sines_into<true>);
1176 }
1177
1179 template <typename T>
1181 {
1182 return static_cast<T>(std::tan(x));
1183 }
1184
1186 template <split_complex T>
1187 [[nodiscard]] T elementary_tan(const T& x)
1188 {
1189 return complex_tan(x);
1190 }
1191
1193 template <typename T>
1195 {
1196 return static_cast<T>(std::tanh(x));
1197 }
1198
1200 template <split_complex T>
1201 [[nodiscard]] T elementary_tanh(const T& x)
1202 {
1203 return complex_tanh(x);
1204 }
1205
1208 [[nodiscard]] inline double elementary_tanh(double x)
1209 {
1210#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
1211 if (!exp_by_table()) {
1212 return std::tanh(x);
1213 }
1214#endif
1215 return table_tanh(x);
1216 }
1217
1221 inline void tanh_into(const double* in,
1222 double* out,
1223 std::size_t count)
1224 {
1225#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
1226 if (!exp_by_table()) {
1227 std::transform(in, in + count, out, [](double x) { return std::tanh(x); });
1228 return;
1229 }
1230#endif
1231#ifdef NUMERICAL_EXP_NEON
1232 std::size_t i {0};
1233 for (; i + 2 <= count; i += 2) {
1234 vst1q_f64(out + i, table_tanh(vld1q_f64(in + i)));
1235 }
1236 if (i < count) {
1237 out[i] = table_tanh(in[i]);
1238 }
1239#else
1240 run_fused([&](std::size_t) NUMERICAL_INLINE_LAMBDA {
1241 for (std::size_t i {0}; i < count; ++i) {
1242 out[i] = table_tanh(in[i]);
1243 }
1244 });
1245#endif
1246 }
1247
1249 template <typename T>
1250 [[nodiscard]] tensor<T> tanh_of(const T* data,
1251 const std::vector<std::size_t>& shape)
1252 {
1253 return parallel_map(data, shape, [](const T& x) { return elementary_tanh(x); });
1254 }
1255
1258 [[nodiscard]] inline tensor<double> tanh_of(const double* data,
1259 const std::vector<std::size_t>& shape)
1260 {
1261 const std::size_t count {element_count(shape.begin(), shape.end())};
1262 buffer<double> out(count);
1263 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
1264 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
1265 const std::size_t first {c * elementwise_chunk};
1266 tanh_into(data + first, out.data() + first, std::min(elementwise_chunk, count - first));
1267 });
1268 return tensor<double> {shape, std::move(out)};
1269 }
1270
1272 [[nodiscard]] inline float elementary_tanh(float x)
1273 {
1274 return static_cast<float>(elementary_tanh(static_cast<double>(x)));
1275 }
1276
1279 [[nodiscard]] inline tensor<float> tanh_of(const float* data,
1280 const std::vector<std::size_t>& shape)
1281 {
1282 return through_doubles(data, shape, &tanh_into);
1283 }
1284
1291 template <split_complex C>
1292 [[nodiscard]] typename C::value_type numpy_absolute(const C& x,
1293 bool fused)
1294 {
1295 using R = typename C::value_type;
1296 constexpr R inf {std::numeric_limits<R>::infinity()};
1297 constexpr R nan {std::numeric_limits<R>::quiet_NaN()};
1298 R re {std::abs(x.real())};
1299 R im {std::abs(x.imag())};
1300 const bool re_infinite {re == inf};
1301 const bool im_infinite {im == inf};
1302 if (re_infinite) {
1303 im = inf;
1304 }
1305 if (im_infinite) {
1306 re = inf;
1307 }
1308 const bool re_nan {std::isnan(re)};
1309 const bool im_nan {std::isnan(im)};
1310 if (re_nan) {
1311 im = nan;
1312 }
1313 if (im_nan) {
1314 re = nan;
1315 }
1316 const R larger {std::max(re, im)};
1317 const R smaller {std::min(im, re)};
1318 R ratio {0};
1319 if (larger != R {0} && smaller != inf) {
1320 ratio = smaller / larger;
1321 }
1322 return std::sqrt(numpy_muladd(ratio, ratio, R {1}, fused)) * larger;
1323 }
1324
1328 {
1329#if defined(__aarch64__) || defined(_M_ARM64)
1330 return true;
1331#elif defined(NUMERICAL_X86_COMPILED)
1332 return cpu_features().avx2_fma;
1333#else
1334 return false;
1335#endif
1336 }
1337
1342 {
1343#if defined(__APPLE__) || defined(_WIN32)
1344 return numpy_fuses_products();
1345#else
1346 return false;
1347#endif
1348 }
1349
1355 {
1357 }
1358
1363 {
1364#if defined(__APPLE__) && defined(__aarch64__)
1365 return true;
1366#else
1367 return false;
1368#endif
1369 }
1370
1377 [[nodiscard]] inline bool numpy_loop_is_scalar(const std::vector<std::size_t>& a_shape,
1378 const std::vector<std::size_t>& b_shape) noexcept
1379 {
1380 const auto ones {[](const std::vector<std::size_t>& shape) { return std::ranges::all_of(shape, [](std::size_t e) { return e == 1; }); }};
1381 const std::size_t rank {std::max(a_shape.size(), b_shape.size())};
1382 const auto broadcast {[rank](const std::vector<std::size_t>& shape) { return !shape.empty() && shape.size() != rank; }};
1383 if (!ones(a_shape) || !ones(b_shape)) {
1384 return false;
1385 }
1386 return broadcast(a_shape) || broadcast(b_shape);
1387 }
1388
1390 template <typename T>
1392 const tensor<T>& b) noexcept
1393 {
1394 return numpy_loop_is_scalar(a.shape(), b.shape());
1395 }
1396
1402 template <std::floating_point T>
1404 std::size_t n)
1405 {
1406 using wide = std::common_type_t<T, double>;
1407 return static_cast<T>(static_cast<wide>(value) / static_cast<wide>(n));
1408 }
1409
1415 template <split_complex T>
1416 [[nodiscard]] T divided_by_count(T value,
1417 std::size_t n)
1418 {
1419 using R = typename T::value_type;
1420 const std::complex<double> wide {static_cast<double>(value.real()), static_cast<double>(value.imag())};
1421 // Smith's division by n + 0j: a ratio of 0 and a scale of 1 / n, each product with that 0 exact, fused or not.
1422 const double scale {1.0 / static_cast<double>(n)};
1423 const double zero {0.0};
1424 return T {static_cast<R>((wide.real() + (wide.imag() * zero)) * scale), static_cast<R>((wide.imag() - (wide.real() * zero)) * scale)};
1425 }
1426
1429 template <typename T>
1430 [[nodiscard]] T divided_by_count(T value,
1431 std::size_t n)
1432 {
1433 if (n == 0) {
1434 throw dimension_error("a mean over no element has no value in an integer type");
1435 }
1436 return value / static_cast<T>(n);
1437 }
1438
1445 template <std::floating_point T,
1446 typename Order>
1448 const T* first,
1449 std::size_t count,
1450 std::size_t stride,
1451 const Order& larger)
1452 {
1453 if (value != T {0}) {
1454 return value;
1455 }
1456 const bool maximum {larger(T {1}, T {0})};
1457 for (std::size_t i {0}; i < count; ++i) {
1458 const T x {first[i * stride]};
1459 if (x == T {0} && std::signbit(x) != maximum) {
1460 return x;
1461 }
1462 }
1463 return value;
1464 }
1465
1467 template <typename T,
1468 typename Order>
1469 [[nodiscard]] T settled_zero(T value,
1470 const T* /*first*/,
1471 std::size_t /*count*/,
1472 std::size_t /*stride*/,
1473 const Order& /*larger*/)
1474 {
1475 return value;
1476 }
1477
1479 [[nodiscard]] inline std::vector<std::size_t> without_axis(std::vector<std::size_t> shape,
1480 std::size_t axis)
1481 {
1482 shape.erase(shape.begin() + static_cast<std::ptrdiff_t>(axis));
1483 return shape;
1484 }
1485
1490 template <typename T,
1491 typename Reduce>
1493 std::size_t axis,
1494 const Reduce& reduce)
1495 {
1496 const axis_walk walk {walk_along(t.shape(), axis)};
1497 buffer<T> out(walk.outer * walk.inner);
1498 parallel_for(walk.outer * walk.inner, walk.extent, [&](std::size_t at) { // each line its own, on whichever thread
1499 const std::size_t o {at / walk.inner};
1500 const std::size_t i {at % walk.inner};
1501 out[at] = reduce(t.data().data() + (o * walk.extent * walk.inner) + i, walk.extent, walk.inner);
1502 });
1503 return tensor<T> {without_axis(t.shape(), axis), std::move(out)};
1504 }
1505
1507 inline constexpr std::size_t band_columns {256};
1508
1515 template <typename R,
1516 typename T,
1517 typename Step>
1519 std::size_t axis,
1520 const R& init,
1521 const Step& step)
1522 {
1523 const axis_walk walk {walk_along(t.shape(), axis)};
1524 buffer<R> out(walk.outer * walk.inner);
1525 const T* const in {t.data().data()};
1526 if (walk.inner == 1) {
1527 parallel_for(walk.outer, walk.extent, [&](std::size_t o) {
1528 R running {init};
1529 for (std::size_t k {0}; k < walk.extent; ++k) {
1530 running = step(running, in[(o * walk.extent) + k]);
1531 }
1532 out[o] = running;
1533 });
1534 return tensor<R> {without_axis(t.shape(), axis), std::move(out)};
1535 }
1536 const std::size_t bands {(walk.inner + band_columns - 1) / band_columns};
1537 parallel_for(walk.outer * bands, walk.extent * band_columns, [&](std::size_t at) {
1538 const std::size_t o {at / bands};
1539 const std::size_t first {(at % bands) * band_columns};
1540 const std::size_t width {std::min(band_columns, walk.inner - first)};
1541 const std::size_t row {(o * walk.inner) + first}; // indexed: a buffer of bools has no data()
1542 for (std::size_t w {0}; w < width; ++w) {
1543 out[row + w] = init;
1544 }
1545 for (std::size_t k {0}; k < walk.extent; ++k) {
1546 const T* const x {in + (((o * walk.extent) + k) * walk.inner) + first};
1547 for (std::size_t w {0}; w < width; ++w) {
1548 out[row + w] = step(out[row + w], x[w]);
1549 }
1550 }
1551 });
1552 return tensor<R> {without_axis(t.shape(), axis), std::move(out)};
1553 }
1554
1556 template <typename T>
1557 [[nodiscard]] T square(const T& x)
1558 {
1559 return static_cast<T>(x * x);
1560 }
1561
1563 template <typename T>
1564 [[nodiscard]] bool is_nan(const T& /*x*/)
1565 {
1566 return false;
1567 }
1568
1570 [[nodiscard]] inline bool is_nan(double x)
1571 {
1572 return std::isnan(x);
1573 }
1574
1575#if defined(__GNUC__)
1577 template <typename V,
1578 typename T>
1579 [[nodiscard]] auto better(const V& x,
1580 const V& best,
1581 const std::greater<T>& /*larger*/)
1582 {
1583 return x > best;
1584 }
1585
1587 template <typename V,
1588 typename T>
1589 [[nodiscard]] auto better(const V& x,
1590 const V& best,
1591 const std::less<T>& /*smaller*/)
1592 {
1593 return x < best;
1594 }
1595
1597 template <typename V>
1598 [[nodiscard]] auto unordered(const V& a,
1599 const V& b)
1600 {
1601 return a != b;
1602 }
1603
1604#endif
1605
1606#ifdef NUMERICAL_NEON_COMPILED
1608 [[nodiscard]] inline float64x2_t lane_extreme(float64x2_t x,
1609 float64x2_t y,
1610 const std::greater<double>& /*larger*/) noexcept
1611 {
1612 return vmaxq_f64(x, y);
1613 }
1614
1616 [[nodiscard]] inline float64x2_t lane_extreme(float64x2_t x,
1617 float64x2_t y,
1618 const std::less<double>& /*smaller*/) noexcept
1619 {
1620 return vminq_f64(x, y);
1621 }
1622
1624 [[nodiscard]] inline float32x4_t lane_extreme(float32x4_t x,
1625 float32x4_t y,
1626 const std::greater<float>& /*larger*/) noexcept
1627 {
1628 return vmaxq_f32(x, y);
1629 }
1630
1632 [[nodiscard]] inline float32x4_t lane_extreme(float32x4_t x,
1633 float32x4_t y,
1634 const std::less<float>& /*smaller*/) noexcept
1635 {
1636 return vminq_f32(x, y);
1637 }
1638
1640 [[nodiscard]] inline double across_lanes(float64x2_t x,
1641 const std::greater<double>& /*larger*/) noexcept
1642 {
1643 return vmaxvq_f64(x);
1644 }
1645
1647 [[nodiscard]] inline double across_lanes(float64x2_t x,
1648 const std::less<double>& /*smaller*/) noexcept
1649 {
1650 return vminvq_f64(x);
1651 }
1652
1654 [[nodiscard]] inline float across_lanes(float32x4_t x,
1655 const std::greater<float>& /*larger*/) noexcept
1656 {
1657 return vmaxvq_f32(x);
1658 }
1659
1661 [[nodiscard]] inline float across_lanes(float32x4_t x,
1662 const std::less<float>& /*smaller*/) noexcept
1663 {
1664 return vminvq_f32(x);
1665 }
1666
1674 template <typename T,
1675 typename Order>
1676 [[nodiscard]] std::size_t neon_extreme(const T* first,
1677 std::size_t count,
1678 T& result,
1679 bool& nan,
1680 const Order& larger) noexcept
1681 {
1682 constexpr std::size_t width {16 / sizeof(T)}; // a register's lanes
1683 constexpr std::size_t block {4 * width};
1684 const std::size_t done {count - (count % block)};
1685 if (done == 0) {
1686 return 0;
1687 }
1688 auto b0 {neon_load(first)};
1689 auto b1 {neon_load(first + width)};
1690 auto b2 {neon_load(first + (2 * width))};
1691 auto b3 {neon_load(first + (3 * width))};
1692 for (std::size_t i {block}; i < done; i += block) {
1693 b0 = lane_extreme(b0, neon_load(first + i), larger);
1694 b1 = lane_extreme(b1, neon_load(first + i + width), larger);
1695 b2 = lane_extreme(b2, neon_load(first + i + (2 * width)), larger);
1696 b3 = lane_extreme(b3, neon_load(first + i + (3 * width)), larger);
1697 }
1698 const T value {across_lanes(lane_extreme(lane_extreme(b0, b1, larger), lane_extreme(b2, b3, larger), larger), larger)};
1699 nan = std::isnan(value);
1700 result = larger(value, result) ? value : result;
1701 return done;
1702 }
1703
1704#endif
1705
1711 template <typename T,
1712 typename Order>
1713 [[nodiscard]] T contiguous_extreme(const T* first,
1714 std::size_t count,
1715 const Order& larger)
1716 {
1717 T result {first[0]};
1718 bool nan {false};
1719 std::size_t done {0};
1720#if defined(NUMERICAL_NEON_COMPILED)
1721 done = neon_extreme(first, count, result, nan, larger);
1722#else
1723# ifdef NUMERICAL_X86_COMPILED
1724 bool wide {false}; // the leading elements done by AVX2's wider vectors
1725 if constexpr (std::is_same_v<T, double> || std::is_same_v<T, float>) {
1726 // The processor's AVX2, where the baseline build's vectors hold half as many: the same answer, whatever the
1727 // width (each lane its own best, a NaN flagged apart, a zero's sign settled after).
1728 if (const gemm_path path {gemm_path_current()}; path == gemm_path::avx2 || path == gemm_path::avx512) {
1729 done = avx2_extreme<std::is_same_v<Order, std::greater<T>>>(first, count, result, nan);
1730 wide = true;
1731 }
1732 }
1733# endif
1734# if defined(__GNUC__)
1735 using lane = typename native_vector<T>::type;
1736 constexpr std::size_t width {native_vector_bytes / sizeof(T)};
1737 constexpr std::size_t lanes {4}; // independent vectors, so no comparison waits on the one before it
1738# ifdef NUMERICAL_X86_COMPILED
1739 if (!wide && count >= width * lanes) {
1740# else
1741 if (count >= width * lanes) {
1742# endif
1743 using mask = decltype(lane {} > lane {}); // all ones in a lane where a comparison holds
1744 std::array<lane, lanes> best {};
1745 std::array<mask, lanes> nans {};
1746 for (std::size_t l {0}; l < lanes; ++l) {
1747 std::memcpy(&best[l], first + (l * width), sizeof(lane));
1748 }
1749 done = count - (count % (width * lanes));
1750 for (std::size_t i {0}; i < done; i += width * lanes) {
1751# pragma GCC unroll 4 // the lanes in registers: GCC keeps a rolled array of vectors in memory
1752 for (std::size_t l {0}; l < lanes; ++l) {
1753 lane x;
1754 std::memcpy(&x, first + i + (l * width), sizeof(lane));
1755 // Blended through the comparison's all-ones / all-zeros mask: both compilers keep it in vector
1756 // registers (GCC's `?:` on vectors does not).
1757 const mask take {better(x, best[l], larger)};
1758 best[l] = std::bit_cast<lane>((std::bit_cast<mask>(x) & take) | (std::bit_cast<mask>(best[l]) & ~take));
1759 nans[l] |= unordered(x, x);
1760 }
1761 }
1762 for (std::size_t l {0}; l < lanes; ++l) {
1763 for (std::size_t k {0}; k < width; ++k) {
1764 nan = nan || nans[l][k] != 0;
1765 result = larger(best[l][k], result) ? best[l][k] : result;
1766 }
1767 }
1768 }
1769# endif
1770#endif
1771 std::array<T, 8> best {};
1772 std::array<unsigned, 8> nans {};
1773 best.fill(result);
1774 for (std::size_t i {done}; i < count; ++i) {
1775 const T x {first[i]};
1776 nans[i % 8] |= static_cast<unsigned>(std::isnan(x));
1777 best[i % 8] = larger(x, best[i % 8]) ? x : best[i % 8];
1778 }
1779 for (std::size_t l {0}; l < 8; ++l) {
1780 nan = nan || nans[l] != 0U;
1781 result = larger(best[l], result) ? best[l] : result;
1782 }
1783 return nan ? std::numeric_limits<T>::quiet_NaN() : result;
1784 }
1785
1791 template <typename T,
1792 typename Order>
1793 [[nodiscard]] T parallel_extreme(const T* first,
1794 std::size_t count,
1795 const Order& larger)
1796 {
1797 if (count <= 4 * pairwise_leaf) {
1798 return contiguous_extreme(first, count, larger);
1799 }
1800 const std::size_t runs {(count + pairwise_leaf - 1) / pairwise_leaf};
1801 buffer<T> each(runs);
1802 parallel_for(runs, pairwise_leaf, [&](std::size_t r) {
1803 const std::size_t start {r * pairwise_leaf};
1804 each[r] = contiguous_extreme(first + start, std::min(pairwise_leaf, count - start), larger);
1805 });
1806 return contiguous_extreme(each.data(), runs, larger);
1807 }
1808
1811 template <typename T,
1812 typename Order>
1813 [[nodiscard]] T walked_extreme(const T* first,
1814 std::size_t count,
1815 std::size_t stride,
1816 const Order& larger)
1817 {
1818 T best {first[0]};
1819 for (std::size_t i {1}; i < count; ++i) {
1820 const T& candidate {first[i * stride]};
1821 if (is_nan(candidate) || larger(candidate, best)) {
1822 best = candidate;
1823 }
1824 }
1825 return settled_zero(best, first, count, stride, larger);
1826 }
1827
1831 template <std::floating_point T,
1832 typename Order>
1833 [[nodiscard]] T extreme(const T* first,
1834 std::size_t count,
1835 std::size_t stride,
1836 const Order& larger)
1837 {
1838 if (count == 0) {
1839 throw dimension_error("a zero-size reduction to a maximum or a minimum has no identity");
1840 }
1841 if (stride == 1) {
1842 return settled_zero(parallel_extreme(first, count, larger), first, count, 1, larger);
1843 }
1844 return walked_extreme(first, count, stride, larger);
1845 }
1846
1849 template <typename T,
1850 typename Order>
1851 requires(!std::floating_point<T>)
1852 [[nodiscard]] T extreme(const T* first,
1853 std::size_t count,
1854 std::size_t stride,
1855 const Order& larger)
1856 {
1857 if (count == 0) {
1858 throw dimension_error("a zero-size reduction to a maximum or a minimum has no identity");
1859 }
1860 return walked_extreme(first, count, stride, larger);
1861 }
1862
1866 template <typename T>
1867 [[nodiscard]] constexpr bool by_reduction(std::size_t inner) noexcept
1868 {
1869 return !std::is_floating_point_v<T> || inner == 1;
1870 }
1871
1877 template <typename T,
1878 typename Order>
1879 [[nodiscard]] tensor<T> extremes_along(const tensor<T>& t,
1880 std::size_t axis,
1881 const Order& larger)
1882 {
1883 const axis_walk walk {walk_along(t.shape(), axis)};
1884 // An empty axis has no maximum even when the result is empty too (NumPy refuses both).
1885 if (walk.extent == 0) {
1886 throw dimension_error("a zero-size reduction to a maximum or a minimum has no identity");
1887 }
1888 if (by_reduction<T>(walk.inner)) {
1889 return reduce_along(t, axis, [&larger](const T* first, std::size_t count, std::size_t stride) {
1890 return extreme(first, count, stride, larger);
1891 });
1892 }
1893 // Each slab's runs of extreme_columns columns shared among the threads: each element compared alike by whichever.
1894 // Each slab a task, compared row by row along its whole width (a strip of its columns at a time read each row in
1895 // pieces, 2.6 times as long on one arm64 thread): each element compared alike by whichever thread.
1896 buffer<T> out(walk.outer * walk.inner);
1897 const std::size_t inner {walk.inner};
1898 const std::size_t extent {walk.extent};
1899 const T* const data {t.data().data()};
1900 T* const into {out.data()};
1901 parallel_for(walk.outer, extent * inner, [ =, &larger](std::size_t o) {
1902 const T* const slab {data + (o * extent * inner)};
1903 T* const best {into + (o * inner)};
1904 std::vector<unsigned> nan(inner, 0U);
1905 std::copy(slab, slab + inner, best);
1906 for (std::size_t e {0}; e < extent; ++e) {
1907 const T* const row {slab + (e * inner)};
1908 for (std::size_t i {0}; i < inner; ++i) {
1909 const T x {row[i]};
1910 nan[i] |= static_cast<unsigned>(std::isnan(x));
1911 best[i] = larger(x, best[i]) ? x : best[i];
1912 }
1913 }
1914 for (std::size_t i {0}; i < inner; ++i) {
1915 best[i] = settled_zero(best[i], slab + i, extent, inner, larger);
1916 if (nan[i] != 0U) {
1917 best[i] = std::numeric_limits<T>::quiet_NaN();
1918 }
1919 }
1920 });
1921 return tensor<T> {without_axis(t.shape(), axis), std::move(out)};
1922 }
1923 } // namespace detail
1924
1925 namespace detail {
1926
1929 [[nodiscard]] inline std::vector<std::size_t> view_axes(const std::vector<std::size_t>& order)
1930 {
1931 std::vector<std::size_t> back(order.size());
1932 for (std::size_t k {0}; k < order.size(); ++k) {
1933 back[order[k]] = k;
1934 }
1935 return back;
1936 }
1937
1940 [[nodiscard]] inline std::vector<std::size_t> reduced_axes(const std::vector<std::size_t>& order,
1941 std::size_t axis)
1942 {
1943 std::vector<std::size_t> rest;
1944 for (const std::size_t view_axis : order) {
1945 if (view_axis != axis) {
1946 rest.push_back(view_axis > axis ? view_axis - 1 : view_axis);
1947 }
1948 }
1949 return view_axes(rest);
1950 }
1951
1953 template <typename T>
1955 {
1957 std::vector<std::size_t> order;
1958 std::size_t along {0};
1959 };
1960
1963 template <typename T>
1964 [[nodiscard]] std::optional<laid_axis<T>> laid_along(const tensor<T>& t,
1965 std::size_t axis)
1966 {
1967 auto laid {t.laid_out()};
1968 if (!laid || axis >= t.rank()) {
1969 return std::nullopt;
1970 }
1971 const auto along {static_cast<std::size_t>(std::ranges::find(laid->second, axis) - laid->second.begin())};
1972 return laid_axis<T> {std::move(laid->first), std::move(laid->second), along};
1973 }
1974
1984 template <typename T,
1985 typename F>
1986 [[nodiscard]] auto as_laid_along(const tensor<T>& t,
1987 std::size_t axis,
1988 const F& f) -> std::optional<decltype(f(t, axis))>
1989 {
1990 const auto laid {laid_along(t, axis)};
1991 if (!laid || (axis + 1 == t.rank() && laid->along + 1 != t.rank())) {
1992 return std::nullopt;
1993 }
1994 return f(laid->elements, laid->along).transposed(view_axes(laid->order));
1995 }
1996
1999 template <typename T,
2000 typename F>
2001 [[nodiscard]] auto as_laid_reduced(const tensor<T>& t,
2002 std::size_t axis,
2003 const F& f) -> std::optional<decltype(f(t, axis))>
2004 {
2005 const auto laid {laid_along(t, axis)};
2006 if (!laid) {
2007 return std::nullopt;
2008 }
2009 return f(laid->elements, laid->along).transposed(reduced_axes(laid->order, axis));
2010 }
2011
2018 template <typename T>
2019 [[nodiscard]] bool lies_close(const tensor<T>& t) noexcept
2020 {
2021 const std::size_t rank {t.shape().size()}; // a view has an axis: a rank-0 tensor's one element lies row-major
2022 const std::ptrdiff_t step {t.strides()[rank - 1]};
2023 const auto span {static_cast<std::size_t>(step < 0 ? -step : step) * sizeof(T)};
2024 return t.shape()[rank - 1] < 2 ? rank == 1 : span <= 64;
2025 }
2026
2031 template <typename T,
2032 typename F>
2033 [[nodiscard]] auto parallel_map(const T* data,
2034 const std::vector<std::size_t>& shape,
2036 {
2037 using R = std::decay_t<decltype(f(std::declval<const T&>()))>;
2038 const std::size_t count {element_count(shape.begin(), shape.end())};
2039 buffer<R> out(count);
2040 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
2041 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
2042 const std::size_t first {c * elementwise_chunk};
2043 const std::size_t last {std::min(first + elementwise_chunk, count)};
2044 std::transform(data + first, data + last, out.begin() + static_cast<std::ptrdiff_t>(first), f);
2045 });
2046 return tensor<R> {shape, std::move(out)};
2047 }
2048
2053 template <typename T,
2054 typename F>
2055 [[nodiscard]] auto parallel_map(const tensor<T>& t,
2057 {
2058 if (t.contiguous()) { // an empty view holds a buffer of its own, row-major
2059 return parallel_map(t.data().data(), t.shape(), f);
2060 }
2061 if (!lies_close(t)) {
2062 if (const auto laid {t.laid_out()}) { // a transpose: its elements as they lie, the result's axes put back
2063 return parallel_map(laid->first.data().data(), laid->first.shape(), f).transposed(view_axes(laid->second));
2064 }
2065 return parallel_map(t.data().data(), t.shape(), f);
2066 }
2067 using R = std::decay_t<decltype(f(std::declval<const T&>()))>;
2068 const std::vector<std::size_t>& shape {t.shape()};
2069 const std::vector<std::ptrdiff_t>& strides {t.strides()};
2070 const std::size_t rank {shape.size()};
2071 const std::size_t count {t.size()};
2072 const std::size_t inner {shape[rank - 1]};
2073 const std::ptrdiff_t step {strides[rank - 1]};
2074 const T* const origin {t.origin()};
2075 buffer<R> out(count);
2076 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
2077 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
2078 const std::size_t first {c * elementwise_chunk};
2079 const std::size_t last {std::min(first + elementwise_chunk, count)};
2080 std::vector<std::size_t> index(rank, 0);
2081 std::ptrdiff_t at {0};
2082 std::size_t column {first % inner};
2083 for (std::size_t axis {rank - 1}, rest {first / inner}; axis-- > 0;) { // the first row's position
2084 index[axis] = rest % shape[axis];
2085 rest /= shape[axis];
2086 at += static_cast<std::ptrdiff_t>(index[axis]) * strides[axis];
2087 }
2088 for (std::size_t done {first}; done < last;) {
2089 const std::size_t n {std::min(inner - column, last - done)};
2090 const T* const p {origin + at + (static_cast<std::ptrdiff_t>(column) * step)};
2091 R* const o {out.data() + done};
2092 if (step == 1) {
2093 std::transform(p, p + n, o, f);
2094 }
2095 else {
2096 for (std::size_t i {0}; i < n; ++i) {
2097 o[i] = f(p[static_cast<std::ptrdiff_t>(i) * step]);
2098 }
2099 }
2100 done += n;
2101 column = 0;
2102 for (std::size_t axis {rank - 1}; axis-- > 0;) { // odometer over the outer axes: carry on wrap
2103 ++index[axis];
2104 at += strides[axis];
2105 if (index[axis] < shape[axis]) {
2106 break;
2107 }
2108 at -= strides[axis] * static_cast<std::ptrdiff_t>(shape[axis]);
2109 index[axis] = 0;
2110 }
2111 }
2112 });
2113 return tensor<R> {shape, std::move(out)};
2114 }
2115
2123 template <typename T,
2124 typename Op>
2125 [[nodiscard]] auto broadcast_rows(const T* a,
2126 const std::vector<std::size_t>& a_shape,
2127 const T* b,
2128 const std::vector<std::size_t>& b_shape,
2129 const Op& op)
2130 {
2131 if (a_shape == b_shape) {
2132 return pairs_into(a, b, a_shape, op);
2133 }
2134 if (element_count(b_shape.begin(), b_shape.end()) == 1 && b_shape.size() <= a_shape.size()) {
2135 return parallel_map(a, a_shape, [&op, y = *b](const T& x) { return op(x, y); });
2136 }
2137 if (element_count(a_shape.begin(), a_shape.end()) == 1 && a_shape.size() <= b_shape.size()) {
2138 return parallel_map(b, b_shape, [&op, x = *a](const T& y) { return op(x, y); });
2139 }
2140 return broadcast_into(a, a_shape, b, b_shape, op);
2141 }
2142 } // namespace detail
2143
2151 template <typename T,
2152 typename Op>
2153 [[nodiscard]] auto broadcast_apply(const tensor<T>& a,
2154 const tensor<T>& b,
2155 const Op& op)
2156 {
2157 if (a.contiguous() && b.contiguous()) { // the common case, straight through
2158 return detail::broadcast_rows(a.origin(), a.shape(), b.origin(), b.shape(), op);
2159 }
2160 if (b.size() == 1 && b.rank() <= a.rank()) { // the result of a's shape
2161 return detail::parallel_map(a, [&op, y = *b.origin()](const T& x) { return op(x, y); });
2162 }
2163 if (a.size() == 1 && a.rank() <= b.rank()) {
2164 return detail::parallel_map(b, [&op, x = *a.origin()](const T& y) { return op(x, y); });
2165 }
2166 if (a.shape() == b.shape()) { // the same permutation's elements, the same element of each at the same place
2167 const auto a_laid {a.laid_out()};
2168 const auto b_laid {b.laid_out()};
2169 if (a_laid && b_laid && a_laid->second == b_laid->second) {
2170 return detail::pairs_into(a_laid->first.origin(), b_laid->first.origin(), a_laid->first.shape(), op)
2171 .transposed(detail::view_axes(a_laid->second));
2172 }
2173 }
2174 const bool a_close {a.contiguous() || detail::lies_close(a)};
2175 const bool b_close {b.contiguous() || detail::lies_close(b)};
2176 if (a_close && b_close) { // a row-major tensor's strides are its own: nothing to build
2177 return detail::broadcast_into(a.origin(), a.shape(), a.strides(), b.origin(), b.shape(), b.strides(), op);
2178 }
2179 return detail::broadcast_into(a_close ? a.origin() : a.data().data(), a.shape(),
2180 a_close ? a.strides() : detail::row_major_strides(a.shape()),
2181 b_close ? b.origin() : b.data().data(), b.shape(),
2182 b_close ? b.strides() : detail::row_major_strides(b.shape()), op);
2183 }
2184
2185 namespace detail {
2186 // NumPy's loops over tensors, for the binding beside a NumPy: no part of the library's own algebra, which keeps
2187 // C's Annex G (the operators, pow).
2188
2199
2201 template <split_complex C>
2202 [[nodiscard]] C numpy_reduction_multiply(const C& a,
2203 const C& b,
2204 const numpy_loops& loops)
2205 {
2206 return loops.reductions && loops.late_reductions ? numpy_multiply_late(a, b)
2207 : numpy_multiply(a, b, loops.reductions);
2208 }
2209
2212 [[nodiscard]] inline numpy_loops numpy_wheel_loops() noexcept
2213 {
2214 return numpy_loops {.products = detail::numpy_fuses_products(),
2215 .scalar_products = detail::numpy_fuses_scalar_products(),
2216 .quotients = detail::numpy_fuses_quotients(),
2217 .reductions = detail::numpy_fuses_reductions(),
2218 .late_reductions = false};
2219 }
2220
2226 template <detail::split_complex C>
2227 [[nodiscard]] tensor<C> numpy_multiply(const tensor<C>& a,
2228 const tensor<C>& b,
2229 const numpy_loops& loops = numpy_wheel_loops())
2230 {
2231 const bool fused {detail::numpy_loop_is_scalar(a, b) ? loops.scalar_products : loops.products};
2232 return broadcast_apply(a, b, [fused](const C& x, const C& y) { return detail::numpy_multiply(x, y, fused); });
2233 }
2234
2237 template <detail::split_complex C>
2238 [[nodiscard]] tensor<C> numpy_divide(const tensor<C>& a,
2239 const tensor<C>& b,
2240 const numpy_loops& loops = numpy_wheel_loops())
2241 {
2242 const bool fused {loops.quotients};
2243 return broadcast_apply(a, b, [fused](const C& x, const C& y) { return detail::numpy_divide(x, y, fused); });
2244 }
2245
2253 template <detail::split_complex C>
2254 [[nodiscard]] tensor<C> numpy_prod(const tensor<C>& t,
2255 std::size_t axis,
2256 const numpy_loops& loops = numpy_wheel_loops())
2257 {
2258 // The loop the shape asks for, whichever axis of a transpose's elements the line lies along.
2259 const bool reduction {walk_along(t.shape(), axis).inner == 1};
2260 const auto multiply {[reduction, &loops](const C& product, const C& x) {
2261 return reduction ? numpy_reduction_multiply(product, x, loops) : numpy_multiply(product, x, loops.products);
2262 }};
2263 if (auto laid {as_laid_reduced(t, axis, [&multiply](const tensor<C>& elements, std::size_t along) {
2264 return fold_along(elements, along, C {1}, multiply);
2265 })}) {
2266 return std::move(*laid);
2267 }
2268 return fold_along(t, axis, C {1}, multiply);
2269 }
2270
2273 template <detail::split_complex C>
2274 [[nodiscard]] C numpy_prod(const tensor<C>& t,
2275 const numpy_loops& loops = numpy_wheel_loops())
2276 {
2277 C product {1};
2278 for (const C& x : t.data()) {
2279 product = numpy_reduction_multiply(product, x, loops);
2280 }
2281 return product;
2282 }
2283
2286 template <detail::split_complex C>
2288 const numpy_loops& loops = numpy_wheel_loops())
2289 {
2290 const bool fused {loops.products};
2291 return detail::parallel_map(a, [fused](const C& x) { return detail::numpy_absolute(x, fused); });
2292 }
2293
2296 template <detail::split_complex C>
2297 [[nodiscard]] tensor<C> numpy_square(const tensor<C>& a,
2298 const numpy_loops& loops = numpy_wheel_loops())
2299 {
2300 const bool fused {loops.products};
2301 return detail::parallel_map(a, [fused](const C& x) { return detail::numpy_multiply(x, x, fused); });
2302 }
2303
2306 template <detail::split_complex C>
2307 [[nodiscard]] tensor<C> numpy_reciprocal(const tensor<C>& a,
2308 const numpy_loops& loops = numpy_wheel_loops())
2309 {
2310 const bool fused {loops.quotients};
2311 return detail::parallel_map(a, [fused](const C& x) { return detail::numpy_reciprocal(x, fused); });
2312 }
2313
2316 template <detail::split_complex C>
2317 [[nodiscard]] tensor<C> numpy_power(const tensor<C>& a,
2318 const tensor<C>& b,
2319 const numpy_loops& loops = numpy_wheel_loops())
2320 {
2321 const bool fused {loops.quotients};
2322 return broadcast_apply(a, b, [fused](const C& x, const C& y) { return detail::numpy_power(x, y, fused); });
2323 }
2324 } // namespace detail
2325
2326 namespace detail {
2328 template <typename T>
2329 [[nodiscard]] tensor<T> exp_of(const T* data,
2330 const std::vector<std::size_t>& shape)
2331 {
2332 return parallel_map(data, shape, [](const T& x) { return elementary_exp(x); });
2333 }
2334
2337 [[nodiscard]] inline tensor<double> exp_of(const double* data,
2338 const std::vector<std::size_t>& shape)
2339 {
2340 const std::size_t count {element_count(shape.begin(), shape.end())};
2341 buffer<double> out(count);
2342 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
2343 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
2344 const std::size_t first {c * elementwise_chunk};
2345 exp_into(data + first, out.data() + first, std::min(elementwise_chunk, count - first));
2346 });
2347 return tensor<double> {shape, std::move(out)};
2348 }
2349
2351 using doubles_into = void (*)(const double*, double*, std::size_t);
2352
2356 {
2358
2360 void operator()(const float* in,
2361 float* out,
2362 std::size_t count) const
2363 {
2364 constexpr std::size_t piece {512};
2365 std::array<double, piece> wide {};
2366 std::array<double, piece> image {};
2367 for (std::size_t first {0}; first < count; first += piece) {
2368 const std::size_t size {std::min(piece, count - first)};
2369 std::copy_n(in + first, size, wide.begin());
2370 into(wide.data(), image.data(), size);
2371 std::transform(image.begin(), image.begin() + static_cast<std::ptrdiff_t>(size), out + first, [](double y) { return static_cast<float>(y); });
2372 }
2373 }
2374 };
2375
2384 template <typename T,
2385 typename Kernel>
2386 [[nodiscard]] tensor<T> runs_into(const tensor<T>& t,
2387 const Kernel& kernel)
2388 {
2389 const std::vector<std::size_t>& shape {t.shape()};
2390 const std::size_t count {t.size()};
2391 buffer<T> out(count);
2392 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
2393 const bool far {!t.contiguous() && !lies_close(t)};
2394 if (far) {
2395 if (const auto laid {t.laid_out()}) { // a transpose: its elements as they lie, the result's axes put back
2396 return runs_into(laid->first, kernel).transposed(view_axes(laid->second));
2397 }
2398 }
2399 if (t.contiguous() || far) {
2400 const T* const data {t.contiguous() ? t.origin() : t.data().data()};
2401 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
2402 const std::size_t first {c * elementwise_chunk};
2403 kernel(data + first, out.data() + first, std::min(elementwise_chunk, count - first));
2404 });
2405 return tensor<T> {shape, std::move(out)};
2406 }
2407 const std::vector<std::ptrdiff_t>& strides {t.strides()};
2408 const std::size_t rank {shape.size()};
2409 const std::size_t inner {shape[rank - 1]};
2410 const std::ptrdiff_t step {strides[rank - 1]};
2411 const T* const origin {t.origin()};
2412 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
2413 constexpr std::size_t piece {512};
2414 std::array<T, piece> gathered {};
2415 const std::size_t first {c * elementwise_chunk};
2416 const std::size_t last {std::min(first + elementwise_chunk, count)};
2417 std::vector<std::size_t> index(rank, 0);
2418 std::ptrdiff_t at {0};
2419 std::size_t column {first % inner};
2420 for (std::size_t axis {rank - 1}, rest {first / inner}; axis-- > 0;) { // the first row's position
2421 index[axis] = rest % shape[axis];
2422 rest /= shape[axis];
2423 at += static_cast<std::ptrdiff_t>(index[axis]) * strides[axis];
2424 }
2425 for (std::size_t done {first}; done < last;) {
2426 const std::size_t n {std::min(inner - column, last - done)};
2427 const T* const p {origin + at + (static_cast<std::ptrdiff_t>(column) * step)};
2428 if (step == 1) {
2429 kernel(p, out.data() + done, n);
2430 }
2431 else {
2432 for (std::size_t k {0}; k < n; k += piece) {
2433 const std::size_t size {std::min(piece, n - k)};
2434 for (std::size_t i {0}; i < size; ++i) {
2435 gathered[i] = p[static_cast<std::ptrdiff_t>(k + i) * step];
2436 }
2437 kernel(gathered.data(), out.data() + done + k, size);
2438 }
2439 }
2440 done += n;
2441 column = 0;
2442 for (std::size_t axis {rank - 1}; axis-- > 0;) { // odometer over the outer axes: carry on wrap
2443 ++index[axis];
2444 at += strides[axis];
2445 if (index[axis] < shape[axis]) {
2446 break;
2447 }
2448 at -= strides[axis] * static_cast<std::ptrdiff_t>(shape[axis]);
2449 index[axis] = 0;
2450 }
2451 }
2452 });
2453 return tensor<T> {shape, std::move(out)};
2454 }
2455
2457 [[nodiscard]] inline tensor<double> by_runs(const tensor<double>& a,
2458 doubles_into into)
2459 {
2460 return runs_into(a, into);
2461 }
2462
2464 [[nodiscard]] inline tensor<float> by_runs(const tensor<float>& a,
2465 doubles_into into)
2466 {
2467 return runs_into(a, through_doubles_piece {into});
2468 }
2469
2471 template <typename T>
2472 [[nodiscard]] tensor<T> exp_viewed(const tensor<T>& a)
2473 {
2474 return exp_of(a.data().data(), a.shape());
2475 }
2476
2478 [[nodiscard]] inline tensor<double> exp_viewed(const tensor<double>& a)
2479 {
2480 return by_runs(a, &exp_into);
2481 }
2482
2484 template <typename T>
2485 [[nodiscard]] tensor<T> expm1_viewed(const tensor<T>& a)
2486 {
2487 return expm1_of(a.data().data(), a.shape());
2488 }
2489
2491 [[nodiscard]] inline tensor<double> expm1_viewed(const tensor<double>& a)
2492 {
2493 return by_runs(a, &expm1_into);
2494 }
2495
2497 [[nodiscard]] inline tensor<float> expm1_viewed(const tensor<float>& a)
2498 {
2499 return by_runs(a, &expm1_into);
2500 }
2501
2503 template <typename T>
2504 [[nodiscard]] tensor<T> atan_viewed(const tensor<T>& a)
2505 {
2506 return atan_of(a.data().data(), a.shape());
2507 }
2508
2510 [[nodiscard]] inline tensor<double> atan_viewed(const tensor<double>& a)
2511 {
2512 return by_runs(a, &atan_into);
2513 }
2514
2516 [[nodiscard]] inline tensor<float> atan_viewed(const tensor<float>& a)
2517 {
2518 return by_runs(a, &atan_into);
2519 }
2520
2522 template <typename T>
2523 [[nodiscard]] tensor<T> sinh_viewed(const tensor<T>& a)
2524 {
2525 return sinh_of(a.data().data(), a.shape());
2526 }
2527
2529 [[nodiscard]] inline tensor<double> sinh_viewed(const tensor<double>& a)
2530 {
2531 return by_runs(a, &sinh_into);
2532 }
2533
2535 [[nodiscard]] inline tensor<float> sinh_viewed(const tensor<float>& a)
2536 {
2537 return by_runs(a, &sinh_into);
2538 }
2539
2541 template <typename T>
2542 [[nodiscard]] tensor<T> cosh_viewed(const tensor<T>& a)
2543 {
2544 return cosh_of(a.data().data(), a.shape());
2545 }
2546
2548 [[nodiscard]] inline tensor<double> cosh_viewed(const tensor<double>& a)
2549 {
2550 return by_runs(a, &cosh_into);
2551 }
2552
2554 [[nodiscard]] inline tensor<float> cosh_viewed(const tensor<float>& a)
2555 {
2556 return by_runs(a, &cosh_into);
2557 }
2558
2560 template <typename T>
2561 [[nodiscard]] tensor<T> exp2_viewed(const tensor<T>& a)
2562 {
2563 return exp2_of(a.data().data(), a.shape());
2564 }
2565
2567 [[nodiscard]] inline tensor<double> exp2_viewed(const tensor<double>& a)
2568 {
2569 return by_runs(a, &exp2_into);
2570 }
2571
2573 template <typename T>
2574 [[nodiscard]] tensor<T> log1p_viewed(const tensor<T>& a)
2575 {
2576 return log1p_of(a.data().data(), a.shape());
2577 }
2578
2580 [[nodiscard]] inline tensor<double> log1p_viewed(const tensor<double>& a)
2581 {
2582 return by_runs(a, &log1p_into);
2583 }
2584
2586 [[nodiscard]] inline tensor<float> log1p_viewed(const tensor<float>& a)
2587 {
2588 return by_runs(a, &log1p_into);
2589 }
2590
2592 template <typename T>
2593 [[nodiscard]] tensor<T> sin_viewed(const tensor<T>& a)
2594 {
2595 return sin_of(a.data().data(), a.shape());
2596 }
2597
2599 [[nodiscard]] inline tensor<double> sin_viewed(const tensor<double>& a)
2600 {
2601 return by_runs(a, &double_sines_into<false>);
2602 }
2603
2605 [[nodiscard]] inline tensor<float> sin_viewed(const tensor<float>& a)
2606 {
2607 return by_runs(a, &float_sines_into<false>);
2608 }
2609
2611 template <typename T>
2612 [[nodiscard]] tensor<T> cos_viewed(const tensor<T>& a)
2613 {
2614 return cos_of(a.data().data(), a.shape());
2615 }
2616
2618 [[nodiscard]] inline tensor<double> cos_viewed(const tensor<double>& a)
2619 {
2620 return by_runs(a, &double_sines_into<true>);
2621 }
2622
2624 [[nodiscard]] inline tensor<float> cos_viewed(const tensor<float>& a)
2625 {
2626 return by_runs(a, &float_sines_into<true>);
2627 }
2628
2630 template <typename T>
2631 [[nodiscard]] tensor<T> tanh_viewed(const tensor<T>& a)
2632 {
2633 return tanh_of(a.data().data(), a.shape());
2634 }
2635
2637 [[nodiscard]] inline tensor<double> tanh_viewed(const tensor<double>& a)
2638 {
2639 return by_runs(a, &tanh_into);
2640 }
2641
2643 [[nodiscard]] inline tensor<float> tanh_viewed(const tensor<float>& a)
2644 {
2645 return by_runs(a, &tanh_into);
2646 }
2647 } // namespace detail
2648
2653 template <typename T>
2654 [[nodiscard]] tensor<T> exp(const tensor<T>& a)
2655 {
2656 return detail::exp_viewed(a);
2657 }
2658
2663 template <std::floating_point T>
2664 [[nodiscard]] tensor<T> expm1(const tensor<T>& a)
2665 {
2666 return detail::expm1_viewed(a);
2667 }
2668
2673 template <std::floating_point T>
2674 [[nodiscard]] tensor<T> atan(const tensor<T>& a)
2675 {
2676 return detail::atan_viewed(a);
2677 }
2678
2683 template <std::floating_point T>
2684 [[nodiscard]] tensor<T> sinh(const tensor<T>& a)
2685 {
2686 return detail::sinh_viewed(a);
2687 }
2688
2693 template <std::floating_point T>
2694 [[nodiscard]] tensor<T> cosh(const tensor<T>& a)
2695 {
2696 return detail::cosh_viewed(a);
2697 }
2698
2703 template <std::floating_point T>
2704 [[nodiscard]] tensor<T> exp2(const tensor<T>& a)
2705 {
2706 return detail::exp2_viewed(a);
2707 }
2708
2710 template <typename T>
2711 [[nodiscard]] tensor<T> log(const tensor<T>& a)
2712 {
2713 return detail::parallel_map(a, [](const T& x) { return detail::elementary_log(x); });
2714 }
2715
2720 template <std::floating_point T>
2721 [[nodiscard]] tensor<T> log1p(const tensor<T>& a)
2722 {
2723 return detail::log1p_viewed(a);
2724 }
2725
2728 template <typename T>
2729 [[nodiscard]] tensor<T> sin(const tensor<T>& a)
2730 {
2731 return detail::sin_viewed(a);
2732 }
2733
2736 template <typename T>
2737 [[nodiscard]] tensor<T> cos(const tensor<T>& a)
2738 {
2739 return detail::cos_viewed(a);
2740 }
2741
2743 template <typename T>
2744 [[nodiscard]] tensor<T> tan(const tensor<T>& a)
2745 {
2746 return detail::parallel_map(a, [](const T& x) { return detail::elementary_tan(x); });
2747 }
2748
2753 template <typename T>
2754 [[nodiscard]] tensor<T> tanh(const tensor<T>& a)
2755 {
2756 return detail::tanh_viewed(a);
2757 }
2758
2760 template <typename T>
2761 [[nodiscard]] tensor<T> sqrt(const tensor<T>& a)
2762 {
2763 return detail::parallel_map(a, [](const T& x) { return detail::root(x); });
2764 }
2765
2766 namespace detail {
2768 template <typename T>
2769 struct arithmetic {
2770 using plus = std::plus<T>;
2771 using minus = std::minus<T>;
2772 using multiplies = std::multiplies<T>;
2773 using negate = std::negate<T>;
2774 };
2775
2778 template <std::signed_integral T>
2779 struct arithmetic<T> {
2780 using unsigned_type = std::make_unsigned_t<decltype(T {} + 0U)>;
2781
2783 struct plus {
2784 [[nodiscard]] T operator()(T a,
2785 T b) const noexcept
2786 {
2787 return static_cast<T>(static_cast<unsigned_type>(a) + static_cast<unsigned_type>(b));
2788 }
2789 };
2790
2792 struct minus {
2793 [[nodiscard]] T operator()(T a,
2794 T b) const noexcept
2795 {
2796 return static_cast<T>(static_cast<unsigned_type>(a) - static_cast<unsigned_type>(b));
2797 }
2798 };
2799
2801 struct multiplies {
2802 [[nodiscard]] T operator()(T a,
2803 T b) const noexcept
2804 {
2805 return static_cast<T>(static_cast<unsigned_type>(a) * static_cast<unsigned_type>(b));
2806 }
2807 };
2808
2810 struct negate {
2811 [[nodiscard]] T operator()(T a) const noexcept
2812 {
2813 return static_cast<T>(unsigned_type {0} - static_cast<unsigned_type>(a));
2814 }
2815 };
2816 };
2817 } // namespace detail
2818
2821 template <typename T>
2822 [[nodiscard]] tensor<T> operator+(const tensor<T>& a,
2823 const tensor<T>& b)
2824 {
2825 return broadcast_apply(a, b, typename detail::arithmetic<T>::plus {});
2826 }
2827
2828 namespace detail {
2829
2832 template <std::floating_point T>
2833 [[nodiscard]] T ieee_maximum(T a,
2834 T b) noexcept
2835 {
2836 if (std::isnan(a) || std::isnan(b)) {
2837 return std::isnan(a) ? a : b;
2838 }
2839 if (a == b) {
2840 return a == T {0} ? a + b : a;
2841 }
2842 return a > b ? a : b;
2843 }
2844
2847 template <std::floating_point T>
2848 [[nodiscard]] T ieee_minimum(T a,
2849 T b) noexcept
2850 {
2851 if (std::isnan(a) || std::isnan(b)) {
2852 return std::isnan(a) ? a : b;
2853 }
2854 if (a == b) {
2855 return a == T {0} ? -(-a - b) : a;
2856 }
2857 return a < b ? a : b;
2858 }
2859
2862 template <std::floating_point T>
2863 [[nodiscard]] T masked_select(bool condition,
2864 T if_true,
2865 T if_false) noexcept
2866 {
2867 using bits = std::conditional_t<sizeof(T) == 8, std::uint64_t, std::uint32_t>;
2868 static_assert(sizeof(T) == sizeof(bits), "a float or a double");
2869 const bits mask {bits {0} - static_cast<bits>(condition)};
2870 return std::bit_cast<T>((std::bit_cast<bits>(if_true) & mask) | (std::bit_cast<bits>(if_false) & ~mask));
2871 }
2872
2875 template <std::floating_point T>
2876 [[nodiscard]] T clipped(T x,
2877 T low,
2878 T high) noexcept
2879 {
2880 const T up {masked_select((static_cast<unsigned>(x == low) & static_cast<unsigned>(x == T {0})) != 0U, x + low, masked_select(x < low, low, x))};
2881 return masked_select((static_cast<unsigned>(up == high) & static_cast<unsigned>(up == T {0})) != 0U, -(-up - high), masked_select(up > high, high, up));
2882 }
2883 } // namespace detail
2884
2887 template <std::floating_point T>
2888 [[nodiscard]] tensor<T> maximum(const tensor<T>& a,
2889 const tensor<T>& b)
2890 {
2891 if (b.shape().empty()) { // a number (clip's bound): a map over a, where broadcasting walks both
2892 const T bound {b.data()[0]};
2893 return detail::parallel_map(a, [bound](T x) { return detail::ieee_maximum(x, bound); });
2894 }
2895 return broadcast_apply(a, b, [](T x, T y) { return detail::ieee_maximum(x, y); });
2896 }
2897
2900 template <std::floating_point T>
2901 [[nodiscard]] tensor<T> minimum(const tensor<T>& a,
2902 const tensor<T>& b)
2903 {
2904 if (b.shape().empty()) { // a number (clip's bound): a map over a, where broadcasting walks both
2905 const T bound {b.data()[0]};
2906 return detail::parallel_map(a, [bound](T x) { return detail::ieee_minimum(x, bound); });
2907 }
2908 return broadcast_apply(a, b, [](T x, T y) { return detail::ieee_minimum(x, y); });
2909 }
2910
2911 namespace detail {
2921 template <std::floating_point R,
2922 std::floating_point C>
2927
2928 explicit decimal_rounding(int places) noexcept
2929 : decimals {places}
2930 , scale {std::pow(C {10}, static_cast<C>(places < 0 ? -places : places))}
2931 , exact_from {std::ldexp(C {1}, std::numeric_limits<R>::digits) / scale}
2932 {}
2933
2934 [[nodiscard]] R operator()(R x) const noexcept
2935 {
2936 const C wide {static_cast<C>(x)};
2937 if (decimals > 0) {
2938 return std::fabs(wide) < exact_from ? static_cast<R>(std::nearbyint(wide * scale) / scale) : x;
2939 }
2940 if (std::isinf(scale)) {
2941 return std::isfinite(x) ? std::copysign(R {0}, x) : x;
2942 }
2943 return static_cast<R>(std::nearbyint(wide / scale) * scale);
2944 }
2945 };
2946
2949 template <std::floating_point R,
2950 typename F>
2951 requires(!std::same_as<R, float>)
2952 [[nodiscard]] auto with_decimal_rounding(int decimals,
2953 const F& f)
2954 {
2955 return f(decimal_rounding<R, R> {decimals});
2956 }
2957
2959 template <std::same_as<float> R,
2960 typename F>
2961 [[nodiscard]] auto with_decimal_rounding(int decimals,
2962 const F& f)
2963 {
2964 if (decimals > std::numeric_limits<float>::max_exponent10) {
2965 return f(decimal_rounding<float, double> {decimals});
2966 }
2967 return f(decimal_rounding<float, float> {decimals});
2968 }
2969 } // namespace detail
2970
2973 template <std::floating_point T>
2974 [[nodiscard]] tensor<T> round(const tensor<T>& a,
2975 int decimals = 0)
2976 {
2977 return detail::with_decimal_rounding<T>(decimals, [&a](const auto& rounding) { return detail::parallel_map(a, rounding); });
2978 }
2979
2982 template <detail::split_complex C>
2983 [[nodiscard]] tensor<C> round(const tensor<C>& a,
2984 int decimals = 0)
2985 {
2986 return detail::with_decimal_rounding<typename C::value_type>(decimals, [&a](const auto& rounding) {
2987 return detail::parallel_map(a, [&rounding](const C& z) { return C {rounding(z.real()), rounding(z.imag())}; });
2988 });
2989 }
2990
2994 template <std::floating_point T>
2995 [[nodiscard]] tensor<T> clip(const tensor<T>& a,
2996 T low,
2997 T high)
2998 {
2999 if (std::isnan(low) || std::isnan(high)) {
3000 return detail::parallel_map(a, [low, high](T x) { return detail::ieee_minimum(detail::ieee_maximum(x, low), high); });
3001 }
3002 return detail::parallel_map(a, [low, high](T x) { return detail::clipped(x, low, high); });
3003 }
3004
3007 template <std::integral T>
3008 [[nodiscard]] tensor<T> clip(const tensor<T>& a,
3009 T low,
3010 T high)
3011 {
3012 return detail::parallel_map(a, [low, high](T x) { return std::min(std::max(x, low), high); });
3013 }
3014
3017 template <std::integral T>
3018 [[nodiscard]] tensor<T> maximum(const tensor<T>& a,
3019 const tensor<T>& b)
3020 {
3021 return broadcast_apply(a, b, [](T x, T y) { return std::max(x, y); });
3022 }
3023
3026 template <std::integral T>
3027 [[nodiscard]] tensor<T> minimum(const tensor<T>& a,
3028 const tensor<T>& b)
3029 {
3030 return broadcast_apply(a, b, [](T x, T y) { return std::min(x, y); });
3031 }
3032
3034 template <typename T>
3035 [[nodiscard]] tensor<T> operator-(const tensor<T>& a,
3036 const tensor<T>& b)
3037 {
3038 return broadcast_apply(a, b, typename detail::arithmetic<T>::minus {});
3039 }
3040
3042 template <typename T>
3043 [[nodiscard]] tensor<T> operator*(const tensor<T>& a,
3044 const tensor<T>& b)
3045 {
3046 return broadcast_apply(a, b, typename detail::arithmetic<T>::multiplies {});
3047 }
3048
3050 template <typename T>
3051 [[nodiscard]] tensor<T> operator/(const tensor<T>& a,
3052 const tensor<T>& b)
3053 {
3054 return broadcast_apply(a, b, std::divides<T> {});
3055 }
3056
3057 namespace detail {
3063 template <std::floating_point T>
3064 [[nodiscard]] std::pair<T, T> floor_divmod(T a,
3065 T b) noexcept
3066 {
3067 T mod {std::fmod(a, b)};
3068 if (b == T {0}) {
3069 return {a / b, mod};
3070 }
3071 T quotient {(a - mod) / b};
3072 if (mod != T {0}) {
3073 if ((b < T {0}) != (mod < T {0})) {
3074 mod += b;
3075 quotient -= T {1};
3076 }
3077 }
3078 else {
3079 mod = std::copysign(T {0}, b);
3080 }
3081 if (quotient == T {0}) {
3082 return {std::copysign(T {0}, a / b), mod};
3083 }
3084 T floored {std::floor(quotient)};
3085 if (quotient - floored > T {0.5}) {
3086 floored += T {1};
3087 }
3088 return {floored, mod};
3089 }
3090
3096 template <std::signed_integral T>
3097 [[nodiscard]] std::pair<T, T> floor_divmod(T a,
3098 T b) noexcept
3099 {
3100 if (b == T {0}) {
3101 return {T {0}, T {0}};
3102 }
3103 if (b == T {-1}) {
3104 return {typename arithmetic<T>::negate {}(a), T {0}};
3105 }
3106 T quotient {static_cast<T>(a / b)};
3107 T mod {static_cast<T>(a % b)};
3108 if (mod != T {0} && ((mod < T {0}) != (b < T {0}))) {
3109 quotient = static_cast<T>(quotient - T {1});
3110 mod = static_cast<T>(mod + b);
3111 }
3112 return {quotient, mod};
3113 }
3114 } // namespace detail
3115
3118 template <typename T>
3119 [[nodiscard]] tensor<T> floor_divide(const tensor<T>& a,
3120 const tensor<T>& b)
3121 {
3122 return broadcast_apply(a, b, [](T x, T y) { return detail::floor_divmod(x, y).first; });
3123 }
3124
3127 template <typename T>
3128 [[nodiscard]] tensor<T> remainder(const tensor<T>& a,
3129 const tensor<T>& b)
3130 {
3131 return broadcast_apply(a, b, [](T x, T y) { return detail::floor_divmod(x, y).second; });
3132 }
3133
3135 template <typename T>
3136 [[nodiscard]] tensor<T> operator-(const tensor<T>& a)
3137 {
3138 return detail::parallel_map(a, typename detail::arithmetic<T>::negate {});
3139 }
3140
3142 template <typename T,
3143 typename F>
3144 [[nodiscard]] auto map(const tensor<T>& t,
3146 {
3147 using R = std::decay_t<decltype(f(std::declval<const T&>()))>;
3148 buffer<R> out(t.size());
3149 std::transform(t.data().begin(), t.data().end(), out.begin(), f);
3150 return tensor<R> {t.shape(), std::move(out)};
3151 }
3152
3159 template <typename T>
3160 [[nodiscard]] tensor<T> pow(const tensor<T>& a,
3161 const tensor<T>& b)
3162 {
3163 if constexpr (!std::is_integral_v<T>) {
3164 if (b.size() == 1 && b.shape().size() <= a.shape().size()) {
3165 const T y {b.data()[0]};
3166 if (y == T(2)) {
3167 return detail::parallel_map(a, [](const T& x) { return detail::square(x); });
3168 }
3169 if (y == T(0.5)) {
3170 return detail::parallel_map(a, [](const T& x) { return detail::root(x); });
3171 }
3172 if (y == T(-1)) {
3173 return detail::parallel_map(a, [](const T& x) { return static_cast<T>(T(1) / x); });
3174 }
3175 }
3176 }
3177 return broadcast_apply(a, b, [](const T& x, const T& y) { return static_cast<T>(std::pow(x, y)); });
3178 }
3179
3180 namespace detail {
3183 template <std::signed_integral T>
3184 [[nodiscard]] T integer_power(T base,
3185 T exponent) noexcept
3186 {
3187 const typename arithmetic<T>::multiplies times {};
3188 T result {1};
3189 while (exponent != 0) {
3190 if ((exponent & 1) != 0) {
3191 result = times(result, base);
3192 }
3193 base = times(base, base);
3194 exponent >>= 1;
3195 }
3196 return result;
3197 }
3198 } // namespace detail
3199
3201 class negative_power_error : public detail::tagged<std::domain_error>
3202 {
3203 public:
3204
3206 explicit negative_power_error(const std::string& message)
3207 : tagged(message)
3208 {}
3209 };
3210
3216 template <std::signed_integral T>
3217 [[nodiscard]] tensor<T> pow(const tensor<T>& a,
3218 const tensor<T>& b)
3219 {
3220 if (std::ranges::any_of(b.data(), [](T exponent) { return exponent < 0; })) {
3221 throw negative_power_error("Integers to negative integer powers are not allowed.");
3222 }
3223 return broadcast_apply(a, b, [](T x, T y) { return detail::integer_power(x, y); });
3224 }
3225
3228 template <typename T>
3229 [[nodiscard]] T sum(const tensor<T>& t)
3230 {
3231 return detail::parallel_pairwise_sum(t.data().data(), t.size());
3232 }
3233
3236 template <typename T>
3237 [[nodiscard]] tensor<T> sum(const tensor<T>& t,
3238 std::size_t axis)
3239 {
3240 if (auto laid {detail::as_laid_reduced(t, axis, [](const tensor<T>& elements, std::size_t along) { return sum(elements, along); })}) {
3241 return std::move(*laid);
3242 }
3243 const detail::axis_walk walk {detail::walk_along(t.shape(), axis)};
3244 buffer<T> out(walk.outer * walk.inner);
3245 detail::pairwise_sums(t.data().data(), walk.outer, walk.extent, walk.inner, out.data());
3246 return tensor<T> {detail::without_axis(t.shape(), axis), std::move(out)};
3247 }
3248
3252 template <std::signed_integral T>
3253 [[nodiscard]] T sum(const tensor<T>& t)
3254 {
3255 if (const auto laid {t.laid_out()}) {
3256 return sum(laid->first);
3257 }
3258 using unsigned_type = typename detail::arithmetic<T>::unsigned_type;
3259 const T* const data {t.data().data()};
3260 const std::size_t count {t.size()};
3261 const std::size_t chunks {(count + detail::elementwise_chunk - 1) / detail::elementwise_chunk};
3262 std::vector<unsigned_type> partial(chunks, 0);
3263 detail::parallel_for(chunks, detail::elementwise_chunk, [&](std::size_t c) {
3264 const std::size_t last {std::min((c + 1) * detail::elementwise_chunk, count)};
3265 unsigned_type total {0};
3266 for (std::size_t i {c * detail::elementwise_chunk}; i < last; ++i) {
3267 total += static_cast<unsigned_type>(data[i]);
3268 }
3269 partial[c] = total;
3270 });
3271 unsigned_type total {0};
3272 for (const unsigned_type part : partial) {
3273 total += part;
3274 }
3275 return static_cast<T>(total);
3276 }
3277
3280 template <std::signed_integral T>
3281 [[nodiscard]] tensor<T> sum(const tensor<T>& t,
3282 std::size_t axis)
3283 {
3284 if (auto laid {detail::as_laid_reduced(t, axis, [](const tensor<T>& elements, std::size_t along) { return sum(elements, along); })}) {
3285 return std::move(*laid);
3286 }
3287 return detail::reduce_along(t, axis, [](const T* first, std::size_t count, std::size_t stride) {
3288 const typename detail::arithmetic<T>::plus plus {};
3289 T total {0};
3290 for (std::size_t i {0}; i < count; ++i) {
3291 total = plus(total, first[i * stride]);
3292 }
3293 return total;
3294 });
3295 }
3296
3300 template <typename T>
3301 [[nodiscard]] T mean(const tensor<T>& t)
3302 {
3303 return detail::divided_by_count(sum(t), t.size());
3304 }
3305
3308 template <typename T>
3309 [[nodiscard]] tensor<T> mean(const tensor<T>& t,
3310 std::size_t axis)
3311 {
3312 if (auto laid {detail::as_laid_reduced(t, axis, [](const tensor<T>& elements, std::size_t along) { return mean(elements, along); })}) {
3313 return std::move(*laid);
3314 }
3315 const detail::axis_walk walk {detail::walk_along(t.shape(), axis)};
3316 buffer<T> out(walk.outer * walk.inner);
3317 detail::pairwise_sums(t.data().data(), walk.outer, walk.extent, walk.inner, out.data());
3318 for (T& x : out) {
3319 x = detail::divided_by_count(x, walk.extent);
3320 }
3321 return tensor<T> {detail::without_axis(t.shape(), axis), std::move(out)};
3322 }
3323
3326 template <std::totally_ordered T>
3327 [[nodiscard]] T max(const tensor<T>& t)
3328 {
3329 if (const auto laid {t.laid_out()}) {
3330 return max(laid->first);
3331 }
3332 return detail::extreme(t.data().data(), t.size(), 1, std::greater<T> {});
3333 }
3334
3337 template <std::totally_ordered T>
3338 [[nodiscard]] T min(const tensor<T>& t)
3339 {
3340 if (const auto laid {t.laid_out()}) {
3341 return min(laid->first);
3342 }
3343 return detail::extreme(t.data().data(), t.size(), 1, std::less<T> {});
3344 }
3345
3350 template <typename T>
3351 [[nodiscard]] T prod(const tensor<T>& t)
3352 {
3353 const typename detail::arithmetic<T>::multiplies times {};
3354 T product {1};
3355 for (const T& x : t.data()) {
3356 product = times(product, x);
3357 }
3358 return product;
3359 }
3360
3363 template <typename T>
3364 [[nodiscard]] tensor<T> prod(const tensor<T>& t,
3365 std::size_t axis)
3366 {
3367 if (auto laid {detail::as_laid_reduced(t, axis, [](const tensor<T>& elements, std::size_t along) { return prod(elements, along); })}) {
3368 return std::move(*laid);
3369 }
3370 return detail::fold_along(t, axis, T {1}, typename detail::arithmetic<T>::multiplies {});
3371 }
3372
3373 namespace detail {
3374
3376 template <typename T,
3377 typename Step>
3378 [[nodiscard]] tensor<T> scan_along(const tensor<T>& t,
3379 std::size_t axis,
3380 const Step& step)
3381 {
3382 const axis_walk walk {walk_along(t.shape(), axis)};
3383 buffer<T> out(t.size());
3384 const T* const in {t.data().data()};
3385 if (walk.inner == 1) {
3386 parallel_for(walk.outer, walk.extent, [&](std::size_t o) { // each line its own, contiguous
3387 const std::size_t first {o * walk.extent};
3388 for (std::size_t k {0}; k < walk.extent; ++k) {
3389 out[first + k] = k == 0 ? in[first] : step(out[first + k - 1], in[first + k]);
3390 }
3391 });
3392 return tensor<T> {t.shape(), std::move(out)};
3393 }
3394 // the lines are columns: a band of them per task, row after row (each line's steps in the same order)
3395 const std::size_t bands {(walk.inner + band_columns - 1) / band_columns};
3396 parallel_for(walk.outer * bands, walk.extent * band_columns, [&](std::size_t at) {
3397 const std::size_t o {at / bands};
3398 const std::size_t first {(at % bands) * band_columns};
3399 const std::size_t width {std::min(band_columns, walk.inner - first)};
3400 const std::size_t base {(o * walk.extent * walk.inner) + first};
3401 for (std::size_t k {0}; k < walk.extent; ++k) {
3402 const std::size_t row {base + (k * walk.inner)};
3403 for (std::size_t w {0}; w < width; ++w) {
3404 out[row + w] = k == 0 ? in[row + w] : step(out[row - walk.inner + w], in[row + w]);
3405 }
3406 }
3407 });
3408 return tensor<T> {t.shape(), std::move(out)};
3409 }
3410
3417 template <detail::split_complex C>
3418 [[nodiscard]] tensor<C> numpy_cumprod(const tensor<C>& t,
3419 std::size_t axis,
3420 const numpy_loops& loops = numpy_wheel_loops())
3421 {
3422 const bool vector {walk_along(t.shape(), axis).extent == 2};
3423 const auto multiply {[vector, &loops](const C& running, const C& x) {
3424 return vector ? numpy_multiply(running, x, loops.products) : numpy_reduction_multiply(running, x, loops);
3425 }};
3426 if (auto laid {as_laid_along(t, axis, [&multiply](const tensor<C>& elements, std::size_t along) {
3427 return scan_along(elements, along, multiply);
3428 })}) {
3429 return std::move(*laid);
3430 }
3431 return scan_along(t, axis, multiply);
3432 }
3433
3435 template <std::floating_point T>
3436 [[nodiscard]] bool sorts_before(T a,
3437 T b) noexcept
3438 {
3439 return std::isnan(b) ? !std::isnan(a) : a < b;
3440 }
3441
3443 template <std::integral T>
3444 [[nodiscard]] bool sorts_before(T a,
3445 T b) noexcept
3446 {
3447 return a < b;
3448 }
3449
3455 template <split_complex C>
3456 [[nodiscard]] bool sorts_before(const C& a,
3457 const C& b) noexcept
3458 {
3459 const bool a_imaginary_nan {std::isnan(a.imag())};
3460 const bool b_imaginary_nan {std::isnan(b.imag())};
3461 if (a.real() < b.real()) {
3462 return !a_imaginary_nan || b_imaginary_nan;
3463 }
3464 if (a.real() > b.real()) {
3465 return b_imaginary_nan && !a_imaginary_nan;
3466 }
3467 if (a.real() == b.real() || (std::isnan(a.real()) && std::isnan(b.real()))) {
3468 return a.imag() < b.imag() || (b_imaginary_nan && !a_imaginary_nan);
3469 }
3470 return std::isnan(b.real());
3471 }
3472 } // namespace detail
3473
3476 template <typename T>
3477 [[nodiscard]] tensor<T> cumsum(const tensor<T>& t,
3478 std::size_t axis)
3479 {
3480 if (auto laid {detail::as_laid_along(t, axis, [](const tensor<T>& elements, std::size_t along) { return cumsum(elements, along); })}) {
3481 return std::move(*laid);
3482 }
3483 return detail::scan_along(t, axis, typename detail::arithmetic<T>::plus {});
3484 }
3485
3488 template <typename T>
3489 [[nodiscard]] tensor<T> cumprod(const tensor<T>& t,
3490 std::size_t axis)
3491 {
3492 if (auto laid {detail::as_laid_along(t, axis, [](const tensor<T>& elements, std::size_t along) { return cumprod(elements, along); })}) {
3493 return std::move(*laid);
3494 }
3495 return detail::scan_along(t, axis, typename detail::arithmetic<T>::multiplies {});
3496 }
3497
3498 namespace detail {
3500 inline constexpr std::size_t radix_sort_least {256};
3501
3503 template <typename T>
3504 using order_key_t = std::conditional_t<sizeof(T) == 8, std::uint64_t, std::uint32_t>;
3505
3507 template <typename U>
3508 inline constexpr U sign_bit {U {1} << ((sizeof(U) * 8) - 1)};
3509
3512 template <std::floating_point T>
3513 [[nodiscard]] order_key_t<T> order_key(T x) noexcept
3514 {
3515 using U = order_key_t<T>;
3516 const auto bits {std::bit_cast<U>(x)};
3517 return (bits & sign_bit<U>) != 0 ? static_cast<U>(~bits) : static_cast<U>(bits | sign_bit<U>);
3518 }
3519
3521 template <std::integral T>
3522 [[nodiscard]] order_key_t<T> order_key(T x) noexcept
3523 {
3524 using U = order_key_t<T>;
3525 return static_cast<U>(std::bit_cast<U>(x) ^ sign_bit<U>);
3526 }
3527
3529 template <std::floating_point T>
3530 [[nodiscard]] T from_order_key(order_key_t<T> key) noexcept
3531 {
3532 using U = order_key_t<T>;
3533 return std::bit_cast<T>((key & sign_bit<U>) != 0 ? static_cast<U>(key & ~sign_bit<U>) : static_cast<U>(~key));
3534 }
3535
3537 template <std::integral T>
3538 [[nodiscard]] T from_order_key(order_key_t<T> key) noexcept
3539 {
3540 return std::bit_cast<T>(static_cast<order_key_t<T>>(key ^ sign_bit<order_key_t<T>>));
3541 }
3542
3544 template <typename U>
3545 using byte_counts = std::array<std::array<std::size_t, 256>, sizeof(U)>;
3546
3549 template <typename U>
3551 U key) noexcept
3552 {
3553 for (std::size_t b {0}; b < sizeof(U); ++b) {
3554 ++counts[b][(key >> (8 * b)) & 0xffU];
3555 }
3556 }
3557
3559 [[nodiscard]] inline std::size_t part_begin(std::size_t count,
3560 std::size_t parts,
3561 std::size_t p) noexcept
3562 {
3563 return (count / parts * p) + std::min(p, count % parts);
3564 }
3565
3575 template <typename U>
3576 [[nodiscard]] bool radix_passes(U* keys,
3577 U* key_spare,
3578 std::int64_t* indices,
3579 std::int64_t* index_spare,
3580 std::size_t count,
3581 std::vector<byte_counts<U>>& counts)
3582 {
3583 const std::size_t parts {counts.size()};
3584 const std::size_t work {count / parts * 16};
3585 U* key_from {keys};
3586 U* key_to {key_spare};
3587 std::int64_t* index_from {indices};
3588 std::int64_t* index_to {index_spare};
3589 bool moved {false};
3590 for (std::size_t b {0}; b < sizeof(U); ++b) {
3591 const std::size_t shift {8 * b};
3592 std::array<std::size_t, 256> total {};
3593 for (const byte_counts<U>& part : counts) {
3594 std::transform(total.begin(), total.end(), part[b].begin(), total.begin(), std::plus {});
3595 }
3596 if (std::ranges::find(total, count) != total.end()) { // one byte value alone: in order already
3597 continue;
3598 }
3599 if (moved && parts > 1) {
3600 parallel_for(parts, work, [&](std::size_t p) {
3601 std::array<std::size_t, 256> part {}; // a local array, which no store to a key can alias
3602 const std::size_t end {part_begin(count, parts, p + 1)};
3603 for (std::size_t i {part_begin(count, parts, p)}; i < end; ++i) {
3604 ++part[(key_from[i] >> shift) & 0xffU];
3605 }
3606 counts[p][b] = part;
3607 });
3608 }
3609 std::size_t place {0};
3610 for (std::size_t v {0}; v < 256; ++v) {
3611 for (byte_counts<U>& part : counts) {
3612 const std::size_t held {part[b][v]};
3613 part[b][v] = place;
3614 place += held;
3615 }
3616 }
3617 parallel_for(parts, work, [&](std::size_t p) {
3618 // a local copy: a store to a key (std::uint64_t, std::size_t's type on LP64 Linux) may alias
3619 // the counts where they lie, and each would be read again after it (10⁶ float64 on x86-64
3620 // GCC: 22 → 27 ms)
3621 std::array<std::size_t, 256> at {counts[p][b]};
3622 const std::size_t end {part_begin(count, parts, p + 1)};
3623 if (index_from == nullptr) {
3624 for (std::size_t i {part_begin(count, parts, p)}; i < end; ++i) {
3625 key_to[at[(key_from[i] >> shift) & 0xffU]++] = key_from[i];
3626 }
3627 return;
3628 }
3629 for (std::size_t i {part_begin(count, parts, p)}; i < end; ++i) {
3630 const std::size_t k {at[(key_from[i] >> shift) & 0xffU]++};
3631 key_to[k] = key_from[i];
3632 index_to[k] = index_from[i];
3633 }
3634 });
3635 std::swap(key_from, key_to);
3636 std::swap(index_from, index_to);
3637 moved = true;
3638 }
3639 return key_from != keys;
3640 }
3641
3647 template <typename T>
3648 void radix_sort(T* first,
3649 T* last,
3650 std::size_t parts = 1)
3651 {
3652 using U = order_key_t<T>;
3653 const auto count {static_cast<std::size_t>(last - first)};
3654 const std::size_t shares {std::clamp<std::size_t>(parts, 1, std::max<std::size_t>(count, 1))};
3655 buffer<U> keys(count);
3656 buffer<U> spare(count);
3657 std::vector<byte_counts<U>> counts(shares);
3658 parallel_for(shares, count / shares * 16, [&](std::size_t p) {
3659 byte_counts<U> part {}; // a local array, which no store to a key can alias
3660 const std::size_t end {part_begin(count, shares, p + 1)};
3661 for (std::size_t i {part_begin(count, shares, p)}; i < end; ++i) {
3662 keys[i] = order_key(first[i]);
3663 count_bytes(part, keys[i]);
3664 }
3665 counts[p] = part;
3666 });
3667 const U* const sorted {radix_passes(keys.data(), spare.data(), nullptr, nullptr, count, counts) ? spare.data() : keys.data()};
3668 parallel_for(shares, count / shares * 16, [&](std::size_t p) {
3669 for (std::size_t i {part_begin(count, shares, p)}; i < part_begin(count, shares, p + 1); ++i) {
3670 first[i] = from_order_key<T>(sorted[i]);
3671 }
3672 });
3673 }
3674
3682 template <std::floating_point T>
3683 void sort_run(T* first,
3684 T* last,
3685 std::size_t parts = 1)
3686 {
3687 T* const numbers_end {std::stable_partition(first, last, [](T x) { return !std::isnan(x); })};
3688 if (static_cast<std::size_t>(numbers_end - first) < radix_sort_least) {
3689 std::stable_sort(first, numbers_end, [](T a, T b) { return a < b; });
3690 return;
3691 }
3692 std::vector<bool> negative_zeros; // each zero's sign, in their order
3693 for (const T* x {first}; x != numbers_end; ++x) {
3694 if (*x == T {0}) {
3695 negative_zeros.push_back(std::signbit(*x));
3696 }
3697 }
3698 radix_sort(first, numbers_end, parts);
3699 T* zero {std::lower_bound(first, numbers_end, T {0}, [](T a, T b) { return a < b; })};
3700 for (const bool negative : negative_zeros) {
3701 *zero++ = negative ? -T {0} : T {0};
3702 }
3703 }
3704
3706 template <typename T>
3707 concept radix_orderable = std::same_as<T, double> || std::same_as<T, float> || std::same_as<T, std::int64_t>;
3708
3711 template <std::floating_point T>
3712 [[nodiscard]] order_key_t<T> argsort_key(T x) noexcept
3713 {
3714 return order_key(x == T {0} ? T {0} : x);
3715 }
3716
3718 [[nodiscard]] inline order_key_t<std::int64_t> argsort_key(std::int64_t x) noexcept
3719 {
3720 return order_key(x);
3721 }
3722
3724 template <std::floating_point T>
3725 [[nodiscard]] std::int64_t* numbers_end(std::int64_t* first,
3726 std::int64_t* last,
3727 const T* values,
3728 std::size_t stride)
3729 {
3730 return std::stable_partition(first, last, [&](std::int64_t k) { return !std::isnan(values[static_cast<std::size_t>(k) * stride]); });
3731 }
3732
3734 [[nodiscard]] inline std::int64_t* numbers_end(std::int64_t* /*first*/,
3735 std::int64_t* last,
3736 const std::int64_t* /*values*/,
3737 std::size_t /*stride*/)
3738 {
3739 return last;
3740 }
3741
3749 template <radix_orderable T,
3750 typename Less>
3751 void order_run(std::int64_t* first,
3752 std::int64_t* last,
3753 const T* values,
3754 std::size_t stride,
3755 const Less& less,
3756 std::size_t parts = 1)
3757 {
3758 if (static_cast<std::size_t>(last - first) < radix_sort_least) {
3759 std::stable_sort(first, last, less);
3760 return;
3761 }
3762 using U = order_key_t<T>;
3763 std::int64_t* const numbers_last {numbers_end(first, last, values, stride)};
3764 const auto count {static_cast<std::size_t>(numbers_last - first)};
3765 const std::size_t shares {std::clamp<std::size_t>(parts, 1, std::max<std::size_t>(count, 1))};
3766 buffer<U> keys(count);
3767 buffer<U> key_spare(count);
3768 buffer<std::int64_t> index_spare(count);
3769 std::vector<byte_counts<U>> counts(shares);
3770 parallel_for(shares, count / shares * 16, [&](std::size_t p) {
3771 byte_counts<U> part {}; // a local array, which no store to a key can alias
3772 const std::size_t end {part_begin(count, shares, p + 1)};
3773 for (std::size_t i {part_begin(count, shares, p)}; i < end; ++i) {
3774 keys[i] = argsort_key(values[static_cast<std::size_t>(first[i]) * stride]);
3775 count_bytes(part, keys[i]);
3776 }
3777 counts[p] = part;
3778 });
3779 if (radix_passes(keys.data(), key_spare.data(), first, index_spare.data(), count, counts)) {
3780 std::copy_n(index_spare.data(), count, first);
3781 }
3782 }
3783
3786 template <typename T,
3787 typename Less>
3788 void order_run(std::int64_t* first,
3789 std::int64_t* last,
3790 const T* /*values*/,
3791 std::size_t /*stride*/,
3792 const Less& less,
3793 std::size_t /*parts*/ = 1)
3794 {
3795 std::stable_sort(first, last, less);
3796 }
3797
3799 template <std::integral T>
3800 void sort_run(T* first,
3801 T* last)
3802 {
3803 std::stable_sort(first, last);
3804 }
3805
3808 inline void sort_run(std::int64_t* first,
3809 std::int64_t* last,
3810 std::size_t parts = 1)
3811 {
3812 if (static_cast<std::size_t>(last - first) < radix_sort_least) {
3813 std::stable_sort(first, last);
3814 return;
3815 }
3816 radix_sort(first, last, parts);
3817 }
3818
3820 template <split_complex C>
3821 void sort_run(C* first,
3822 C* last)
3823 {
3824 std::stable_sort(first, last, [](const C& a, const C& b) { return sorts_before(a, b); });
3825 }
3826
3828 [[nodiscard]] inline std::size_t merged_pieces(std::size_t threads) noexcept
3829 {
3830 std::size_t pieces {4};
3831 while (pieces < threads) {
3832 pieces *= 4;
3833 }
3834 return pieces;
3835 }
3836
3844 template <typename T,
3845 typename SortRun,
3846 typename Less>
3847 void parallel_sort_run(T* data,
3848 std::size_t count,
3849 std::size_t pieces,
3850 const SortRun& sort_piece,
3851 const Less& less)
3852 {
3853 std::vector<std::size_t> bounds(pieces + 1);
3854 for (std::size_t k {0}; k <= pieces; ++k) {
3855 bounds[k] = part_begin(count, pieces, k);
3856 }
3857 parallel_for(pieces, count / pieces * 16, [&](std::size_t k) { sort_piece(data + bounds[k], data + bounds[k + 1]); });
3858 buffer<T> scratch(count);
3859 T* from {data};
3860 T* to {scratch.data()};
3861 for (std::size_t width {1}; width < pieces; width *= 2) {
3862 const std::size_t pairs {(pieces + (2 * width) - 1) / (2 * width)};
3863 parallel_for(pairs, count / pairs, [&](std::size_t pair) {
3864 const std::size_t first {bounds[pair * 2 * width]};
3865 const std::size_t middle {bounds[std::min((pair * 2 * width) + width, pieces)]};
3866 const std::size_t last {bounds[std::min((pair + 1) * 2 * width, pieces)]};
3867 std::merge(from + first, from + middle, from + middle, from + last, to + first, less);
3868 });
3869 std::swap(from, to);
3870 }
3871 }
3872
3875 template <radix_orderable T>
3876 void sort_shared(T* run,
3877 std::size_t count,
3878 std::size_t threads)
3879 {
3880 sort_run(run, run + count, threads);
3881 }
3882
3885 template <typename T>
3886 void sort_shared(T* run,
3887 std::size_t count,
3888 std::size_t threads)
3889 {
3890 parallel_sort_run(run, count, merged_pieces(threads), [](T* first, T* last) { sort_run(first, last); },
3891 [](const T& a, const T& b) { return sorts_before(a, b); });
3892 }
3893
3895 template <radix_orderable T,
3896 typename Less>
3897 void order_shared(std::int64_t* order,
3898 std::size_t count,
3899 const T* values,
3900 std::size_t stride,
3901 const Less& less,
3902 std::size_t threads)
3903 {
3904 order_run(order, order + count, values, stride, less, threads);
3905 }
3906
3908 template <typename T,
3909 typename Less>
3910 void order_shared(std::int64_t* order,
3911 std::size_t count,
3912 const T* values,
3913 std::size_t stride,
3914 const Less& less,
3915 std::size_t threads)
3916 {
3917 parallel_sort_run(order, count, merged_pieces(threads),
3918 [&](std::int64_t* first, std::int64_t* last) { order_run(first, last, values, stride, less); }, less);
3919 }
3920 } // namespace detail
3921
3930 template <typename T>
3931 [[nodiscard]] tensor<T> sort(const tensor<T>& t,
3932 std::size_t axis)
3933 {
3934 if (auto laid {detail::as_laid_along(t, axis, [](const tensor<T>& elements, std::size_t along) { return sort(elements, along); })}) {
3935 return std::move(*laid);
3936 }
3937 const detail::axis_walk walk {detail::walk_along(t.shape(), axis)};
3938 const std::size_t lines {walk.outer * walk.inner};
3939 buffer<T> out(t.size());
3940 const T* const in {t.data().data()};
3941 const auto gather {[&](std::size_t line, T* into) {
3942 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
3943 for (std::size_t k {0}; k < walk.extent; ++k) {
3944 into[k] = in[first + (k * walk.inner)];
3945 }
3946 }};
3947 const auto scatter {[&](std::size_t line, const T* from) {
3948 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
3949 for (std::size_t k {0}; k < walk.extent; ++k) {
3950 out[first + (k * walk.inner)] = from[k];
3951 }
3952 }};
3953 const std::size_t threads {detail::parallel_threads(walk.extent, 16)};
3954 if (threads >= 2 && detail::parallel_threads(lines, walk.extent * 16) * 2 <= threads) {
3955 buffer<T> line_buffer(walk.inner == 1 ? 0 : walk.extent);
3956 for (std::size_t line {0}; line < lines; ++line) {
3957 T* const run {walk.inner == 1 ? out.data() + (line * walk.extent) : line_buffer.data()};
3958 gather(line, run);
3959 detail::sort_shared(run, walk.extent, threads);
3960 if (walk.inner != 1) {
3961 scatter(line, run);
3962 }
3963 }
3964 return tensor<T> {t.shape(), std::move(out)};
3965 }
3966 detail::parallel_for(lines, walk.extent * 16, [&](std::size_t line) {
3967 if (walk.inner == 1) {
3968 T* const run {out.data() + (line * walk.extent)};
3969 gather(line, run);
3970 detail::sort_run(run, run + walk.extent);
3971 return;
3972 }
3973 buffer<T> run(walk.extent);
3974 gather(line, run.data());
3975 detail::sort_run(run.data(), run.data() + walk.extent);
3976 scatter(line, run.data());
3977 });
3978 return tensor<T> {t.shape(), std::move(out)};
3979 }
3980
3986 template <typename T>
3988 std::size_t axis)
3989 {
3990 if (auto laid {detail::as_laid_along(t, axis, [](const tensor<T>& elements, std::size_t along) { return argsort(elements, along); })}) {
3991 return std::move(*laid);
3992 }
3993 const detail::axis_walk walk {detail::walk_along(t.shape(), axis)};
3994 const std::size_t lines {walk.outer * walk.inner};
3995 buffer<std::int64_t> out(t.size());
3996 const T* const in {t.data().data()};
3997 const std::size_t threads {detail::parallel_threads(walk.extent, 16)};
3998 const bool shared {threads >= 2 && detail::parallel_threads(lines, walk.extent * 16) * 2 <= threads};
3999 const auto order_line {[&](std::size_t line, bool among_threads) {
4000 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
4001 const auto value {[&](std::int64_t k) -> const T& {
4002 return in[first + (static_cast<std::size_t>(k) * walk.inner)];
4003 }};
4004 const auto less {[&](std::int64_t a, std::int64_t b) { return detail::sorts_before(value(a), value(b)); }};
4005 buffer<std::int64_t> order(walk.extent);
4006 std::iota(order.begin(), order.end(), std::int64_t {0});
4007 const T* const line_values {in + first};
4008 if (among_threads) {
4009 detail::order_shared(order.data(), walk.extent, line_values, walk.inner, less, threads);
4010 }
4011 else {
4012 detail::order_run(order.data(), order.data() + walk.extent, line_values, walk.inner, less);
4013 }
4014 for (std::size_t k {0}; k < walk.extent; ++k) {
4015 out[first + (k * walk.inner)] = order[k];
4016 }
4017 }};
4018 if (shared) {
4019 for (std::size_t line {0}; line < lines; ++line) {
4020 order_line(line, true);
4021 }
4022 }
4023 else {
4024 detail::parallel_for(lines, walk.extent * 16, [&](std::size_t line) { order_line(line, false); });
4025 }
4026 return tensor<std::int64_t> {t.shape(), std::move(out)};
4027 }
4028
4029 namespace detail {
4032 [[nodiscard]] inline std::vector<std::size_t> partition_positions(std::vector<std::size_t> kths,
4033 std::size_t extent)
4034 {
4035 std::sort(kths.begin(), kths.end());
4036 kths.erase(std::unique(kths.begin(), kths.end()), kths.end());
4037 if (!kths.empty() && kths.back() >= extent) {
4038 throw index_error("partition position " + std::to_string(kths.back()) + " is out of bounds for an extent of "
4039 + std::to_string(extent));
4040 }
4041 return kths;
4042 }
4043
4046 template <typename It,
4047 typename Less>
4048 void select_positions(It first,
4049 std::size_t count,
4050 const std::vector<std::size_t>& kths,
4051 const Less& less)
4052 {
4053 std::size_t from {0};
4054 for (const std::size_t k : kths) {
4055 std::nth_element(first + static_cast<std::ptrdiff_t>(from), first + static_cast<std::ptrdiff_t>(k),
4056 first + static_cast<std::ptrdiff_t>(count), less);
4057 from = k + 1;
4058 }
4059 }
4060 } // namespace detail
4061
4069 template <typename T>
4070 [[nodiscard]] tensor<T> partition(const tensor<T>& t,
4071 const std::vector<std::size_t>& kths,
4072 std::size_t axis)
4073 {
4074 if (auto laid {detail::as_laid_along(t, axis, [&kths](const tensor<T>& elements, std::size_t along) { return partition(elements, kths, along); })}) {
4075 return std::move(*laid);
4076 }
4077 const detail::axis_walk walk {detail::walk_along(t.shape(), axis)};
4078 const std::vector<std::size_t> positions {detail::partition_positions(kths, walk.extent)};
4079 buffer<T> out(t.size());
4080 const T* const in {t.data().data()};
4081 detail::parallel_for(walk.outer * walk.inner, walk.extent * 4, [&](std::size_t line) {
4082 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
4083 buffer<T> run(walk.extent);
4084 for (std::size_t k {0}; k < walk.extent; ++k) {
4085 run[k] = in[first + (k * walk.inner)];
4086 }
4087 detail::select_positions(run.begin(), walk.extent, positions,
4088 [](const T& a, const T& b) { return detail::sorts_before(a, b); });
4089 for (std::size_t k {0}; k < walk.extent; ++k) {
4090 out[first + (k * walk.inner)] = run[k];
4091 }
4092 });
4093 return tensor<T> {t.shape(), std::move(out)};
4094 }
4095
4101 template <typename T>
4103 const std::vector<std::size_t>& kths,
4104 std::size_t axis)
4105 {
4106 if (auto laid {detail::as_laid_along(t, axis, [&kths](const tensor<T>& elements, std::size_t along) { return argpartition(elements, kths, along); })}) {
4107 return std::move(*laid);
4108 }
4109 const detail::axis_walk walk {detail::walk_along(t.shape(), axis)};
4110 const std::vector<std::size_t> positions {detail::partition_positions(kths, walk.extent)};
4111 buffer<std::int64_t> out(t.size());
4112 const T* const in {t.data().data()};
4113 detail::parallel_for(walk.outer * walk.inner, walk.extent * 4, [&](std::size_t line) {
4114 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
4115 buffer<std::int64_t> order(walk.extent);
4116 std::iota(order.begin(), order.end(), std::int64_t {0});
4117 const auto value {[&](std::int64_t k) -> const T& {
4118 return in[first + (static_cast<std::size_t>(k) * walk.inner)];
4119 }};
4120 detail::select_positions(order.begin(), walk.extent, positions, [&](std::int64_t a, std::int64_t b) {
4121 return detail::sorts_before(value(a), value(b));
4122 });
4123 for (std::size_t k {0}; k < walk.extent; ++k) {
4124 out[first + (k * walk.inner)] = order[k];
4125 }
4126 });
4127 return tensor<std::int64_t> {t.shape(), std::move(out)};
4128 }
4129
4130 namespace detail {
4132 template <typename T>
4133 [[nodiscard]] T conjugated_chunks(const T* a,
4134 const T* b,
4135 std::size_t count)
4136 {
4137 return chunked_dot(a, b, count);
4138 }
4139
4141 template <split_complex C>
4142 [[nodiscard]] C conjugated_chunks(const C* a,
4143 const C* b,
4144 std::size_t count)
4145 {
4146 return chunked_conjugate_dot(a, b, count);
4147 }
4148 } // namespace detail
4149
4156 template <typename T>
4157 [[nodiscard]] T vdot(const tensor<T>& a,
4158 const tensor<T>& b)
4159 {
4160 if (a.size() != b.size()) {
4161 throw dimension_error("vdot of " + std::to_string(a.size()) + " and " + std::to_string(b.size()) + " elements");
4162 }
4163 return detail::conjugated_chunks(a.data().data(), b.data().data(), a.size());
4164 }
4165
4173 template <typename T>
4175 const tensor<T>& key,
4176 std::size_t axis)
4177 {
4178 if (order.shape() != key.shape()) {
4179 throw dimension_error("argsort_by: the order and the key of one shape");
4180 }
4181 const detail::axis_walk walk {detail::walk_along(key.shape(), axis)};
4182 const auto extent {static_cast<std::int64_t>(walk.extent)};
4183 const std::int64_t* const from {order.data().data()};
4184 if (std::any_of(from, from + order.size(), [extent](std::int64_t k) { return k < 0 || k >= extent; })) {
4185 throw dimension_error("argsort_by: an index out of the axis's range");
4186 }
4187 const std::size_t lines {walk.outer * walk.inner};
4188 buffer<std::int64_t> out(order.size());
4189 const T* const in {key.data().data()};
4190 const std::size_t threads {detail::parallel_threads(walk.extent, 16)};
4191 const bool shared {threads >= 2 && detail::parallel_threads(lines, walk.extent * 16) * 2 <= threads};
4192 const auto order_line {[&](std::size_t line, bool among_threads) {
4193 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
4194 const auto value {[&](std::int64_t k) -> const T& {
4195 return in[first + (static_cast<std::size_t>(k) * walk.inner)];
4196 }};
4197 const auto less {[&](std::int64_t a, std::int64_t b) { return detail::sorts_before(value(a), value(b)); }};
4198 buffer<std::int64_t> run(walk.extent);
4199 for (std::size_t k {0}; k < walk.extent; ++k) {
4200 run[k] = from[first + (k * walk.inner)];
4201 }
4202 if (among_threads) {
4203 detail::order_shared(run.data(), walk.extent, in + first, walk.inner, less, threads);
4204 }
4205 else {
4206 detail::order_run(run.data(), run.data() + walk.extent, in + first, walk.inner, less);
4207 }
4208 for (std::size_t k {0}; k < walk.extent; ++k) {
4209 out[first + (k * walk.inner)] = run[k];
4210 }
4211 }};
4212 if (shared) {
4213 for (std::size_t line {0}; line < lines; ++line) {
4214 order_line(line, true);
4215 }
4216 }
4217 else {
4218 detail::parallel_for(lines, walk.extent * 16, [&](std::size_t line) { order_line(line, false); });
4219 }
4220 return tensor<std::int64_t> {key.shape(), std::move(out)};
4221 }
4222
4223 namespace detail {
4225 template <std::floating_point T>
4226 [[nodiscard]] bool wins_any_order(T x) noexcept
4227 {
4228 return std::isnan(x);
4229 }
4230
4232 template <std::integral T>
4233 [[nodiscard]] bool wins_any_order(T /*x*/) noexcept
4234 {
4235 return false;
4236 }
4237
4240 template <typename T,
4241 typename Match>
4242 [[nodiscard]] bool holds(const T* first,
4243 std::size_t count,
4244 const Match& match) noexcept
4245 {
4246 unsigned hit {0};
4247 for (std::size_t i {0}; i < count; ++i) {
4248 hit |= static_cast<unsigned>(match(first[i]));
4249 }
4250 return hit != 0U;
4251 }
4252
4255 template <typename T,
4256 typename Match>
4257 [[nodiscard]] std::size_t first_held(const T* first,
4258 std::size_t count,
4259 const Match& match) noexcept
4260 {
4261 constexpr std::size_t block {64};
4262 std::size_t start {0};
4263 while (!holds(first + start, std::min(block, count - start), match)) {
4264 start += block;
4265 }
4266 std::size_t at {start};
4267 while (!match(first[at])) {
4268 ++at;
4269 }
4270 return at;
4271 }
4272
4277 template <std::floating_point T,
4278 typename Order>
4279 [[nodiscard]] std::int64_t extreme_index(const T* first,
4280 std::size_t count,
4281 const Order& larger)
4282 {
4283 const T value {contiguous_extreme(first, count, larger)};
4284 if (std::isnan(value)) {
4285 return static_cast<std::int64_t>(first_held(first, count, [](T x) { return std::isnan(x); }));
4286 }
4287 return static_cast<std::int64_t>(first_held(first, count, [value](T x) { return x == value; }));
4288 }
4289
4291 template <std::integral T,
4292 typename Order>
4293 [[nodiscard]] std::int64_t extreme_index(const T* first,
4294 std::size_t count,
4295 const Order& larger)
4296 {
4297 std::size_t best {0};
4298 for (std::size_t i {1}; i < count; ++i) {
4299 if (larger(first[i], first[best])) {
4300 best = i;
4301 }
4302 }
4303 return static_cast<std::int64_t>(best);
4304 }
4305
4307 template <typename T,
4308 typename Order>
4310 std::size_t axis,
4311 const Order& larger)
4312 {
4313 const axis_walk walk {walk_along(t.shape(), axis)};
4314 if (walk.extent == 0) {
4315 throw dimension_error("attempt to get argmax or argmin of an empty sequence");
4316 }
4317 buffer<std::int64_t> out(walk.outer * walk.inner);
4318 const T* const in {t.data().data()};
4319 if (walk.inner == 1) {
4320 parallel_for(walk.outer, walk.extent, [&](std::size_t line) {
4321 out[line] = extreme_index(in + (line * walk.extent), walk.extent, larger);
4322 });
4323 return tensor<std::int64_t> {without_axis(t.shape(), axis), std::move(out)};
4324 }
4325 // the lines are columns: a band of them per task, row after row, each column's first winner kept as its own
4326 // line's walk keeps it (a NaN's at once, else the first of equals)
4327 const std::size_t bands {(walk.inner + band_columns - 1) / band_columns};
4328 parallel_for(walk.outer * bands, walk.extent * band_columns, [&](std::size_t at) {
4329 const std::size_t o {at / bands};
4330 const std::size_t first {(at % bands) * band_columns};
4331 const std::size_t width {std::min(band_columns, walk.inner - first)};
4332 const T* const base {in + (o * walk.extent * walk.inner) + first};
4333 std::array<std::size_t, band_columns> best {};
4334 std::array<bool, band_columns> won {};
4335 for (std::size_t w {0}; w < width; ++w) {
4336 won[w] = wins_any_order(base[w]); // the first element itself a NaN
4337 }
4338 for (std::size_t k {1}; k < walk.extent; ++k) {
4339 const T* const row {base + (k * walk.inner)};
4340 for (std::size_t w {0}; w < width; ++w) {
4341 if (won[w]) {
4342 continue;
4343 }
4344 if (wins_any_order(row[w])) {
4345 best[w] = k;
4346 won[w] = true;
4347 }
4348 else if (larger(row[w], base[(best[w] * walk.inner) + w])) {
4349 best[w] = k;
4350 }
4351 }
4352 }
4353 for (std::size_t w {0}; w < width; ++w) {
4354 out[(o * walk.inner) + first + w] = static_cast<std::int64_t>(best[w]);
4355 }
4356 });
4357 return tensor<std::int64_t> {without_axis(t.shape(), axis), std::move(out)};
4358 }
4359
4362 template <typename T,
4363 typename Order>
4364 [[nodiscard]] std::int64_t parallel_extreme_index(const T* first,
4365 std::size_t count,
4366 const Order& larger)
4367 {
4368 const std::size_t chunks {(count + elementwise_chunk - 1) / elementwise_chunk};
4369 std::vector<std::int64_t> best(chunks);
4370 parallel_for(chunks, elementwise_chunk, [&](std::size_t c) {
4371 const std::size_t start {c * elementwise_chunk};
4372 best[c] = static_cast<std::int64_t>(start) + extreme_index(first + start, std::min(elementwise_chunk, count - start), larger);
4373 });
4374 std::int64_t winner {best[0]};
4375 for (const std::int64_t candidate : best) {
4376 const T& x {first[candidate]};
4377 if (wins_any_order(x)) {
4378 return candidate; // the earliest chunk's NaN: chunks are in order
4379 }
4380 if (larger(x, first[winner])) {
4381 winner = candidate;
4382 }
4383 }
4384 return winner;
4385 }
4386 } // namespace detail
4387
4390 template <typename T>
4391 [[nodiscard]] std::int64_t argmax(const tensor<T>& t)
4392 {
4393 if (t.size() == 0) {
4394 throw dimension_error("attempt to get argmax of an empty sequence");
4395 }
4396 return detail::parallel_extreme_index(t.data().data(), t.size(), std::greater<T> {});
4397 }
4398
4401 template <typename T>
4402 [[nodiscard]] std::int64_t argmin(const tensor<T>& t)
4403 {
4404 if (t.size() == 0) {
4405 throw dimension_error("attempt to get argmin of an empty sequence");
4406 }
4407 return detail::parallel_extreme_index(t.data().data(), t.size(), std::less<T> {});
4408 }
4409
4412 template <typename T>
4413 [[nodiscard]] tensor<std::int64_t> argmax(const tensor<T>& t,
4414 std::size_t axis)
4415 {
4416 if (auto laid {detail::as_laid_reduced(t, axis, [](const tensor<T>& elements, std::size_t along) { return argmax(elements, along); })}) {
4417 return std::move(*laid);
4418 }
4419 return detail::extreme_indices(t, axis, std::greater<T> {});
4420 }
4421
4424 template <typename T>
4425 [[nodiscard]] tensor<std::int64_t> argmin(const tensor<T>& t,
4426 std::size_t axis)
4427 {
4428 if (auto laid {detail::as_laid_reduced(t, axis, [](const tensor<T>& elements, std::size_t along) { return argmin(elements, along); })}) {
4429 return std::move(*laid);
4430 }
4431 return detail::extreme_indices(t, axis, std::less<T> {});
4432 }
4433
4441 template <typename T>
4442 [[nodiscard]] std::vector<tensor<std::int64_t>> nonzero(const tensor<T>& t)
4443 {
4444 const std::vector<std::size_t>& shape {t.shape()};
4445 const T* const data {t.data().data()};
4446 const std::size_t count {t.size()};
4447 const std::size_t rank {shape.size()};
4448 const std::size_t chunk {detail::elementwise_chunk};
4449 const std::size_t chunks {(count + chunk - 1) / chunk};
4450 std::vector<std::size_t> offsets(chunks + 1, 0);
4451 detail::parallel_for(chunks, chunk, [&](std::size_t c) {
4452 const std::size_t last {std::min((c + 1) * chunk, count)};
4453 offsets[c + 1] = static_cast<std::size_t>(std::count_if(data + (c * chunk), data + last, [](const T& x) { return x != T {}; }));
4454 });
4455 std::partial_sum(offsets.begin(), offsets.end(), offsets.begin());
4456 const std::size_t found {offsets.back()};
4457 std::vector<buffer<std::int64_t>> out(rank);
4458 for (buffer<std::int64_t>& axis_out : out) {
4459 axis_out = buffer<std::int64_t>(found);
4460 }
4461 detail::parallel_for(chunks, chunk * rank, [&](std::size_t c) {
4462 const std::size_t first {c * chunk};
4463 const std::size_t last {std::min(first + chunk, count)};
4464 const std::size_t span {last - first + 1}; // one past the part: the last write's
4465 buffer<std::int64_t> scratch(span * rank);
4466 std::vector<std::size_t> coordinate(rank);
4467 std::size_t rest {first};
4468 for (std::size_t a {rank}; a-- > 0;) {
4469 coordinate[a] = rest % shape[a];
4470 rest /= shape[a];
4471 }
4472 std::size_t at {0};
4473 for (std::size_t k {first}; k < last; ++k) {
4474 for (std::size_t a {0}; a < rank; ++a) {
4475 scratch[(a * span) + at] = static_cast<std::int64_t>(coordinate[a]);
4476 }
4477 at += static_cast<std::size_t>(data[k] != T {});
4478 for (std::size_t a {rank}; a-- > 0;) { // the next element's indices
4479 if (++coordinate[a] < shape[a]) {
4480 break;
4481 }
4482 coordinate[a] = 0;
4483 }
4484 }
4485 for (std::size_t a {0}; a < rank; ++a) {
4486 std::copy_n(scratch.data() + (a * span), at, out[a].data() + offsets[c]);
4487 }
4488 });
4489 std::vector<tensor<std::int64_t>> indices;
4490 indices.reserve(rank);
4491 for (buffer<std::int64_t>& axis_out : out) {
4492 indices.emplace_back(std::vector<std::size_t> {found}, std::move(axis_out));
4493 }
4494 return indices;
4495 }
4496
4500 template <typename T>
4501 [[nodiscard]] std::size_t count_nonzero(const tensor<T>& t)
4502 {
4503 if (const auto laid {t.laid_out()}) {
4504 return count_nonzero(laid->first);
4505 }
4506 const auto& values {t.data()}; // indexed: a buffer of bools has no data()
4507 const std::size_t count {t.size()};
4508 const std::size_t chunk {detail::elementwise_chunk};
4509 const std::size_t chunks {(count + chunk - 1) / chunk};
4510 std::vector<std::size_t> found(chunks, 0);
4511 detail::parallel_for(chunks, chunk, [&](std::size_t c) {
4512 const std::size_t last {std::min((c + 1) * chunk, count)};
4513 std::size_t total {0};
4514 for (std::size_t k {c * chunk}; k < last; ++k) {
4515 total += static_cast<std::size_t>(values[k] != T {});
4516 }
4517 found[c] = total;
4518 });
4519 return std::accumulate(found.begin(), found.end(), std::size_t {0});
4520 }
4521
4524 template <typename T>
4526 std::size_t axis)
4527 {
4528 if (auto laid {detail::as_laid_reduced(t, axis, [](const tensor<T>& elements, std::size_t along) { return count_nonzero(elements, along); })}) {
4529 return std::move(*laid);
4530 }
4531 return detail::fold_along(t, axis, std::int64_t {0},
4532 [](std::int64_t total, const T& x) { return total + static_cast<std::int64_t>(x != T {}); });
4533 }
4534
4541 template <typename T>
4542 [[nodiscard]] tensor<T> take_along_axis(const tensor<T>& t,
4543 const tensor<std::int64_t>& indices,
4544 std::size_t axis)
4545 {
4546 const detail::axis_walk walk {detail::walk_along(t.shape(), axis)};
4547 bool matched {indices.rank() == t.rank()};
4548 for (std::size_t k {0}; matched && k < t.rank(); ++k) {
4549 matched = k == axis || indices.shape()[k] == t.shape()[k];
4550 }
4551 if (!matched) {
4552 throw dimension_error("take_along_axis: the indices' shape is the tensor's but along the axis");
4553 }
4554 const auto extent {static_cast<std::int64_t>(t.shape()[axis])};
4555 if (std::ranges::any_of(indices.data(), [extent](std::int64_t i) { return i < 0 || i >= extent; })) {
4556 throw dimension_error("take_along_axis: an index out of the axis's range");
4557 }
4558 const std::size_t count {indices.shape()[axis]};
4559 const std::size_t inner {walk.inner};
4560 buffer<T> out(indices.size());
4561 const T* const in {t.data().data()};
4562 const std::int64_t* at {indices.data().data()};
4563 detail::parallel_for(indices.size() / std::max<std::size_t>(inner, 1), inner, [&](std::size_t row) {
4564 const std::size_t o {row / std::max<std::size_t>(count, 1)};
4565 for (std::size_t i {0}; i < inner; ++i) {
4566 const auto index {static_cast<std::size_t>(at[(row * inner) + i])};
4567 out[(row * inner) + i] = in[(((o * walk.extent) + index) * inner) + i];
4568 }
4569 });
4570 return tensor<T> {indices.shape(), std::move(out)};
4571 }
4572
4575 enum class take_mode : unsigned char
4576 {
4577 raise,
4578 wrap,
4579 clip,
4580 };
4581
4592 template <typename T>
4593 [[nodiscard]] tensor<T> take(const tensor<T>& t,
4594 const tensor<std::int64_t>& indices,
4595 std::size_t axis,
4596 take_mode mode)
4597 {
4598 if (axis >= t.rank()) {
4599 throw dimension_error("take: axis " + std::to_string(axis) + " is out of bounds for a tensor of rank "
4600 + std::to_string(t.rank()));
4601 }
4602 const std::vector<std::size_t>& shape {t.shape()};
4603 const auto at {shape.begin() + static_cast<std::ptrdiff_t>(axis)};
4604 std::vector<std::size_t> out_shape(shape.begin(), at);
4605 out_shape.insert(out_shape.end(), indices.shape().begin(), indices.shape().end());
4606 out_shape.insert(out_shape.end(), at + 1, shape.end());
4607 const std::size_t outer {detail::element_count(shape.begin(), at)};
4608 const std::size_t inner {detail::element_count(at + 1, shape.end())};
4609 const std::size_t count {indices.size()};
4610 const std::size_t total {detail::element_count(out_shape.begin(), out_shape.end())};
4611 const std::size_t extent {shape[axis]};
4612 if (extent == 0 && (total != 0 || (mode == take_mode::wrap && count != 0))) {
4613 throw index_error("cannot do a non-empty take from an empty axes.");
4614 }
4615 const auto signed_extent {static_cast<std::int64_t>(extent)};
4616 const buffer<std::int64_t>& index {indices.data()};
4617 const auto outside {[&](std::int64_t value) {
4618 return index_error("index " + std::to_string(value) + " is out of bounds for axis " + std::to_string(axis)
4619 + " with size " + std::to_string(extent));
4620 }};
4621 if (total == 0) {
4622 for (std::size_t k {0}; mode == take_mode::raise && outer != 0 && k < count; ++k) {
4623 if (index[k] < -signed_extent || index[k] >= signed_extent) {
4624 throw outside(index[k]);
4625 }
4626 }
4627 return tensor<T> {std::move(out_shape), buffer<T>(0)};
4628 }
4629 std::vector<std::size_t> offsets(count);
4630 for (std::size_t k {0}; k < count; ++k) {
4631 std::int64_t i {index[k]};
4632 if (mode == take_mode::wrap) {
4633 i %= signed_extent;
4634 i += i < 0 ? signed_extent : 0;
4635 }
4636 else if (mode == take_mode::clip) {
4637 i = std::clamp<std::int64_t>(i, 0, signed_extent - 1);
4638 }
4639 else if (i < -signed_extent || i >= signed_extent) {
4640 throw outside(i);
4641 }
4642 else {
4643 i += i < 0 ? signed_extent : 0;
4644 }
4645 offsets[k] = static_cast<std::size_t>(i);
4646 }
4647 buffer<T> out(total);
4648 const T* const in {t.data().data()};
4649 detail::parallel_for(outer * count, inner, [&](std::size_t row) {
4650 const std::size_t o {row / count};
4651 const T* const from {in + (((o * extent) + offsets[row % count]) * inner)};
4652 for (std::size_t i {0}; i < inner; ++i) {
4653 out[(row * inner) + i] = from[i];
4654 }
4655 });
4656 return tensor<T> {std::move(out_shape), std::move(out)};
4657 }
4658
4659 namespace detail {
4661 inline constexpr std::size_t select_positions_sorted {8};
4662
4669 template <std::floating_point T>
4670 [[nodiscard]] T stable_choice(const T* row,
4671 std::size_t count,
4672 std::size_t k,
4673 T chosen)
4674 {
4675 const bool nan {std::isnan(chosen)};
4676 if (!nan && chosen != T {0}) {
4677 return chosen;
4678 }
4679 std::size_t first {0}; // the class's first rank: the elements below a zero, or every one that is no NaN
4680 for (std::size_t i {0}; i < count; ++i) {
4681 first += static_cast<std::size_t>(nan ? !std::isnan(row[i]) : row[i] < T {0});
4682 }
4683 std::size_t rank {k - first};
4684 std::size_t i {0};
4685 for (;; ++i) { // the class holds rank k: its member of that rank is met before the row ends
4686 if (nan ? std::isnan(row[i]) : row[i] == T {0}) {
4687 if (rank == 0) {
4688 break;
4689 }
4690 --rank;
4691 }
4692 }
4693 return row[i];
4694 }
4695
4697 template <std::integral T>
4698 [[nodiscard]] T stable_choice(const T* /*row*/,
4699 std::size_t /*count*/,
4700 std::size_t /*k*/,
4701 T chosen)
4702 {
4703 return chosen;
4704 }
4705
4713 template <typename T>
4714 requires std::floating_point<T> || std::integral<T>
4715 void select_row(const T* row,
4716 std::size_t count,
4717 const std::int64_t* at,
4718 std::size_t m,
4719 T* work,
4720 T* out)
4721 {
4722 std::copy(row, row + count, work);
4723 if (m > select_positions_sorted) {
4724 sort_run(work, work + count);
4725 for (std::size_t j {0}; j < m; ++j) {
4726 out[j] = work[static_cast<std::size_t>(at[j])];
4727 }
4728 return;
4729 }
4730 std::array<std::size_t, select_positions_sorted> order {};
4731 for (std::size_t j {0}; j < m; ++j) { // sorted by insertion: eight at most
4732 const auto k {static_cast<std::size_t>(at[j])};
4733 std::size_t i {j};
4734 for (; i > 0 && order[i - 1] > k; --i) {
4735 order[i] = order[i - 1];
4736 }
4737 order[i] = k;
4738 }
4739 const auto before {[](T a, T b) { return sorts_before(a, b); }};
4740 std::size_t from {0};
4741 for (std::size_t j {0}; j < m; ++j) {
4742 const std::size_t k {order[j]};
4743 if (k < from) { // a position named twice: in place already
4744 continue;
4745 }
4746 if (k == from || k + 1 == count) { // the least or the largest of the part left: one pass finds it
4747 std::iter_swap(work + k, k == from ? std::min_element(work + from, work + count, before)
4748 : std::max_element(work + from, work + count, before));
4749 }
4750 else {
4751 std::nth_element(work + from, work + k, work + count, before);
4752 }
4753 from = k + 1;
4754 }
4755 for (std::size_t j {0}; j < m; ++j) {
4756 const auto k {static_cast<std::size_t>(at[j])};
4757 out[j] = stable_choice(row, count, k, work[k]);
4758 }
4759 }
4760
4762 template <split_complex C>
4763 void select_row(const C* row,
4764 std::size_t count,
4765 const std::int64_t* at,
4766 std::size_t m,
4767 C* work,
4768 C* out)
4769 {
4770 std::copy(row, row + count, work);
4771 sort_run(work, work + count);
4772 for (std::size_t j {0}; j < m; ++j) {
4773 out[j] = work[static_cast<std::size_t>(at[j])];
4774 }
4775 }
4776 } // namespace detail
4777
4785 template <typename T>
4786 [[nodiscard]] tensor<T> select_along_last(const tensor<T>& t,
4787 const tensor<std::int64_t>& positions)
4788 {
4789 if (t.rank() == 0 || positions.rank() == 0) {
4790 throw dimension_error("select_along_last: a tensor and positions of rank 1 or more");
4791 }
4792 const std::vector<std::size_t> lead(t.shape().begin(), t.shape().end() - 1);
4793 const std::size_t count {t.shape().back()};
4794 const std::size_t m {positions.shape().back()};
4795 const bool every {positions.rank() == 1};
4796 if (!every && !std::equal(lead.begin(), lead.end(), positions.shape().begin(), positions.shape().end() - 1)) {
4797 throw dimension_error("select_along_last: positions of shape (m,) or the lines' shape and m");
4798 }
4799 const auto extent {static_cast<std::int64_t>(count)};
4800 if (std::ranges::any_of(positions.data(), [extent](std::int64_t k) { return k < 0 || k >= extent; })) {
4801 throw dimension_error("select_along_last: a position out of the lines");
4802 }
4803 std::vector<std::size_t> shape {lead};
4804 shape.push_back(m);
4805 const std::size_t lines {std::accumulate(lead.begin(), lead.end(), std::size_t {1}, std::multiplies {})};
4806 buffer<T> out(lines * m);
4807 const T* const in {t.data().data()};
4808 const std::int64_t* at {positions.data().data()};
4809 const std::size_t block {std::max<std::size_t>(1, 4096 / std::max<std::size_t>(count, 1))}; // rows a task takes
4810 const std::size_t blocks {(lines + block - 1) / block};
4811 detail::parallel_for(blocks, block * count * 16, [&](std::size_t b) {
4812 buffer<T> work(count);
4813 for (std::size_t line {b * block}; line < std::min(lines, (b + 1) * block); ++line) {
4814 detail::select_row(in + (line * count), count, at + (every ? 0 : line * m), m, work.data(),
4815 out.data() + (line * m));
4816 }
4817 });
4818 return tensor<T> {std::move(shape), std::move(out)};
4819 }
4820
4821 namespace detail {
4824 inline constexpr std::size_t correlation_block {16};
4825
4827 template <typename T>
4828 [[nodiscard]] T correlation_step(const T& sum,
4829 const T& x,
4830 const T& y)
4831 {
4832 return typename arithmetic<T>::plus {}(sum, typename arithmetic<T>::multiplies {}(x, y));
4833 }
4834
4836 template <split_complex C>
4837 [[nodiscard]] C correlation_step(const C& sum,
4838 const C& x,
4839 const C& y)
4840 {
4841 return sum + numpy_multiply(x, std::conj(y), false);
4842 }
4843 } // namespace detail
4844
4853 template <typename T>
4854 [[nodiscard]] tensor<T> correlate(const tensor<T>& a,
4855 const tensor<T>& v,
4856 std::ptrdiff_t first,
4857 std::size_t count)
4858 {
4859 constexpr std::size_t block {detail::correlation_block};
4860 const auto n {static_cast<std::ptrdiff_t>(a.size())};
4861 const auto m {static_cast<std::ptrdiff_t>(v.size())};
4862 const T* const x {a.data().data()};
4863 const T* const y {v.data().data()};
4864 buffer<T> out(count);
4865 detail::parallel_for((count + block - 1) / block, block * static_cast<std::size_t>(m), [&](std::size_t b) {
4866 const std::size_t lags {std::min(block, count - (b * block))};
4867 const std::ptrdiff_t k {first + static_cast<std::ptrdiff_t>(b * block)};
4868 if (lags == block && k >= 0 && k + static_cast<std::ptrdiff_t>(block) + m <= n + 1) {
4869 std::array<T, block> sum {};
4870 for (std::ptrdiff_t j {0}; j < m; ++j) {
4871 for (std::size_t l {0}; l < block; ++l) {
4872 sum[l] = detail::correlation_step(sum[l], x[j + k + static_cast<std::ptrdiff_t>(l)], y[j]);
4873 }
4874 }
4875 std::copy(sum.begin(), sum.end(), out.data() + (b * block));
4876 return;
4877 }
4878 for (std::size_t l {0}; l < lags; ++l) {
4879 const std::ptrdiff_t lag {k + static_cast<std::ptrdiff_t>(l)};
4880 T sum {};
4881 for (std::ptrdiff_t j {std::max<std::ptrdiff_t>(0, -lag)}; j < std::min(m, n - lag); ++j) {
4882 sum = detail::correlation_step(sum, x[j + lag], y[j]);
4883 }
4884 out[(b * block) + l] = sum;
4885 }
4886 });
4887 return tensor<T> {{count}, std::move(out)};
4888 }
4889
4895 template <typename T>
4897 const tensor<T>& v,
4898 bool right = false)
4899 {
4900 if (a.rank() != 1) {
4901 throw dimension_error("searchsorted: a sorted vector");
4902 }
4903 const T* const first {a.data().data()};
4904 const T* const last {first + a.size()};
4905 const auto less {[](const T& x, const T& y) { return detail::sorts_before(x, y); }};
4906 return detail::parallel_map(v, [&](const T& x) {
4907 const T* const at {right ? std::upper_bound(first, last, x, less) : std::lower_bound(first, last, x, less)};
4908 return static_cast<std::int64_t>(at - first);
4909 });
4910 }
4911
4914 template <std::totally_ordered T>
4915 [[nodiscard]] tensor<T> max(const tensor<T>& t,
4916 std::size_t axis)
4917 {
4918 if (auto laid {detail::as_laid_reduced(t, axis, [](const tensor<T>& elements, std::size_t along) { return max(elements, along); })}) {
4919 return std::move(*laid);
4920 }
4921 return detail::extremes_along(t, axis, std::greater<T> {});
4922 }
4923
4926 template <std::totally_ordered T>
4927 [[nodiscard]] tensor<T> min(const tensor<T>& t,
4928 std::size_t axis)
4929 {
4930 if (auto laid {detail::as_laid_reduced(t, axis, [](const tensor<T>& elements, std::size_t along) { return min(elements, along); })}) {
4931 return std::move(*laid);
4932 }
4933 return detail::extremes_along(t, axis, std::less<T> {});
4934 }
4935
4942 template <typename T>
4943 [[nodiscard]] tensor<T> broadcast_to(const tensor<T>& t,
4944 std::vector<std::size_t> shape)
4945 {
4946 const std::vector<std::size_t>& from {t.shape()};
4947 bool fits {from.size() <= shape.size()};
4948 for (std::size_t i {0}; fits && i < from.size(); ++i) {
4949 const std::size_t have {from[from.size() - 1 - i]};
4950 fits = have == 1 || have == shape[shape.size() - 1 - i];
4951 }
4952 if (!fits) {
4953 throw dimension_error("broadcast_to: the shape " + detail::shape_text(from) + " does not broadcast to "
4954 + detail::shape_text(shape));
4955 }
4956 const std::size_t total {detail::element_count(shape.begin(), shape.end())};
4957 buffer<T> out(total);
4958 if (total == 0) {
4959 return tensor<T> {std::move(shape), std::move(out)};
4960 }
4961 const std::vector<std::size_t> strides {detail::broadcast_strides(from, shape.size())};
4962 const std::size_t inner {shape.empty() ? 1 : shape.back()};
4963 const bool spread {!shape.empty() && strides.back() == 0}; // the last axis repeats one element
4964 const T* const in {t.data().data()};
4965 detail::parallel_for(total / inner, inner, [&](std::size_t row) {
4966 std::size_t rest {row};
4967 std::size_t at {0};
4968 for (std::size_t axis {shape.size() - (shape.empty() ? 0 : 1)}; axis-- > 0;) {
4969 at += (rest % shape[axis]) * strides[axis];
4970 rest /= shape[axis];
4971 }
4972 T* const into {out.data() + (row * inner)};
4973 if (spread) {
4974 std::fill_n(into, inner, in[at]);
4975 }
4976 else {
4977 std::copy_n(in + at, inner, into);
4978 }
4979 });
4980 return tensor<T> {std::move(shape), std::move(out)};
4981 }
4982
4987 template <typename T>
4988 [[nodiscard]] tensor<T> reshape(const tensor<T>& t,
4989 std::vector<std::size_t> shape)
4990 {
4991 std::size_t count {1};
4992 for (const std::size_t extent : shape) {
4993 count = detail::checked_mul(count, extent);
4994 }
4995 if (count != t.size()) {
4996 throw dimension_error("cannot reshape a tensor of size " + std::to_string(t.size()) + " into shape "
4997 + detail::shape_text(shape));
4998 }
4999 return t.reshaped(std::move(shape)); // the same elements, shared: no copy
5000 }
5001
5008 template <typename T>
5009 [[nodiscard]] tensor<T> diagonal(const tensor<T>& t,
5010 std::ptrdiff_t offset)
5011 {
5012 const std::vector<std::size_t>& shape {t.shape()};
5013 if (shape.size() < 2) {
5014 throw dimension_error("diagonal: a matrix or a stack of matrices (rank >= 2)");
5015 }
5016 const std::size_t rows {shape[shape.size() - 2]};
5017 const std::size_t cols {shape[shape.size() - 1]};
5018 const std::size_t magnitude {offset < 0 ? std::size_t {0} - static_cast<std::size_t>(offset) : static_cast<std::size_t>(offset)};
5019 const std::size_t r0 {offset < 0 ? magnitude : 0};
5020 const std::size_t c0 {offset < 0 ? 0 : magnitude};
5021 const std::size_t length {r0 < rows && c0 < cols ? std::min(rows - r0, cols - c0) : 0};
5022 return t.diagonal_view(r0, c0, length);
5023 }
5024
5025 namespace detail {
5028 [[nodiscard]] inline std::size_t triangle_split(std::size_t i,
5029 std::size_t rows,
5030 std::size_t cols,
5031 std::ptrdiff_t k,
5032 bool lower)
5033 {
5034 const auto r {static_cast<std::ptrdiff_t>(rows)};
5035 const auto c {static_cast<std::ptrdiff_t>(cols)};
5036 const auto band {std::clamp<std::ptrdiff_t>(k, -r - 1, c + 1)};
5037 const auto edge {static_cast<std::ptrdiff_t>(i) + band + static_cast<std::ptrdiff_t>(lower)};
5038 return static_cast<std::size_t>(std::clamp<std::ptrdiff_t>(edge, 0, c));
5039 }
5040 } // namespace detail
5041
5047 template <typename T>
5048 [[nodiscard]] tensor<T> triangle(const tensor<T>& t,
5049 std::ptrdiff_t k,
5050 bool lower)
5051 {
5052 const std::vector<std::size_t>& shape {t.shape()};
5053 if (shape.size() < 2) {
5054 throw dimension_error("triangle: a matrix or a stack of matrices (rank >= 2)");
5055 }
5056 if (const auto laid {t.laid_out()}) { // its matrices' axes the last two as its elements lie: zeroed there
5057 const std::vector<std::size_t>& order {laid->second};
5058 const std::size_t rank {order.size()};
5059 const bool kept {order[rank - 2] == rank - 2 && order[rank - 1] == rank - 1};
5060 if (kept || (order[rank - 2] == rank - 1 && order[rank - 1] == rank - 2)) { // a matrix's transpose: the other side of −k
5061 const auto reach {static_cast<std::ptrdiff_t>(shape[rank - 2] + shape[rank - 1])}; // −k defined, the same split
5062 const std::ptrdiff_t within {std::clamp(k, -reach, reach)};
5063 return triangle(laid->first, kept ? within : -within, kept == lower).transposed(detail::view_axes(order));
5064 }
5065 }
5066 const std::size_t rows {shape[shape.size() - 2]};
5067 const std::size_t cols {shape[shape.size() - 1]};
5068 const std::size_t count {t.size()};
5069 buffer<T> out(count);
5070 if (count == 0) {
5071 return tensor<T> {shape, std::move(out)};
5072 }
5073 const T* const in {t.data().data()};
5074 detail::parallel_for(count / cols, cols * detail::memory_element_weight, [&](std::size_t line) {
5075 const std::size_t split {detail::triangle_split(line % rows, rows, cols, k, lower)};
5076 const std::size_t first {lower ? 0 : split}; // the kept columns: [first, last)
5077 const std::size_t last {lower ? split : cols};
5078 T* const row {out.data() + (line * cols)};
5079 std::fill_n(row, first, T {});
5080 std::copy_n(in + (line * cols) + first, last - first, row + first);
5081 std::fill_n(row + last, cols - last, T {});
5082 });
5083 return tensor<T> {shape, std::move(out)};
5084 }
5085
5091 [[nodiscard]] inline std::array<tensor<std::int64_t>, 2> triangle_indices(std::size_t rows,
5092 std::size_t cols,
5093 std::ptrdiff_t k,
5094 bool lower)
5095 {
5096 std::vector<std::size_t> offsets(rows + 1, 0);
5097 for (std::size_t i {0}; i < rows; ++i) {
5098 const std::size_t split {detail::triangle_split(i, rows, cols, k, lower)};
5099 offsets[i + 1] = offsets[i] + (lower ? split : cols - split);
5100 }
5101 const std::size_t found {offsets.back()};
5102 buffer<std::int64_t> row_of(found);
5103 buffer<std::int64_t> col_of(found);
5104 detail::parallel_for(rows, cols * detail::memory_element_weight, [&](std::size_t i) {
5105 const std::size_t split {detail::triangle_split(i, rows, cols, k, lower)};
5106 const std::size_t at {offsets[i]};
5107 auto column {static_cast<std::int64_t>(lower ? 0 : split)};
5108 for (std::size_t j {at}; j < offsets[i + 1]; ++j) {
5109 row_of[j] = static_cast<std::int64_t>(i);
5110 col_of[j] = column++;
5111 }
5112 });
5113 return {tensor<std::int64_t> {{found}, std::move(row_of)}, tensor<std::int64_t> {{found}, std::move(col_of)}};
5114 }
5115
5117 template <typename T>
5118 [[nodiscard]] tensor<T> full(std::vector<std::size_t> shape,
5119 const T& value)
5120 {
5121 std::size_t count {1};
5122 for (const std::size_t extent : shape) {
5123 count = detail::checked_mul(count, extent);
5124 }
5125 buffer<T> values(count); // its pages first touched by the threads that fill them
5126 detail::parallel_fill(values.data(), count, value);
5127 return tensor<T> {std::move(shape), std::move(values)};
5128 }
5129
5131 template <typename T>
5132 [[nodiscard]] tensor<T> zeros(std::vector<std::size_t> shape)
5133 {
5134 std::size_t count {1};
5135 for (const std::size_t extent : shape) {
5136 count = detail::checked_mul(count, extent);
5137 }
5138 buffer<T> values(count);
5139 detail::parallel_zero(values.data(), count);
5140 return tensor<T> {std::move(shape), std::move(values)};
5141 }
5142
5148 template <typename T>
5149 [[nodiscard]] tensor<T> diag(const tensor<T>& v,
5150 std::ptrdiff_t k)
5151 {
5152 if (v.rank() != 1) {
5153 throw dimension_error("diag: a vector");
5154 }
5155 const std::size_t length {v.size()};
5156 const std::size_t magnitude {k < 0 ? std::size_t {0} - static_cast<std::size_t>(k) : static_cast<std::size_t>(k)};
5157 const std::size_t n {length + magnitude}; // within range: |k| ≤ 2⁶³, a vector's length below it
5158 const std::size_t count {detail::checked_mul(n, n)};
5159 buffer<T> values(count);
5160 detail::parallel_zero(values.data(), count);
5161 const std::size_t r0 {k < 0 ? magnitude : 0};
5162 const std::size_t c0 {k < 0 ? 0 : magnitude};
5163 const T* const from {v.data().data()};
5164 for (std::size_t i {0}; i < length; ++i) {
5165 values[((r0 + i) * n) + c0 + i] = from[i];
5166 }
5167 return tensor<T> {{n, n}, std::move(values)};
5168 }
5169
5171 template <typename T>
5172 [[nodiscard]] tensor<T> ones(std::vector<std::size_t> shape)
5173 {
5174 return full<T>(std::move(shape), static_cast<T>(1));
5175 }
5176
5178 template <typename T>
5179 [[nodiscard]] tensor<T> eye(std::size_t n)
5180 {
5181 buffer<T> out(detail::checked_mul(n, n), T {});
5182 for (std::size_t i {0}; i < n; ++i) {
5183 out[(i * n) + i] = static_cast<T>(1);
5184 }
5185 return tensor<T> {{n, n}, std::move(out)};
5186 }
5187
5197 [[nodiscard]] inline tensor<double> arange(double start,
5198 double stop,
5199 double step,
5200 bool fused = true)
5201 {
5202 if (step == 0.0 || !std::isfinite(step) || !std::isfinite(start) || !std::isfinite(stop)) {
5203 throw dimension_error("arange needs finite bounds and a finite, non-zero step");
5204 }
5205 const double length {std::ceil((stop - start) / step)};
5206 if (!(length < 0x1p64)) { // past std::size_t, or an infinite span: no conversion holds it
5207 throw dimension_error("arange: the length does not fit in std::size_t");
5208 }
5209 const std::size_t count {length > 0.0 ? static_cast<std::size_t>(length) : 0};
5210 buffer<double> out(count);
5211 const double second {start + step};
5212 const double delta {second - start};
5213 if (count > 0) {
5214 out[0] = start;
5215 }
5216 if (count > 1) {
5217 out[1] = second;
5218 }
5219 if (fused) {
5220 for (std::size_t i {2}; i < count; ++i) {
5221 out[i] = std::fma(static_cast<double>(i), delta, start);
5222 }
5223 }
5224 else {
5225 for (std::size_t i {2}; i < count; ++i) {
5226 // the product rounded by itself, `i · delta − 0`, so that no compiler fuses it into the sum
5227 out[i] = start + std::fma(static_cast<double>(i), delta, -0.0);
5228 }
5229 }
5230 return tensor<double> {{count}, std::move(out)};
5231 }
5232
5238 [[nodiscard]] inline tensor<double> linspace(double start,
5239 double stop,
5240 std::size_t count)
5241 {
5242 buffer<double> out(count);
5243 const double step {count > 1 ? (stop - start) / static_cast<double>(count - 1) : 0.0};
5244 for (std::size_t i {0}; i < count; ++i) {
5245 // The product rounded before the sum, as NumPy's linspace computes it in two operations. A compiler
5246 // may fuse `i * step` into the addition after it (GCC does, across statements, wherever there is a
5247 // fused multiply-add); a product computed by fma, `i * step - 0`, is exactly the rounded one, signed
5248 // zero included, and no multiplication is left to fuse.
5249 const double offset {std::fma(static_cast<double>(i), step, -0.0)};
5250 out[i] = start + offset;
5251 }
5252 if (count > 1) {
5253 out[count - 1] = stop;
5254 }
5255 return tensor<double> {{count}, std::move(out)};
5256 }
5257} // namespace numerical
5258
5259#endif // NUMERICAL_TENSOR_OPS_HPP
A standard exception Std that numerical::error tags too, its one what answering for both bases.
Definition vector.hpp:61
Thrown when an operation's operands have incompatible shapes.
Definition vector.hpp:76
Thrown when an index or a range leaves its axis.
Definition vector.hpp:98
Thrown when an integer is raised to a negative integer power, as NumPy refuses it: no integer is the ...
negative_power_error(const std::string &message)
Builds the error.
A dense N-dimensional tensor of T with value semantics, read row-major. Its elements are immutable on...
Definition tensor.hpp:88
const std::vector< std::size_t > & shape() const noexcept
The size of each axis.
Definition tensor.hpp:326
std::optional< std::pair< tensor, std::vector< std::size_t > > > laid_out() const
Where this view reaches each of its elements once from the first (a transpose's, transposed): its axe...
Definition tensor.hpp:291
tensor reshaped(std::vector< std::size_t > shape) const
This tensor's elements in shape, without a copy: the new tensor shares the buffer,...
Definition tensor.hpp:152
const buffer< T > & data() const
The elements, row-major. A view's (transposed) are copied out by the first call, one thread copying w...
Definition tensor.hpp:370
std::size_t rank() const noexcept
The number of axes (0 for a scalar).
Definition tensor.hpp:314
const T * origin() const noexcept
The first element where the elements are held (index 0 along every axis): a view (transposed,...
Definition tensor.hpp:333
const std::vector< std::ptrdiff_t > & strides() const noexcept
Each axis's step, in elements, from the first element (origin), row-major unless a view,...
Definition tensor.hpp:340
std::size_t size() const noexcept
The total number of elements.
Definition tensor.hpp:320
bool contiguous() const noexcept
Whether the elements lie row-major where they are held: data then copies nothing.
Definition tensor.hpp:346
tensor diagonal_view(std::size_t row, std::size_t column, std::size_t length) const
The length elements from (row, column) down the diagonal of each matrix (the last two axes),...
Definition tensor.hpp:258
A dense vector of T with value semantics.
Definition vector.hpp:171
The elementary functions of a complex number of float or double as the GNU C library computes them (N...
The types whose indices order_run sorts by the elements' bits.
exp, exp2, expm1, tanh, sinh and cosh of double by a table of 2^(j/128) and a polynomial,...
bool numpy_fuses_reductions() noexcept
Whether NumPy's complex product fuses its multiply-adds in its reduction loop (the product's loop who...
void parallel_sort_run(T *data, std::size_t count, std::size_t pieces, const SortRun &sort_piece, const Less &less)
Sorts the count elements at data stably in pieces, a power of four, among the threads: each piece sor...
bool is_nan(const T &)
A NaN? Never for a scalar without one.
T elementary_exp2(const T &x)
exp2 of a scalar, as std::exp2 gives it.
T divided_by_count(T value, std::size_t n)
A mean: value divided by the count n, in the wider of T and double and back to T, the count exact (to...
void double_sines_into(const double *in, double *out, std::size_t count)
sin (Cosine false) or cos of the count doubles at in into out, each double_sine's: table_sine to sine...
void exp_into(const double *in, double *out, std::size_t count)
exp of the count doubles at in into out, each elementary_exp's: table_exp in vectors (with the FMA in...
constexpr std::size_t band_columns
The columns of a band that one task of fold_along and scan_along walks row by row.
tensor< T > cos_viewed(const tensor< T > &a)
cos of every element of a: of a contiguous copy (cos_of).
auto broadcast_rows(const T *a, const std::vector< std::size_t > &a_shape, const T *b, const std::vector< std::size_t > &b_shape, const Op &op)
op(a, b) element by element of the row-major a (a_shape) and b (b_shape), broadcast against each othe...
std::int64_t parallel_extreme_index(const T *first, std::size_t count, const Order &larger)
extreme_index of the count contiguous elements at first, by chunks on the threads: each chunk's own,...
void select_row(const T *row, std::size_t count, const std::int64_t *at, std::size_t m, T *work, T *out)
Into out, the elements of the count at row that a stable sort puts at the m positions at,...
std::complex< R > complex_log(const std::complex< R > &z)
log(z): log|z| + i arg(z), log|z| as log1p(|z|² − 1) / 2 near the unit circle (|z|² − 1 exact or almo...
double float_sine(double x)
sin (Cosine false) or cos of a double that a float holds: reduced_sine to sine_reduction_bound,...
std::size_t part_begin(std::size_t count, std::size_t parts, std::size_t p) noexcept
Where part p of parts sharing count elements begins: the first count % parts one longer.
double double_sine(double x)
sin (Cosine false) or cos of a double, as double_sines_into gives it: table_sine to sine_reduction_bo...
axis_walk walk_along(const std::vector< std::size_t > &shape, std::size_t axis)
axis_walk of shape along axis; outer and inner 0 where the result is empty.
tensor< R > pairs_into(const T *a, const T *b, const std::vector< std::size_t > &shape, const Op &op)
op(a, b) element by element of the row-major a and b, both of shape shape, in op's result type R,...
Definition tensor.hpp:743
tensor< T > log1p_viewed(const tensor< T > &a)
log1p of every element of a: of a contiguous copy (log1p_of).
T elementary_atan(const T &x)
atan of a scalar, as std::atan gives it.
constexpr std::size_t radix_sort_least
Runs this long or longer are sorted by their bits (radix_sort); shorter ones by comparisons.
C numpy_reciprocal(const C &x, bool fused)
NumPy's complex reciprocal 1 / x (its reciprocal, the loop x ** -1 takes): the ratio of the smaller p...
T clipped(T x, T low, T high) noexcept
ieee_minimum of ieee_maximum of x and low, and high, the bounds no NaN: the same bits (a NaN x kept,...
constexpr double sine_reduction_bound
The magnitude to which reduced_sine reduces its argument: k below 2²⁰, k·π/2's parts exact enough.
bool lies_close(const tensor< T > &t) noexcept
Whether an element-wise operation reads the view t's elements where they lie: stepping within a cache...
bool numpy_loop_is_scalar(const std::vector< std::size_t > &a_shape, const std::vector< std::size_t > &b_shape) noexcept
Whether NumPy multiplies an operand of shape a_shape by one of b_shape in its scalar loop rather than...
void sinh_into(const double *in, double *out, std::size_t count)
sinh of the count doubles at in into out, each elementary_sinh's: table_sinh in vectors (with the FMA...
constexpr U sign_bit
The highest bit of U, a real's or an integer's sign.
T elementary_cosh(const T &x)
cosh of a scalar, as std::cosh gives it.
double table_expm1(double x) noexcept
expm1(x) by table_exp's reduction, x = (128 e + j) ln 2/128 + r: in units of 2^e, 2^(j/128) − 2^−e
std::complex< R > complex_sin(const std::complex< R > &z)
sin(z): sin(re) cosh(im) + i cos(re) sinh(im) of |re|, the sign of re carried by the sine; an imagina...
tensor< T > sin_of(const T *data, const std::vector< std::size_t > &shape)
sin of every one of the row-major scalars at data, of shape shape: std::sin of each.
bool numpy_fuses_scalar_products() noexcept
Whether NumPy's complex product fuses its multiply-adds in its scalar loop (where numpy_loop_is_scala...
void log1p_into(const double *in, double *out, std::size_t count)
log1p of the count doubles at in into out, each elementary_log1p's: table_log1p in vectors (with the ...
void run_fused(const Body &body)
body(0), where std::fma is the instruction (with_fma_instruction) on an x86-64 processor with AVX2 an...
Definition tensor.hpp:698
double table_exp2(double x) noexcept
2^x for an x whose result is normal or infinite (−1022 or above): x = (128 e + j)/128 + f,...
void count_bytes(byte_counts< U > &counts, U key) noexcept
Counts key's bytes in counts. Every byte is counted in the one read that makes the keys,...
constexpr std::size_t correlation_block
The lags numerical::correlate sums together: a block's sums held in registers (four AVX2 vectors of d...
C numpy_power(const C &a, const C &b, bool fused)
NumPy's complex power a ** b (npymath's npy_cpow): 1 for a zero exponent, 0 or NaN for a zero base; a...
tensor< R > broadcast_into(const T *a, const std::vector< std::size_t > &a_shape, const std::vector< std::ptrdiff_t > &a_strides, const T *b, const std::vector< std::size_t > &b_shape, const std::vector< std::ptrdiff_t > &b_strides, const Op &op)
op(a, b) element by element of a, of shape a_shape, its elements a_strides apart along each axis from...
Definition tensor.hpp:769
void float_sines_into(const double *in, double *out, std::size_t count)
sin (Cosine false) or cos of the count doubles at in, each a float's, into out, each float_sine's: re...
carried_t< T > real_part(const T &x)
The real part of a real (or general) scalar, in carried_t.
T square(const T &x)
x · x, by the type's own product (a complex one's C Annex G, as the library's *).
std::complex< R > complex_tan(const std::complex< R > &z)
tan(z): (sin(re) cos(re) + i sinh(im) cosh(im)) / (cos(re)² + sinh(im)²); past half_exp_bound the ima...
T settled_zero(T value, const T *first, std::size_t count, std::size_t stride, const Order &larger)
The extreme value of count elements stride apart from first, its zero's sign settled as IEEE 754-2019...
T elementary_tanh(const T &x)
tanh of a scalar, as std::tanh gives it.
R divided_by_zero(R x) noexcept
x divided by +0, as IEEE divides it, spelled out: a NaN for a zero or a NaN, else an infinity of x's ...
T stable_choice(const T *row, std::size_t count, std::size_t k, T chosen)
Of the count elements at row, the one a stable sort (sort_run) puts at k, from the one chosen that a ...
tensor< T > cosh_viewed(const tensor< T > &a)
cosh of every element of a: of a contiguous copy (cosh_of).
tensor< T > cosh_of(const T *data, const std::vector< std::size_t > &shape)
cosh of every one of the row-major scalars at data, of shape shape: std::cosh of each.
T elementary_cos(const T &x)
cos of a scalar, as std::cos gives it.
bool radix_passes(U *keys, U *key_spare, std::int64_t *indices, std::int64_t *index_spare, std::size_t count, std::vector< byte_counts< U > > &counts)
Sorts the count keys at keys stably by their bytes from the least significant, the indices at indices...
std::int64_t extreme_index(const T *first, std::size_t count, const Order &larger)
The index of the first largest (larger std::greater) or smallest of count contiguous elements,...
bool numpy_fuses_products() noexcept
Whether NumPy's complex product fuses its multiply-adds on this processor: on arm64,...
tensor< R > fold_along(const tensor< T > &t, std::size_t axis, const R &init, const Step &step)
Folds each line of t along axis in order, step(running, x) from init: a tensor of the result type R,...
C numpy_multiply(const C &a, const C &b, bool fused)
a · b of complex numbers as NumPy's array loop computes it: ai·bi and ai·br rounded,...
T elementary_expm1(const T &x)
expm1 of a scalar, as std::expm1 gives it.
T chunked_conjugate_dot(const T *a, const T *b, std::size_t count)
Σ conj(a[i]) · b[i] (zdotc): chunked_dot of each chunk by conjugate_dot_product, without a conjugated...
Definition gemm.hpp:5099
constexpr std::size_t elementwise_chunk
The elements one task of an element-wise operation takes, a quarter of a thread's work: a tensor of a...
Definition parallel.hpp:734
tensor< T > sinh_of(const T *data, const std::vector< std::size_t > &shape)
sinh of every one of the row-major scalars at data, of shape shape: std::sinh of each.
bool numpy_fuses_quotients() noexcept
Whether NumPy's complex quotient fuses its multiply-adds, as its PyPI wheels are built: on macOS arm6...
void exp2_into(const double *in, double *out, std::size_t count)
exp2 of the count doubles at in into out, each elementary_exp2's: table_exp2 in vectors (with the FMA...
tensor< T > log1p_of(const T *data, const std::vector< std::size_t > &shape)
log1p of every one of the row-major scalars at data, of shape shape: std::log1p of each.
void sort_run(T *first, T *last, std::size_t parts=1)
Sorts [first, last) stably as sorts_before orders reals: the NaNs moved behind the rest in their orde...
double table_atan(double x) noexcept
atan(x): y = |x|, or 1/|x| past 1 (its rounding error carried), c = k/64 the nearest point of atan_ta...
auto with_decimal_rounding(int decimals, const F &f)
f(rounding) with the decimal_rounding of R to decimals places: in R, as NumPy's, but in double for a ...
auto times(const A &a, const B &b)
a · b.
Definition eigen.hpp:70
T walked_extreme(const T *first, std::size_t count, std::size_t stride, const Order &larger)
extreme of a line walked element by element: a NaN wins, as in NumPy (every comparison with it is fal...
C numpy_multiply_late(const C &a, const C &b)
a · b of complex numbers as NumPy's reduction loop computes it where MSVC contracts it (its Windows w...
tensor< T > expm1_viewed(const tensor< T > &a)
expm1 of every element of a: of a contiguous copy (expm1_of).
tensor< C > numpy_square(const tensor< C > &a, const numpy_loops &loops=numpy_wheel_loops())
x ** 2 of a complex tensor as NumPy's square computes it (the loop of an array to the Python integer ...
tensor< T > runs_into(const tensor< T > &t, const Kernel &kernel)
kernel (kernel(in, out, count), of contiguous elements) of every element of t: a contiguous tensor at...
tensor< C > numpy_prod(const tensor< C > &t, std::size_t axis, const numpy_loops &loops=numpy_wheel_loops())
The products of a complex t along axis, each line in order from 1 (NumPy's identity,...
tensor< T > exp2_of(const T *data, const std::vector< std::size_t > &shape)
exp2 of every one of the row-major scalars at data, of shape shape: std::exp2 of each.
std::complex< R > complex_tanh(const std::complex< R > &z)
tanh(z): (sinh(re) cosh(re) + i sin(im) cos(im)) / (sinh(re)² + cos(im)²); past half_exp_bound the re...
constexpr bool by_reduction(std::size_t inner) noexcept
Whether extremes_along takes the reduction's path: an exact type, or an axis of single elements....
bool sorts_before(T a, T b) noexcept
Whether a sorts before b as NumPy sorts reals: ascending, NaNs last.
auto parallel_map(const T *data, const std::vector< std::size_t > &shape, const F &f) -> tensor< std::decay_t< decltype(f(std::declval< const T & >()))> >
f(x) for every one of the row-major elements at data, of shape shape (another array's storage,...
std::size_t triangle_split(std::size_t i, std::size_t rows, std::size_t cols, std::ptrdiff_t k, bool lower)
The first column of row i past the part to and below its k diagonal (lower), or the first column of t...
T contiguous_extreme(const T *first, std::size_t count, const Order &larger)
The largest (larger is std::greater) or smallest of count contiguous elements, or a NaN if one is amo...
double table_sinh(double x) noexcept
sinh(x) = (d − u)/2 (hyperbolic_parts), the sign of x: within 0.503 ulp of the exact value up to the ...
tensor< T > sin_viewed(const tensor< T > &a)
sin of every element of a: of a contiguous copy (sin_of).
tensor< T > atan_of(const T *data, const std::vector< std::size_t > &shape)
atan of every one of the row-major scalars at data, of shape shape: std::atan of each.
std::vector< std::size_t > partition_positions(std::vector< std::size_t > kths, std::size_t extent)
kths ascending and each once, every one checked against extent.
void atan_into(const double *in, double *out, std::size_t count)
atan of the count doubles at in into out, each elementary_atan's: table_atan in vectors (with the FMA...
T correlation_step(const T &sum, const T &x, const T &y)
sum + x · y of a real or an integer (wrapped) correlation's terms.
double table_exp(double x) noexcept
exp(x) for an x whose result is normal or infinite (exp_normal_bound or above): x = (128 e + j) ln 2/...
tensor< T > cos_of(const T *data, const std::vector< std::size_t > &shape)
cos of every one of the row-major scalars at data, of shape shape: std::cos of each.
tensor< T > exp2_viewed(const tensor< T > &a)
exp2 of every element of a: of a contiguous copy (exp2_of).
T extreme(const T *first, std::size_t count, std::size_t stride, const Order &larger)
The largest (larger is std::greater) or smallest element of a line of reals: a contiguous one by the ...
void parallel_for(std::size_t count, std::size_t work_per_index, const Body &body)
Runs body(i) for every i in [0, count).
Definition parallel.hpp:790
T from_order_key(order_key_t< T > key) noexcept
The real whose order_key is key.
auto as_laid_reduced(const tensor< T > &t, std::size_t axis, const F &f) -> std::optional< decltype(f(t, axis))>
as_laid_along of an f whose result takes the axis out, along any axis: a reduction of columns runs ac...
T conjugated_chunks(const T *a, const T *b, std::size_t count)
Σ a[i] · b[i] of reals, their own conjugates: chunked_dot.
std::conditional_t< sizeof(T)==8, std::uint64_t, std::uint32_t > order_key_t
The unsigned integer of T's width, whose order order_key maps T's onto.
T elementary_tan(const T &x)
tan of a scalar, as std::tan gives it.
double table_tanh(double x) noexcept
tanh(x) = t / (t + 2), t = expm1(2|x|) as a pair (expm1_parts, its two parts summed exactly),...
gemm_path gemm_path_current() noexcept
The path the next double product takes: the most capable one under the cap that runs here (each path'...
Definition gemm.hpp:148
constexpr double exp_normal_bound
ln 2^-1022 toward zero: below it, exp is subnormal, where table_exp's two scalings would round twice.
std::size_t merged_pieces(std::size_t threads) noexcept
The pieces parallel_sort_run sorts a line in for threads: the least power of four that many.
bool wins_any_order(T x) noexcept
A real's NaN, which wins numpy.argmax and numpy.argmin.
T elementary_log(const T &x)
log of a scalar, as std::log gives it.
T ieee_maximum(T a, T b) noexcept
The larger of a and b as IEEE 754-2019's maximum orders them: a NaN wins (a's where both are),...
tensor< double > by_runs(const tensor< double > &a, doubles_into into)
A function of every element of a, its kernel of doubles (runs_into).
void tanh_into(const double *in, double *out, std::size_t count)
tanh of the count doubles at in into out, each elementary_tanh's: table_tanh in vectors (with the FMA...
std::pair< T, T > floor_divmod(T a, T b) noexcept
a // b and a % b of reals as NumPy's divmod computes them (Python's: the quotient floored,...
constexpr double exp2_normal_bound
2^x below which exp2 is subnormal, where table_exp2's two scalings would round twice.
std::complex< R > complex_exp(const std::complex< R > &z)
exp(z): exp(re) times cos(im) + i sin(im), a real part past exp_bound taken off in steps of it (each ...
tensor< T > exp_viewed(const tensor< T > &a)
exp of every element of a: of a contiguous copy (exp_of).
tensor< T > expm1_of(const T *data, const std::vector< std::size_t > &shape)
expm1 of every one of the row-major scalars at data, of shape shape: std::expm1 of each.
void radix_sort(T *first, T *last, std::size_t parts=1)
Sorts [first, last) by order_key, a byte at a time from the least significant (by radix_passes,...
double table_log1p(double x) noexcept
ln(1 + x) by log_parts of 1 + x as a pair (its two-sum): near 0, where c = 1, r is x exactly....
T parallel_extreme(const T *first, std::size_t count, const Order &larger)
contiguous_extreme of count elements on the threads: runs of pairwise_leaf each its own,...
bool holds(const T *first, std::size_t count, const Match &match) noexcept
Whether one of the count contiguous elements at first passes match: tested without a branch,...
double table_cosh(double x) noexcept
cosh(x) = (d + u)/2 (hyperbolic_parts): within 0.502 ulp of the exact value up to the overflow; NaN f...
C library_power(const C &a, const C &b)
a ** b by std::pow: libstdc++'s calls the C library's cpow, which NumPy calls (the same bits under gl...
T elementary_exp(const T &x)
exp of a scalar, as std::exp gives it.
T elementary_sin(const T &x)
sin of a scalar, as std::sin gives it.
std::complex< R > complex_sqrt(const std::complex< R > &z)
The square root of a complex number, as the GNU C library's csqrt computes it (NumPy's on Linux,...
tensor< float > through_doubles(const float *data, const std::vector< std::size_t > &shape, Into into)
The function into (one of the *_into of doubles) of every one of the row-major floats at data,...
T masked_select(bool condition, T if_true, T if_false) noexcept
if_true where condition holds, else if_false, by a mask of their bits: no branch, so a loop of it vec...
std::size_t checked_mul(std::size_t a, std::size_t b)
a * b, throwing numerical::dimension_error if the product overflows std::size_t. Guards dimension/sha...
Definition vector.hpp:135
std::vector< std::size_t > without_axis(std::vector< std::size_t > shape, std::size_t axis)
The shape with axis axis removed.
tensor< T > atan_viewed(const tensor< T > &a)
atan of every element of a: of a contiguous copy (atan_of).
C numpy_divide(const C &a, const C &b, bool fused)
a / b of complex numbers as NumPy's array loop computes it: Smith's division by the larger part of b ...
void(*)(const double *, double *, std::size_t) doubles_into
A function of count contiguous doubles into as many (the *_into kernels).
T root(const T &x)
The square root of a scalar, as std::sqrt gives it.
void expm1_into(const double *in, double *out, std::size_t count)
expm1 of the count doubles at in into out, each elementary_expm1's: table_expm1 in vectors (with the ...
@ zero
C = A·B: C's elements before the product are not read.
void select_positions(It first, std::size_t count, const std::vector< std::size_t > &kths, const Less &less)
Rearranges [first, first + count) by less so that each of the ascending kths holds the element a sort...
T elementary_log1p(const T &x)
log1p of a scalar, as std::log1p gives it.
tensor< T > tanh_of(const T *data, const std::vector< std::size_t > &shape)
tanh of every one of the row-major scalars at data, of shape shape: elementary_tanh of each.
std::vector< std::size_t > view_axes(const std::vector< std::size_t > &order)
The axes that put the elements of a tensor laid out in order (tensor::laid_out, its view's axes in th...
void cosh_into(const double *in, double *out, std::size_t count)
cosh of the count doubles at in into out, each elementary_cosh's: table_cosh in vectors (with the FMA...
std::size_t element_count(It first, It last)
The elements a shape's extents [first, last) hold: 0 where one of them is 0, whatever the others,...
Definition vector.hpp:151
auto as_laid_along(const tensor< T > &t, std::size_t axis, const F &f) -> std::optional< decltype(f(t, axis))>
Along axis of a transpose (laid_along): f (f(elements, axis)) of its elements as they lie along the m...
matrix< T > integer_power(const matrix< T > &a, double e)
a to the integer power e (a finite whole number) by repeated squaring, as numpy.linalg....
tensor< T > reduce_along(const tensor< T > &t, std::size_t axis, const Reduce &reduce)
Reduces each line of t along axis by reduce, which takes the line's first element,...
std::size_t first_held(const T *first, std::size_t count, const Match &match) noexcept
The index of the first of the contiguous elements at first that passes match, which one of them does:...
tensor< T > extremes_along(const tensor< T > &t, std::size_t axis, const Order &larger)
The largest (larger is std::greater) or smallest elements along axis. Along an inner axis each line i...
order_key_t< T > order_key(T x) noexcept
x's bits as an unsigned integer ordered as x is by <: the sign bit set for a positive real,...
std::optional< laid_axis< T > > laid_along(const tensor< T > &t, std::size_t axis)
t laid out and the axis matching axis: nothing for any tensor but a transpose, or an axis past the ra...
order_key_t< T > argsort_key(T x) noexcept
A real's order_key for an argsort: both zeros the same key, as < holds them equal (their indices then...
tensor< T > scan_along(const tensor< T > &t, std::size_t axis, const Step &step)
Each line of t along axis replaced by its running step (x₀, step(x₀, x₁), …), in order.
void sort_shared(T *run, std::size_t count, std::size_t threads)
Sorts one long line of count elements at run among threads: each pass of its radix sort shared out (r...
numpy_loops numpy_wheel_loops() noexcept
NumPy's loops as its current PyPI wheels build them on this processor (detail::numpy_fuses_products,...
std::complex< R > complex_cos(const std::complex< R > &z)
cos(z), as cosh(i z): complex_cosh of −im + i re.
std::vector< std::size_t > reduced_axes(const std::vector< std::size_t > &order, std::size_t axis)
The axes that put a result along axis of a tensor laid out in order (view_axes), that axis taken out,...
std::array< std::array< std::size_t, 256 >, sizeof(U)> byte_counts
How many keys of U hold each value of each byte: the byte at bit 8·b counted in [b].
tensor< std::int64_t > extreme_indices(const tensor< T > &t, std::size_t axis, const Order &larger)
argmax or argmin along axis by larger.
constexpr std::size_t pairwise_leaf
The longest subtree of pairwise_sum's tree parallel_pairwise_sum sums on one thread: 256 KiB of doubl...
Definition tensor.hpp:1066
T elementary_sinh(const T &x)
sinh of a scalar, as std::sinh gives it.
C::value_type numpy_absolute(const C &x, bool fused)
NumPy's complex magnitude (its absolute's vector loop): the parts' magnitudes, an infinite one making...
T ieee_minimum(T a, T b) noexcept
The smaller of a and b as IEEE 754-2019's minimum orders them: a NaN wins (a's where both are),...
T chunked_dot(const T *a, const T *b, std::size_t count, const Dot &dot)
Σ a[i] · b[i] for a plain dot product: chunks of dot_chunk elements in parallel, each by dot,...
Definition gemm.hpp:5066
std::int64_t * numbers_end(std::int64_t *first, std::int64_t *last, const T *values, std::size_t stride)
Where a real's NaNs begin among the indices [first, last), moved behind the rest in their order.
tensor< T > tanh_viewed(const tensor< T > &a)
tanh of every element of a: of a contiguous copy (tanh_of).
void order_run(std::int64_t *first, std::int64_t *last, const T *values, std::size_t stride, const Less &less, std::size_t parts=1)
Sorts the indices [first, last) stably by their elements (values[k · stride]), as sorts_before orders...
tensor< T > exp_of(const T *data, const std::vector< std::size_t > &shape)
exp of every one of the row-major elements at data, of shape shape: elementary_exp of each.
tensor< T > sinh_viewed(const tensor< T > &a)
sinh of every element of a: of a contiguous copy (sinh_of).
void order_shared(std::int64_t *order, std::size_t count, const T *values, std::size_t stride, const Less &less, std::size_t threads)
Sorts the indices of one long line among threads: each pass of its radix sort shared out.
tensor< C > numpy_cumprod(const tensor< C > &t, std::size_t axis, const numpy_loops &loops=numpy_wheel_loops())
The running products of a complex t along axis, in order, each as NumPy's loops compute it (detail::n...
@ fused
std::fma, where every target has the instruction.
constexpr std::size_t select_positions_sorted
A line with more positions than these is sorted whole rather than selected position by position.
R numpy_muladd(R x, R y, R z, bool fused)
x · y + z: one fused multiply-add where fused, else the product rounded first (statements apart,...
C numpy_reduction_multiply(const C &a, const C &b, const numpy_loops &loops)
a · b as NumPy's reduction loop computes it, fused as loops says.
The NumericAL public API.
Definition amx.hpp:44
std::vector< T, detail::default_init_allocator< T > > buffer
The storage of vector, matrix and tensor, namely a std::vector whose buffer<T>(n) and resize(n) leave...
Definition buffer.hpp:443
tensor< T > broadcast_to(const tensor< T > &t, std::vector< std::size_t > shape)
t repeated along its axes of extent 1 and new leading ones to shape, as numpy.broadcast_to (a copy: a...
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
std::array< tensor< std::int64_t >, 2 > triangle_indices(std::size_t rows, std::size_t cols, std::ptrdiff_t k, bool lower)
The row and the column indices of the rows × cols matrix's part to and below its k diagonal (lower) o...
matrix< T > operator+(const matrix< T > &a, const matrix< T > &b)
Element-wise sum.
Definition matrix.hpp:162
tensor< std::int64_t > argsort(const tensor< T > &t, std::size_t axis)
The indices that sort the elements along axis, as numpy.argsort (its kind="stable"): of each line,...
tensor< T > log1p(const tensor< T > &a)
log(1 + x) of every real element, exact near 0: detail::log1p_into of a double one (within 0....
tensor< T > select_along_last(const tensor< T > &t, const tensor< std::int64_t > &positions)
Of each line along the last axis of t, the elements its stable sort (sort) puts at the positions posi...
tensor< T > log(const tensor< T > &a)
log of every element: std::log of a real one, detail::complex_log of a complex one.
tensor< double > linspace(double start, double stop, std::size_t count)
count values evenly spaced from start to stop, both included, as NumPy's linspace: element i is start...
tensor< T > clip(const tensor< T > &a, T low, T high)
a limited to [low, high] element by element, as numpy.clip of two numbers: minimum(maximum(a,...
matrix< T > operator*(const T &scalar, const matrix< T > &m)
Scalar times matrix.
Definition matrix.hpp:188
tensor< T > exp(const tensor< T > &a)
exp of every element: detail::exp_into of a double one (within 0.506 ulp of the exact value),...
tensor< T > diag(const tensor< T > &v, std::ptrdiff_t k)
The square matrix with the vector v on its k diagonal (above the main one for a positive k) and zeros...
tensor< T > cosh(const tensor< T > &a)
cosh of every real element: detail::cosh_into of a double one (within 0.501 ulp of the exact value,...
tensor< T > operator/(const tensor< T > &a, const tensor< T > &b)
Element-wise quotient, broadcast.
tensor< std::int64_t > argpartition(const tensor< T > &t, const std::vector< std::size_t > &kths, std::size_t axis)
The indices that partition each line along axis, as numpy.argpartition: of the elements partition wou...
tensor< T > ones(std::vector< std::size_t > shape)
A tensor of shape full of ones.
tensor< T > correlate(const tensor< T > &a, const tensor< T > &v, std::ptrdiff_t first, std::size_t count)
The cross-correlation c[k] = Σₙ a[n + k] · conj(v[n]) of two vectors over the lags first … first + co...
tensor< T > triangle(const tensor< T > &t, std::ptrdiff_t k, bool lower)
The matrices of t (its last two axes) zero above their k diagonal (lower) or below it (else),...
tensor< T > take_along_axis(const tensor< T > &t, const tensor< std::int64_t > &indices, std::size_t axis)
The elements of t at indices along axis, the other axes matched, as numpy.take_along_axis: indices of...
tensor< std::int64_t > searchsorted(const tensor< T > &a, const tensor< T > &v, bool right=false)
Where each element of v would go in the sorted line a to keep it sorted, as numpy....
tensor< T > cumsum(const tensor< T > &t, std::size_t axis)
The running sums along axis, in order, as numpy.cumsum.
tensor< T > expm1(const tensor< T > &a)
exp(x) − 1 of every real element, exact near 0: detail::expm1_into of a double one (within 0....
tensor< T > maximum(const tensor< T > &a, const tensor< T > &b)
The larger element of a and b, broadcast, as numpy.maximum (detail::ieee_maximum; a NaN wins,...
tensor< T > take(const tensor< T > &t, const tensor< std::int64_t > &indices, std::size_t axis, take_mode mode)
The elements of t at indices along axis, as numpy.take: the result's shape t's with the axis replaced...
tensor< T > minimum(const tensor< T > &a, const tensor< T > &b)
The smaller element of a and b, broadcast, as numpy.minimum (detail::ieee_minimum; a NaN wins,...
T prod(const tensor< T > &t)
The product of every element, in order (numpy.prod's, whose multiplications are sequential: only its ...
gemm_path
The kernels a double matrix product can run on (a float one too on arm64), from the most portable....
Definition gemm.hpp:83
@ amx
The Apple AMX coprocessor, fused multiply-adds; the products it does not suit on the NEON path's tile...
T max(const tensor< T > &t)
The largest element (a NaN, if there is one), of a transpose's elements as they lie (in any order the...
auto broadcast_apply(const tensor< T > &a, const tensor< T > &b, const Op &op)
op(a, b) element by element, the operands broadcast against each other, in op's result type (a compar...
tensor< T > sqrt(const tensor< T > &a)
The square root of every element: std::sqrt of a real one, detail::complex_sqrt of a complex one.
std::vector< tensor< std::int64_t > > nonzero(const tensor< T > &t)
The indices of the nonzero elements (x != T {}: a NaN is one), one tensor per axis,...
tensor< T > exp2(const tensor< T > &a)
2^x of every real element: detail::exp2_into of a double one (within 0.506 ulp of the exact value whe...
tensor< T > cumprod(const tensor< T > &t, std::size_t axis)
The running products along axis, in order, as numpy.cumprod.
tensor< T > zeros(std::vector< std::size_t > shape)
A tensor of shape full of zeros (each thread's share zeroed as the C library zeroes).
matrix< T > operator-(const matrix< T > &a, const matrix< T > &b)
Element-wise difference.
Definition matrix.hpp:175
auto map(const tensor< T > &t, const F &f) -> tensor< std::decay_t< decltype(f(std::declval< const T & >()))> >
f(x) for every element, in a tensor of the same shape and of f's result type.
tensor< T > cos(const tensor< T > &a)
cos of every element: std::cos of a real one (a float the double's, detail::cos_of),...
tensor< double > arange(double start, double stop, double step, bool fused=true)
start, start + step, … below stop (above it for a negative step), as NumPy's arange: its length is ce...
tensor< T > eye(std::size_t n)
The n × n identity.
std::size_t count_nonzero(const tensor< T > &t)
The number of nonzero elements (x != T {}: a NaN is one), as numpy.count_nonzero without an axis: by ...
tensor< T > sinh(const tensor< T > &a)
sinh of every real element: detail::sinh_into of a double one (within 0.503 ulp of the exact value,...
tensor< T > round(const tensor< T > &a, int decimals=0)
Every element rounded to decimals places (tens, hundreds for a negative one), half to even,...
tensor< T > sort(const tensor< T > &t, std::size_t axis)
The elements sorted along axis, as numpy.sort: ascending, NaNs last, equal elements in their order (N...
tensor< std::int64_t > argsort_by(const tensor< std::int64_t > &order, const tensor< T > &key, std::size_t axis)
order, indices along axis into key of its shape, each line's sorted stably by the elements of key the...
tensor< T > atan(const tensor< T > &a)
atan of every real element: detail::atan_into of a double one (within 0.501 ulp of the exact value),...
tensor< T > reshape(const tensor< T > &t, std::vector< std::size_t > shape)
The same elements in the same order under another shape, shared with t (no copy).
tensor< T > diagonal(const tensor< T > &t, std::ptrdiff_t offset)
The offset diagonal of each matrix of t (its last two axes), as numpy.linalg.diagonal: above the main...
take_mode
What take does with an index outside its axis, as numpy.take's mode: refuse it, take it modulo the ex...
T mean(const tensor< T > &t)
The mean of every element. An empty tensor's mean is NaN, as NumPy's (which also warns).
tensor< T > tanh(const tensor< T > &a)
tanh of every element: detail::tanh_into of a double one (within 0.502 ulp of the exact value),...
tensor< T > floor_divide(const tensor< T > &a, const tensor< T > &b)
a // b element by element, broadcast, as numpy.floor_divide (detail::floor_divmod).
T min(const tensor< T > &t)
The smallest element (a NaN, if there is one), of a transpose's elements as they lie (in any order th...
std::int64_t argmax(const tensor< T > &t)
The flat index of the first largest element, as numpy.argmax (the first NaN where there is one).
std::int64_t argmin(const tensor< T > &t)
The flat index of the first smallest element, as numpy.argmin (the first NaN where there is one).
tensor< T > tan(const tensor< T > &a)
tan of every element: std::tan of a real one, detail::complex_tan of a complex one.
tensor< T > remainder(const tensor< T > &a, const tensor< T > &b)
a % b element by element, broadcast, as numpy.remainder (Python's: of b's sign; a real by zero fmod's...
T vdot(const tensor< T > &a, const tensor< T > &b)
Σ conj(a[i]) · b[i] over every element of a and b in their row-major order, as numpy....
tensor< T > sin(const tensor< T > &a)
sin of every element: std::sin of a real one (a float the double's, detail::sin_of),...
tensor< T > full(std::vector< std::size_t > shape, const T &value)
A tensor of shape with every element value.
tensor< T > partition(const tensor< T > &t, const std::vector< std::size_t > &kths, std::size_t axis)
The elements of each line along axis rearranged as numpy.partition: at each of kths the element sort ...
tensor< T > pow(const tensor< T > &a, const tensor< T > &b)
a to the power b element by element, broadcast, as std::pow; a single exponent of 2,...
The native vector of float and double the kernels hold their lanes in (GNU vector extensions,...
#define NUMERICAL_INLINE_LAMBDA
Definition simd.hpp:33
T operator()(T a, T b) const noexcept
T operator()(T a, T b) const noexcept
std::make_unsigned_t< decltype(T {}+0U)> unsigned_type
Wide enough that no promotion is signed.
The element operations of T the tensor operators apply: the standard ones.
std::multiplies< T > multiplies
a · b.
std::plus< T > plus
a + b.
std::minus< T > minus
a - b.
The extents and strides that walk a tensor of shape with axis axis taken out: outer positions before ...
std::size_t outer
The product of the extents before the axis.
std::size_t inner
The product of the extents after the axis.
std::size_t extent
The axis' extent.
A real of type R rounded to decimals places, half to even, as numpy.round: nearbyint(x · 10ᵈ) / 10ᵈ (...
decimal_rounding(int places) noexcept
C exact_from
The magnitude from which a value is its own rounding (decimals above 0).
C scale
10 to the magnitude of decimals, in C.
int decimals
The places; fewer than none round to tens, hundreds and so on.
A transpose laid out (tensor::laid_out) and the axis matching one of its view's.
tensor< T > elements
Its elements row-major as they lie.
std::vector< std::size_t > order
Its view's axes in the order they lie.
static double call(double x)
std::sin(x).
static double call(double x)
std::cos(x).
The C library's sin (Cosine false) or cos, call(x): one class each, so that each is covered whole.
Which of NumPy's complex loops fuse their multiply-adds: its product's vector loop,...
bool late_reductions
Where it fuses, as detail::numpy_multiply_late does (MSVC's contraction).
bool reductions
The product's reduction loop (detail::numpy_fuses_reductions).
bool scalar_products
The product's scalar loop.
bool quotients
The quotient (Smith's algorithm, a scalar loop).
bool products
The product's vector loop.
A *_into kernel of doubles as a kernel of floats: each piece of 512 converted to doubles on the stack...
doubles_into into
The kernel of doubles.
void operator()(const float *in, float *out, std::size_t count) const
The kernel of the count floats at in into out.
A dense, row-major, N-dimensional tensor and Einstein summation.