11#ifndef NUMERICAL_TENSOR_OPS_HPP
12#define NUMERICAL_TENSOR_OPS_HPP
56 if (
axis >= shape.size()) {
57 throw dimension_error(
"axis " + std::to_string(
axis) +
" is out of bounds for a tensor of rank "
58 + std::to_string(shape.size()));
60 const auto before {shape.begin() +
static_cast<std::ptrdiff_t
>(
axis)};
64 if (std::find(shape.begin(),
before, std::size_t {0}) !=
before || std::find(
before + 1, shape.end(), std::size_t {0}) != shape.end()) {
77 template <std::
floating_po
int R>
84 return std::fma(x,
y, z);
97 template <split_complex C>
102 using R =
typename C::value_type;
103 const R
ii {a.imag() * b.imag()};
104 const R ir {a.imag() * b.real()};
111 template <split_complex C>
115 using R =
typename C::value_type;
116 const R
ii {a.imag() * b.imag()};
117 const R
ri {a.real() * b.imag()};
118 return C {std::fma(a.real(), b.real(), -
ii), std::fma(a.imag(), b.real(),
ri)};
123 template <std::
floating_po
int R>
126 if (x == R {0} || std::isnan(x)) {
127 return std::numeric_limits<R>::quiet_NaN();
129 return std::copysign(std::numeric_limits<R>::infinity(), x);
137 template <split_complex C>
142 using R =
typename C::value_type;
149 const R
ratio {b.imag() / b.real()};
153 const R
ratio {b.real() / b.imag()};
163 template <split_complex C>
167 using R =
typename C::value_type;
168 if (std::abs(x.imag()) <= std::abs(x.real())) {
169 const R
ratio {x.imag() / x.real()};
171 return C {R {1} / d, -
ratio / d};
173 const R
ratio {x.real() / x.imag()};
175 return C {
ratio / d, R {-1} / d};
178#if defined(__APPLE__)
200 const std::complex<double>& b)
noexcept
208 const std::complex<float>& b)
noexcept
217 template <split_complex C>
221 return std::pow(a, b);
232 template <split_complex C>
237 using R =
typename C::value_type;
238 if (b.real() == R {0} && b.imag() == R {0}) {
239 return C {R {1}, R {0}};
241 if (a.real() == R {0} && a.imag() == R {0}) {
242 if (b.real() > R {0}) {
243 return C {R {0}, R {0}};
245 return C {std::numeric_limits<R>::quiet_NaN(), std::numeric_limits<R>::quiet_NaN()};
247 if (b.imag() != R {0} || !(b.real() > R {-100} && b.real() < R {100}) || std::trunc(b.real()) != b.real()) {
250 const auto n {
static_cast<int>(b.real())};
260 const auto magnitude {
static_cast<unsigned>(n < 0 ? -n : n)};
263 for (
unsigned bit {1U};;
bit <<= 1U) {
264 if ((magnitude &
bit) != 0
U) {
267 if (magnitude < (
bit << 1U)) {
279 template <
typename T>
282 return static_cast<T>(std::sqrt(x));
286 template <split_complex T>
299 template <
typename Into>
301 const std::vector<std::size_t>& shape,
304 const std::size_t count {
element_count(shape.begin(), shape.end())};
308 constexpr std::size_t
piece {512};
309 std::array<double, piece>
wide {};
310 std::array<double, piece>
image {};
313 const std::size_t size {std::min(
piece,
last - first)};
314 std::copy_n(data + first, size,
wide.begin());
316 std::transform(
image.begin(),
image.begin() +
static_cast<std::ptrdiff_t
>(size), out.data() + first,
317 [](
double y) { return static_cast<float>(y); });
324 template <
typename T>
327 return static_cast<T>(std::exp(x));
330#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
338# if defined(_MSC_VER) && !defined(__clang__)
351#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
369#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
371 std::transform(
in,
in + count, out, [](
double x) {
return std::exp(x); });
377#ifdef NUMERICAL_EXP_NEON
379 for (; i + 2 <= count; i += 2) {
383 for (std::size_t k {i}; k < i + 2; ++k) {
385 out[k] = std::exp(
in[k]);
395 for (std::size_t i {0}; i < count; ++i) {
399# if defined(__clang__)
400# pragma clang loop vectorize(disable)
402 for (std::size_t i {0}; i < count; ++i) {
404 out[i] = std::exp(
in[i]);
411 template <
typename T>
414 return static_cast<T>(std::expm1(x));
421#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
423 return std::expm1(x);
436#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
438 std::transform(
in,
in + count, out, [](
double x) {
return std::expm1(x); });
442#ifdef NUMERICAL_EXP_NEON
444 for (; i + 2 <= count; i += 2) {
452 for (std::size_t i {0}; i < count; ++i) {
460 template <
typename T>
462 const std::vector<std::size_t>& shape)
470 const std::vector<std::size_t>& shape)
472 const std::size_t count {
element_count(shape.begin(), shape.end())};
491 const std::vector<std::size_t>& shape)
497 template <
typename T>
500 return static_cast<T>(std::atan(x));
507#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
522#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
524 std::transform(
in,
in + count, out, [](
double x) {
return std::atan(x); });
528#ifdef NUMERICAL_EXP_NEON
530 for (; i + 2 <= count; i += 2) {
538 for (std::size_t i {0}; i < count; ++i) {
546 template <
typename T>
548 const std::vector<std::size_t>& shape)
556 const std::vector<std::size_t>& shape)
558 const std::size_t count {
element_count(shape.begin(), shape.end())};
577 const std::vector<std::size_t>& shape)
583 template <
typename T>
586 return static_cast<T>(std::sinh(x));
593#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
608#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
610 std::transform(
in,
in + count, out, [](
double x) {
return std::sinh(x); });
614#ifdef NUMERICAL_EXP_NEON
616 for (; i + 2 <= count; i += 2) {
624 for (std::size_t i {0}; i < count; ++i) {
632 template <
typename T>
634 const std::vector<std::size_t>& shape)
642 const std::vector<std::size_t>& shape)
644 const std::size_t count {
element_count(shape.begin(), shape.end())};
663 const std::vector<std::size_t>& shape)
669 template <
typename T>
672 return static_cast<T>(std::cosh(x));
679#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
694#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
696 std::transform(
in,
in + count, out, [](
double x) {
return std::cosh(x); });
700#ifdef NUMERICAL_EXP_NEON
702 for (; i + 2 <= count; i += 2) {
710 for (std::size_t i {0}; i < count; ++i) {
718 template <
typename T>
720 const std::vector<std::size_t>& shape)
728 const std::vector<std::size_t>& shape)
730 const std::size_t count {
element_count(shape.begin(), shape.end())};
749 const std::vector<std::size_t>& shape)
758 template <
typename T>
761 return static_cast<T>(std::exp2(x));
768#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
785#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
787 std::transform(
in,
in + count, out, [](
double x) {
return std::exp2(x); });
791#ifdef NUMERICAL_EXP_NEON
793 for (; i + 2 <= count; i += 2) {
797 for (std::size_t k {i}; k < i + 2; ++k) {
799 out[k] = std::exp2(
in[k]);
809 for (std::size_t i {0}; i < count; ++i) {
813# if defined(__clang__)
814# pragma clang loop vectorize(disable)
816 for (std::size_t i {0}; i < count; ++i) {
818 out[i] = std::exp2(
in[i]);
825 template <
typename T>
827 const std::vector<std::size_t>& shape)
835 const std::vector<std::size_t>& shape)
837 const std::size_t count {
element_count(shape.begin(), shape.end())};
848 template <split_complex T>
855 template <
typename T>
858 return static_cast<T>(std::log(x));
862 template <split_complex T>
869 template <
typename T>
872 return static_cast<T>(std::log1p(x));
879#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
881 return std::log1p(x);
894#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
896 std::transform(
in,
in + count, out, [](
double x) {
return std::log1p(x); });
900#ifdef NUMERICAL_EXP_NEON
902 for (; i + 2 <= count; i += 2) {
910 for (std::size_t i {0}; i < count; ++i) {
918 template <
typename T>
920 const std::vector<std::size_t>& shape)
928 const std::vector<std::size_t>& shape)
930 const std::size_t count {
element_count(shape.begin(), shape.end())};
949 const std::vector<std::size_t>& shape)
955 template <
typename T>
958 return static_cast<T>(std::sin(x));
962 template <split_complex T>
969 template <
typename T>
972 return static_cast<T>(std::cos(x));
976 template <split_complex T>
983 template <
bool Cosine>
1010 template <
bool Cosine>
1025 template <
bool Cosine>
1030#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
1038 for (std::size_t i {0}; i < count; ++i) {
1044 for (std::size_t i {0}; i < count; ++i) {
1055 template <
bool Cosine>
1058#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
1076 template <
bool Cosine>
1081#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
1088 for (std::size_t i {0}; i < count; ++i) {
1119 template <
typename T>
1121 const std::vector<std::size_t>& shape)
1129 const std::vector<std::size_t>& shape)
1137 const std::vector<std::size_t>& shape)
1139 const std::size_t count {
element_count(shape.begin(), shape.end())};
1149 template <
typename T>
1151 const std::vector<std::size_t>& shape)
1159 const std::vector<std::size_t>& shape)
1161 const std::size_t count {
element_count(shape.begin(), shape.end())};
1173 const std::vector<std::size_t>& shape)
1179 template <
typename T>
1182 return static_cast<T>(std::tan(x));
1186 template <split_complex T>
1193 template <
typename T>
1196 return static_cast<T>(std::tanh(x));
1200 template <split_complex T>
1210#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
1212 return std::tanh(x);
1225#if defined(NUMERICAL_X86_COMPILED) || (defined(_MSC_VER) && !defined(__clang__))
1227 std::transform(
in,
in + count, out, [](
double x) {
return std::tanh(x); });
1231#ifdef NUMERICAL_EXP_NEON
1233 for (; i + 2 <= count; i += 2) {
1241 for (std::size_t i {0}; i < count; ++i) {
1249 template <
typename T>
1251 const std::vector<std::size_t>& shape)
1259 const std::vector<std::size_t>& shape)
1261 const std::size_t count {
element_count(shape.begin(), shape.end())};
1280 const std::vector<std::size_t>& shape)
1291 template <split_complex C>
1295 using R =
typename C::value_type;
1296 constexpr R
inf {std::numeric_limits<R>::infinity()};
1297 constexpr R
nan {std::numeric_limits<R>::quiet_NaN()};
1298 R re {std::abs(x.real())};
1299 R im {std::abs(x.imag())};
1308 const bool re_nan {std::isnan(re)};
1309 const bool im_nan {std::isnan(im)};
1316 const R larger {std::max(re, im)};
1317 const R smaller {std::min(im, re)};
1319 if (larger != R {0} && smaller !=
inf) {
1320 ratio = smaller / larger;
1329#if defined(__aarch64__) || defined(_M_ARM64)
1331#elif defined(NUMERICAL_X86_COMPILED)
1343#if defined(__APPLE__) || defined(_WIN32)
1364#if defined(__APPLE__) && defined(__aarch64__)
1378 const std::vector<std::size_t>&
b_shape)
noexcept
1380 const auto ones {[](
const std::vector<std::size_t>& shape) {
return std::ranges::all_of(shape, [](std::size_t e) {
return e == 1; }); }};
1381 const std::size_t rank {std::max(
a_shape.size(),
b_shape.size())};
1382 const auto broadcast {[rank](
const std::vector<std::size_t>& shape) {
return !shape.empty() && shape.size() != rank; }};
1390 template <
typename T>
1402 template <std::
floating_po
int T>
1406 using wide = std::common_type_t<T, double>;
1407 return static_cast<T>(
static_cast<wide>(value) /
static_cast<wide>(n));
1415 template <split_complex T>
1419 using R =
typename T::value_type;
1420 const std::complex<double>
wide {
static_cast<double>(value.real()),
static_cast<double>(value.imag())};
1422 const double scale {1.0 /
static_cast<double>(n)};
1423 const double zero {0.0};
1424 return T {
static_cast<R
>((
wide.real() + (
wide.imag() *
zero)) * scale),
static_cast<R
>((
wide.imag() - (
wide.real() *
zero)) * scale)};
1429 template <
typename T>
1434 throw dimension_error(
"a mean over no element has no value in an integer type");
1436 return value /
static_cast<T>(n);
1445 template <std::floating_point
T,
1451 const Order& larger)
1453 if (value !=
T {0}) {
1456 const bool maximum {larger(
T {1},
T {0})};
1457 for (std::size_t i {0}; i < count; ++i) {
1458 const T x {first[i *
stride]};
1459 if (x ==
T {0} && std::signbit(x) !=
maximum) {
1467 template <
typename T,
1482 shape.erase(shape.begin() +
static_cast<std::ptrdiff_t
>(
axis));
1490 template <
typename T,
1499 const std::size_t o {at / walk.inner};
1500 const std::size_t i {at %
walk.inner};
1515 template <
typename R,
1525 const T*
const in {t.
data().data()};
1526 if (
walk.inner == 1) {
1529 for (std::size_t k {0}; k <
walk.extent; ++k) {
1536 const std::size_t
bands {(
walk.inner + band_columns - 1) / band_columns};
1537 parallel_for(
walk.outer *
bands,
walk.extent * band_columns, [&](std::size_t at) {
1538 const std::size_t o {at / bands};
1539 const std::size_t first {(at %
bands) * band_columns};
1540 const std::size_t width {std::min(band_columns,
walk.inner - first)};
1541 const std::size_t
row {(
o *
walk.inner) + first};
1542 for (std::size_t w {0}; w < width; ++w) {
1545 for (std::size_t k {0}; k <
walk.extent; ++k) {
1546 const T*
const x {
in + (((
o *
walk.extent) + k) *
walk.inner) + first};
1547 for (std::size_t w {0}; w < width; ++w) {
1552 return tensor<R> {
without_axis(t.shape(), axis), std::move(out)};
1556 template <
typename T>
1559 return static_cast<T
>(x * x);
1563 template <
typename T>
1572 return std::isnan(x);
1575#if defined(__GNUC__)
1577 template <
typename V,
1579 [[nodiscard]]
auto better(
const V& x,
1581 const std::greater<T>& )
1587 template <
typename V,
1589 [[nodiscard]]
auto better(
const V& x,
1591 const std::less<T>& )
1597 template <
typename V>
1598 [[nodiscard]]
auto unordered(
const V& a,
1606#ifdef NUMERICAL_NEON_COMPILED
1608 [[nodiscard]]
inline float64x2_t lane_extreme(float64x2_t x,
1610 const std::greater<double>& )
noexcept
1612 return vmaxq_f64(x, y);
1616 [[nodiscard]]
inline float64x2_t lane_extreme(float64x2_t x,
1618 const std::less<double>& )
noexcept
1620 return vminq_f64(x, y);
1624 [[nodiscard]]
inline float32x4_t lane_extreme(float32x4_t x,
1626 const std::greater<float>& )
noexcept
1628 return vmaxq_f32(x, y);
1632 [[nodiscard]]
inline float32x4_t lane_extreme(float32x4_t x,
1634 const std::less<float>& )
noexcept
1636 return vminq_f32(x, y);
1640 [[nodiscard]]
inline double across_lanes(float64x2_t x,
1641 const std::greater<double>& )
noexcept
1643 return vmaxvq_f64(x);
1647 [[nodiscard]]
inline double across_lanes(float64x2_t x,
1648 const std::less<double>& )
noexcept
1650 return vminvq_f64(x);
1654 [[nodiscard]]
inline float across_lanes(float32x4_t x,
1655 const std::greater<float>& )
noexcept
1657 return vmaxvq_f32(x);
1661 [[nodiscard]]
inline float across_lanes(float32x4_t x,
1662 const std::less<float>& )
noexcept
1664 return vminvq_f32(x);
1674 template <
typename T,
1676 [[nodiscard]] std::size_t neon_extreme(
const T* first,
1680 const Order& larger)
noexcept
1682 constexpr std::size_t width {16 /
sizeof(T)};
1683 constexpr std::size_t block {4 * width};
1684 const std::size_t done {count - (count % block)};
1688 auto b0 {neon_load(first)};
1689 auto b1 {neon_load(first + width)};
1690 auto b2 {neon_load(first + (2 * width))};
1691 auto b3 {neon_load(first + (3 * width))};
1692 for (std::size_t i {block}; i < done; i += block) {
1693 b0 = lane_extreme(b0, neon_load(first + i), larger);
1694 b1 = lane_extreme(b1, neon_load(first + i + width), larger);
1695 b2 = lane_extreme(b2, neon_load(first + i + (2 * width)), larger);
1696 b3 = lane_extreme(b3, neon_load(first + i + (3 * width)), larger);
1698 const T value {across_lanes(lane_extreme(lane_extreme(b0, b1, larger), lane_extreme(b2, b3, larger), larger), larger)};
1699 nan = std::isnan(value);
1700 result = larger(value, result) ? value : result;
1711 template <
typename T,
1715 const Order& larger)
1717 T result {first[0]};
1719 std::size_t done {0};
1720#if defined(NUMERICAL_NEON_COMPILED)
1721 done = neon_extreme(first, count, result, nan, larger);
1723# ifdef NUMERICAL_X86_COMPILED
1725 if constexpr (std::is_same_v<T, double> || std::is_same_v<T, float>) {
1729 done = avx2_extreme<std::is_same_v<Order, std::greater<T>>>(first, count, result, nan);
1734# if defined(__GNUC__)
1735 using lane =
typename native_vector<T>::type;
1736 constexpr std::size_t width {native_vector_bytes /
sizeof(T)};
1737 constexpr std::size_t lanes {4};
1738# ifdef NUMERICAL_X86_COMPILED
1739 if (!wide && count >= width * lanes) {
1741 if (count >= width * lanes) {
1743 using mask =
decltype(lane {} > lane {});
1744 std::array<lane, lanes> best {};
1745 std::array<mask, lanes> nans {};
1746 for (std::size_t l {0}; l < lanes; ++l) {
1747 std::memcpy(&best[l], first + (l * width),
sizeof(lane));
1749 done = count - (count % (width * lanes));
1750 for (std::size_t i {0}; i < done; i += width * lanes) {
1751# pragma GCC unroll 4
1752 for (std::size_t l {0}; l < lanes; ++l) {
1754 std::memcpy(&x, first + i + (l * width),
sizeof(lane));
1757 const mask
take {better(x, best[l], larger)};
1758 best[l] = std::bit_cast<lane>((std::bit_cast<mask>(x) &
take) | (std::bit_cast<mask>(best[l]) & ~
take));
1759 nans[l] |= unordered(x, x);
1762 for (std::size_t l {0}; l < lanes; ++l) {
1763 for (std::size_t k {0}; k < width; ++k) {
1764 nan = nan || nans[l][k] != 0;
1765 result = larger(best[l][k], result) ? best[l][k] : result;
1771 std::array<T, 8> best {};
1772 std::array<unsigned, 8> nans {};
1774 for (std::size_t i {done}; i < count; ++i) {
1775 const T x {first[i]};
1776 nans[i % 8] |=
static_cast<unsigned>(std::isnan(x));
1777 best[i % 8] = larger(x, best[i % 8]) ? x : best[i % 8];
1779 for (std::size_t l {0}; l < 8; ++l) {
1780 nan = nan || nans[l] != 0U;
1781 result = larger(best[l], result) ? best[l] : result;
1783 return nan ? std::numeric_limits<T>::quiet_NaN() : result;
1791 template <
typename T,
1795 const Order& larger)
1811 template <
typename T,
1816 const Order& larger)
1819 for (std::size_t i {1}; i < count; ++i) {
1820 const T& candidate {first[i * stride]};
1821 if (
is_nan(candidate) || larger(candidate, best)) {
1825 return settled_zero(best, first, count, stride, larger);
1831 template <std::floating_point T,
1836 const Order& larger)
1839 throw dimension_error(
"a zero-size reduction to a maximum or a minimum has no identity");
1849 template <
typename T,
1851 requires(!std::floating_point<T>)
1855 const Order& larger)
1858 throw dimension_error(
"a zero-size reduction to a maximum or a minimum has no identity");
1866 template <
typename T>
1869 return !std::is_floating_point_v<T> || inner == 1;
1877 template <
typename T,
1881 const Order& larger)
1885 if (walk.extent == 0) {
1886 throw dimension_error(
"a zero-size reduction to a maximum or a minimum has no identity");
1888 if (by_reduction<T>(walk.inner)) {
1889 return reduce_along(t, axis, [&larger](
const T* first, std::size_t count, std::size_t stride) {
1890 return extreme(first, count, stride, larger);
1897 const std::size_t inner {walk.inner};
1898 const std::size_t extent {walk.extent};
1899 const T*
const data {t.
data().data()};
1900 T*
const into {out.data()};
1901 parallel_for(walk.outer, extent * inner, [ =, &larger](std::size_t o) {
1902 const T* const slab {data + (o * extent * inner)};
1903 T*
const best {into + (o * inner)};
1904 std::vector<unsigned> nan(inner, 0U);
1905 std::copy(slab, slab + inner, best);
1906 for (std::size_t e {0}; e < extent; ++e) {
1907 const T*
const row {slab + (e * inner)};
1908 for (std::size_t i {0}; i < inner; ++i) {
1910 nan[i] |=
static_cast<unsigned>(std::isnan(x));
1911 best[i] = larger(x, best[i]) ? x : best[i];
1914 for (std::size_t i {0}; i < inner; ++i) {
1915 best[i] =
settled_zero(best[i], slab + i, extent, inner, larger);
1917 best[i] = std::numeric_limits<T>::quiet_NaN();
1921 return tensor<T> {without_axis(t.shape(), axis), std::move(out)};
1929 [[nodiscard]]
inline std::vector<std::size_t>
view_axes(
const std::vector<std::size_t>& order)
1931 std::vector<std::size_t> back(order.size());
1932 for (std::size_t k {0}; k < order.size(); ++k) {
1940 [[nodiscard]]
inline std::vector<std::size_t>
reduced_axes(
const std::vector<std::size_t>& order,
1943 std::vector<std::size_t> rest;
1944 for (
const std::size_t view_axis : order) {
1945 if (view_axis != axis) {
1946 rest.push_back(view_axis > axis ? view_axis - 1 : view_axis);
1953 template <
typename T>
1958 std::size_t along {0};
1963 template <
typename T>
1968 if (!laid || axis >= t.
rank()) {
1969 return std::nullopt;
1971 const auto along {
static_cast<std::size_t
>(std::ranges::find(laid->second, axis) - laid->second.begin())};
1972 return laid_axis<T> {std::move(laid->first), std::move(laid->second), along};
1984 template <
typename T,
1988 const F& f) -> std::optional<
decltype(f(t, axis))>
1991 if (!laid || (axis + 1 == t.rank() && laid->along + 1 != t.rank())) {
1992 return std::nullopt;
1994 return f(laid->elements, laid->along).transposed(
view_axes(laid->order));
1999 template <
typename T,
2003 const F& f) -> std::optional<
decltype(f(t, axis))>
2007 return std::nullopt;
2009 return f(laid->elements, laid->along).transposed(
reduced_axes(laid->order, axis));
2018 template <
typename T>
2021 const std::size_t rank {t.shape().size()};
2022 const std::ptrdiff_t step {t.strides()[rank - 1]};
2023 const auto span {
static_cast<std::size_t
>(step < 0 ? -step : step) *
sizeof(T)};
2024 return t.shape()[rank - 1] < 2 ? rank == 1 : span <= 64;
2031 template <
typename T,
2034 const std::vector<std::size_t>& shape,
2037 using R = std::decay_t<decltype(f(std::declval<const T&>()))>;
2038 const std::size_t count {
element_count(shape.begin(), shape.end())};
2044 std::transform(data + first, data + last, out.begin() +
static_cast<std::ptrdiff_t
>(first), f);
2046 return tensor<R> {shape, std::move(out)};
2053 template <
typename T,
2058 if (t.contiguous()) {
2062 if (
const auto laid {t.laid_out()}) {
2063 return parallel_map(laid->first.data().data(), laid->first.shape(), f).transposed(
view_axes(laid->second));
2067 using R = std::decay_t<decltype(f(std::declval<const T&>()))>;
2068 const std::vector<std::size_t>& shape {t.shape()};
2069 const std::vector<std::ptrdiff_t>& strides {t.strides()};
2070 const std::size_t rank {shape.size()};
2071 const std::size_t count {t.size()};
2072 const std::size_t inner {shape[rank - 1]};
2073 const std::ptrdiff_t step {strides[rank - 1]};
2074 const T*
const origin {t.origin()};
2080 std::vector<std::size_t> index(rank, 0);
2081 std::ptrdiff_t at {0};
2082 std::size_t column {first % inner};
2083 for (std::size_t axis {rank - 1}, rest {first / inner}; axis-- > 0;) {
2084 index[axis] = rest % shape[axis];
2085 rest /= shape[axis];
2086 at +=
static_cast<std::ptrdiff_t
>(index[axis]) * strides[axis];
2088 for (std::size_t done {first}; done < last;) {
2089 const std::size_t n {std::min(inner - column, last - done)};
2090 const T*
const p {origin + at + (
static_cast<std::ptrdiff_t
>(column) * step)};
2091 R*
const o {out.data() + done};
2093 std::transform(p, p + n, o, f);
2096 for (std::size_t i {0}; i < n; ++i) {
2097 o[i] = f(p[
static_cast<std::ptrdiff_t
>(i) * step]);
2102 for (std::size_t axis {rank - 1}; axis-- > 0;) {
2104 at += strides[axis];
2105 if (index[axis] < shape[axis]) {
2108 at -= strides[axis] *
static_cast<std::ptrdiff_t
>(shape[axis]);
2113 return tensor<R> {shape, std::move(out)};
2123 template <
typename T,
2126 const std::vector<std::size_t>& a_shape,
2128 const std::vector<std::size_t>& b_shape,
2131 if (a_shape == b_shape) {
2134 if (
element_count(b_shape.begin(), b_shape.end()) == 1 && b_shape.size() <= a_shape.size()) {
2135 return parallel_map(a, a_shape, [&op, y = *b](
const T& x) {
return op(x, y); });
2137 if (
element_count(a_shape.begin(), a_shape.end()) == 1 && a_shape.size() <= b_shape.size()) {
2138 return parallel_map(b, b_shape, [&op, x = *a](
const T& y) {
return op(x, y); });
2151 template <
typename T,
2161 return detail::parallel_map(a, [&op, y = *b.
origin()](
const T& x) { return op(x, y); });
2164 return detail::parallel_map(b, [&op, x = *a.
origin()](
const T& y) { return op(x, y); });
2169 if (a_laid && b_laid && a_laid->second == b_laid->second) {
2170 return detail::pairs_into(a_laid->first.origin(), b_laid->first.origin(), a_laid->first.shape(), op)
2171 .transposed(detail::view_axes(a_laid->second));
2174 const bool a_close {a.
contiguous() || detail::lies_close(a)};
2175 const bool b_close {b.
contiguous() || detail::lies_close(b)};
2176 if (a_close && b_close) {
2179 return detail::broadcast_into(a_close ? a.
origin() : a.
data().data(), a.
shape(),
2180 a_close ? a.
strides() : detail::row_major_strides(a.
shape()),
2182 b_close ? b.
strides() : detail::row_major_strides(b.
shape()), op);
2201 template <split_complex C>
2215 .scalar_products = detail::numpy_fuses_scalar_products(),
2216 .quotients = detail::numpy_fuses_quotients(),
2217 .reductions = detail::numpy_fuses_reductions(),
2218 .late_reductions =
false};
2226 template <detail::split_complex C>
2231 const bool fused {detail::numpy_loop_is_scalar(a, b) ? loops.scalar_products : loops.products};
2237 template <detail::split_complex C>
2242 const bool fused {loops.quotients};
2253 template <detail::split_complex C>
2260 const auto multiply {[reduction, &loops](
const C& product,
const C& x) {
2264 return fold_along(elements, along, C {1}, multiply);
2266 return std::move(*laid);
2273 template <detail::split_complex C>
2278 for (
const C& x : t.
data()) {
2286 template <detail::split_complex C>
2290 const bool fused {loops.products};
2291 return detail::parallel_map(a, [
fused](
const C& x) {
return detail::numpy_absolute(x,
fused); });
2296 template <detail::split_complex C>
2300 const bool fused {loops.products};
2301 return detail::parallel_map(a, [
fused](
const C& x) {
return detail::numpy_multiply(x, x,
fused); });
2306 template <detail::split_complex C>
2310 const bool fused {loops.quotients};
2311 return detail::parallel_map(a, [
fused](
const C& x) {
return detail::numpy_reciprocal(x,
fused); });
2316 template <detail::split_complex C>
2321 const bool fused {loops.quotients};
2328 template <
typename T>
2330 const std::vector<std::size_t>& shape)
2338 const std::vector<std::size_t>& shape)
2340 const std::size_t count {
element_count(shape.begin(), shape.end())};
2362 std::size_t count)
const
2364 constexpr std::size_t piece {512};
2365 std::array<double, piece> wide {};
2366 std::array<double, piece> image {};
2367 for (std::size_t first {0}; first < count; first += piece) {
2368 const std::size_t size {std::min(piece, count - first)};
2369 std::copy_n(in + first, size, wide.begin());
2370 into(wide.data(), image.data(), size);
2371 std::transform(image.begin(), image.begin() +
static_cast<std::ptrdiff_t
>(size), out + first, [](
double y) { return static_cast<float>(y); });
2384 template <
typename T,
2387 const Kernel& kernel)
2389 const std::vector<std::size_t>& shape {t.
shape()};
2390 const std::size_t count {t.
size()};
2395 if (
const auto laid {t.
laid_out()}) {
2403 kernel(data + first, out.data() + first, std::min(
elementwise_chunk, count - first));
2405 return tensor<T> {shape, std::move(out)};
2407 const std::vector<std::ptrdiff_t>& strides {t.
strides()};
2408 const std::size_t rank {shape.size()};
2409 const std::size_t inner {shape[rank - 1]};
2410 const std::ptrdiff_t step {strides[rank - 1]};
2411 const T*
const origin {t.
origin()};
2413 constexpr std::size_t piece {512};
2414 std::array<T, piece> gathered {};
2417 std::vector<std::size_t> index(rank, 0);
2418 std::ptrdiff_t at {0};
2419 std::size_t column {first % inner};
2420 for (std::size_t axis {rank - 1}, rest {first / inner}; axis-- > 0;) {
2421 index[axis] = rest % shape[axis];
2422 rest /= shape[axis];
2423 at +=
static_cast<std::ptrdiff_t
>(index[axis]) * strides[axis];
2425 for (std::size_t done {first}; done < last;) {
2426 const std::size_t n {std::min(inner - column, last - done)};
2427 const T*
const p {origin + at + (
static_cast<std::ptrdiff_t
>(column) * step)};
2429 kernel(p, out.data() + done, n);
2432 for (std::size_t k {0}; k < n; k += piece) {
2433 const std::size_t size {std::min(piece, n - k)};
2434 for (std::size_t i {0}; i < size; ++i) {
2435 gathered[i] = p[
static_cast<std::ptrdiff_t
>(k + i) * step];
2437 kernel(gathered.data(), out.data() + done + k, size);
2442 for (std::size_t axis {rank - 1}; axis-- > 0;) {
2444 at += strides[axis];
2445 if (index[axis] < shape[axis]) {
2448 at -= strides[axis] *
static_cast<std::ptrdiff_t
>(shape[axis]);
2453 return tensor<T> {shape, std::move(out)};
2471 template <
typename T>
2484 template <
typename T>
2503 template <
typename T>
2522 template <
typename T>
2541 template <
typename T>
2560 template <
typename T>
2573 template <
typename T>
2592 template <
typename T>
2601 return by_runs(a, &double_sines_into<false>);
2607 return by_runs(a, &float_sines_into<false>);
2611 template <
typename T>
2620 return by_runs(a, &double_sines_into<true>);
2626 return by_runs(a, &float_sines_into<true>);
2630 template <
typename T>
2653 template <
typename T>
2656 return detail::exp_viewed(a);
2663 template <std::
floating_po
int T>
2666 return detail::expm1_viewed(a);
2673 template <std::
floating_po
int T>
2676 return detail::atan_viewed(a);
2683 template <std::
floating_po
int T>
2686 return detail::sinh_viewed(a);
2693 template <std::
floating_po
int T>
2696 return detail::cosh_viewed(a);
2703 template <std::
floating_po
int T>
2706 return detail::exp2_viewed(a);
2710 template <
typename T>
2713 return detail::parallel_map(a, [](
const T& x) {
return detail::elementary_log(x); });
2720 template <std::
floating_po
int T>
2723 return detail::log1p_viewed(a);
2728 template <
typename T>
2731 return detail::sin_viewed(a);
2736 template <
typename T>
2739 return detail::cos_viewed(a);
2743 template <
typename T>
2746 return detail::parallel_map(a, [](
const T& x) {
return detail::elementary_tan(x); });
2753 template <
typename T>
2756 return detail::tanh_viewed(a);
2760 template <
typename T>
2763 return detail::parallel_map(a, [](
const T& x) {
return detail::root(x); });
2768 template <
typename T>
2778 template <std::
signed_
integral T>
2821 template <
typename T>
2832 template <std::
floating_po
int T>
2836 if (std::isnan(a) || std::isnan(b)) {
2837 return std::isnan(a) ? a : b;
2840 return a == T {0} ? a + b : a;
2842 return a > b ? a : b;
2847 template <std::
floating_po
int T>
2851 if (std::isnan(a) || std::isnan(b)) {
2852 return std::isnan(a) ? a : b;
2855 return a == T {0} ? -(-a - b) : a;
2857 return a < b ? a : b;
2862 template <std::
floating_po
int T>
2865 T if_false)
noexcept
2867 using bits = std::conditional_t<
sizeof(T) == 8, std::uint64_t, std::uint32_t>;
2868 static_assert(
sizeof(T) ==
sizeof(bits),
"a float or a double");
2869 const bits mask {bits {0} -
static_cast<bits
>(condition)};
2870 return std::bit_cast<T>((std::bit_cast<bits>(if_true) & mask) | (std::bit_cast<bits>(if_false) & ~mask));
2875 template <std::
floating_po
int T>
2880 const T up {
masked_select((
static_cast<unsigned>(x == low) &
static_cast<unsigned>(x == T {0})) != 0U, x + low,
masked_select(x < low, low, x))};
2881 return masked_select((
static_cast<unsigned>(up == high) &
static_cast<unsigned>(up == T {0})) != 0U, -(-up - high),
masked_select(up > high, high, up));
2887 template <std::
floating_po
int T>
2891 if (b.
shape().empty()) {
2892 const T bound {b.
data()[0]};
2893 return detail::parallel_map(a, [bound](T x) {
return detail::ieee_maximum(x, bound); });
2895 return broadcast_apply(a, b, [](T x, T y) {
return detail::ieee_maximum(x, y); });
2900 template <std::
floating_po
int T>
2904 if (b.
shape().empty()) {
2905 const T bound {b.
data()[0]};
2906 return detail::parallel_map(a, [bound](T x) {
return detail::ieee_minimum(x, bound); });
2908 return broadcast_apply(a, b, [](T x, T y) {
return detail::ieee_minimum(x, y); });
2921 template <std::floating_point R,
2922 std::floating_point C>
2930 , scale {std::pow(C {10},
static_cast<C
>(places < 0 ? -places : places))}
2931 , exact_from {std::ldexp(C {1}, std::numeric_limits<R>::digits) / scale}
2936 const C wide {
static_cast<C
>(x)};
2938 return std::fabs(wide) < exact_from ?
static_cast<R
>(std::nearbyint(wide * scale) / scale) : x;
2940 if (std::isinf(scale)) {
2941 return std::isfinite(x) ? std::copysign(R {0}, x) : x;
2943 return static_cast<R
>(std::nearbyint(wide / scale) * scale);
2949 template <std::floating_point R,
2951 requires(!std::same_as<R, float>)
2959 template <std::same_as<
float> R,
2964 if (decimals > std::numeric_limits<float>::max_exponent10) {
2973 template <std::
floating_po
int T>
2977 return detail::with_decimal_rounding<T>(decimals, [&a](
const auto& rounding) {
return detail::parallel_map(a, rounding); });
2982 template <detail::split_complex C>
2986 return detail::with_decimal_rounding<typename C::value_type>(decimals, [&a](
const auto& rounding) {
2987 return detail::parallel_map(a, [&rounding](
const C& z) {
return C {rounding(z.real()), rounding(z.imag())}; });
2994 template <std::
floating_po
int T>
2999 if (std::isnan(low) || std::isnan(high)) {
3000 return detail::parallel_map(a, [low, high](T x) {
return detail::ieee_minimum(detail::ieee_maximum(x, low), high); });
3002 return detail::parallel_map(a, [low, high](T x) {
return detail::clipped(x, low, high); });
3007 template <std::
integral T>
3008 [[nodiscard]] tensor<T> clip(
const tensor<T>& a,
3012 return detail::parallel_map(a, [low, high](T x) {
return std::min(std::max(x, low), high); });
3017 template <std::
integral T>
3018 [[nodiscard]] tensor<T>
maximum(
const tensor<T>& a,
3021 return broadcast_apply(a, b, [](T x, T y) {
return std::max(x, y); });
3026 template <std::
integral T>
3027 [[nodiscard]] tensor<T>
minimum(
const tensor<T>& a,
3030 return broadcast_apply(a, b, [](T x, T y) {
return std::min(x, y); });
3034 template <
typename T>
3042 template <
typename T>
3050 template <
typename T>
3063 template <std::
floating_po
int T>
3067 T mod {std::fmod(a, b)};
3069 return {a / b, mod};
3071 T quotient {(a - mod) / b};
3073 if ((b < T {0}) != (mod < T {0})) {
3079 mod = std::copysign(T {0}, b);
3081 if (quotient == T {0}) {
3082 return {std::copysign(T {0}, a / b), mod};
3084 T floored {std::floor(quotient)};
3085 if (quotient - floored > T {0.5}) {
3088 return {floored, mod};
3096 template <std::
signed_
integral T>
3097 [[nodiscard]] std::pair<T, T> floor_divmod(T a,
3101 return {T {0}, T {0}};
3104 return {
typename arithmetic<T>::negate {}(a), T {0}};
3106 T quotient {
static_cast<T
>(a / b)};
3107 T mod {
static_cast<T
>(a % b)};
3108 if (mod != T {0} && ((mod < T {0}) != (b < T {0}))) {
3109 quotient =
static_cast<T
>(quotient - T {1});
3110 mod =
static_cast<T
>(mod + b);
3112 return {quotient, mod};
3118 template <
typename T>
3122 return broadcast_apply(a, b, [](T x, T y) {
return detail::floor_divmod(x, y).first; });
3127 template <
typename T>
3131 return broadcast_apply(a, b, [](T x, T y) {
return detail::floor_divmod(x, y).second; });
3135 template <
typename T>
3142 template <
typename T,
3147 using R = std::decay_t<decltype(f(std::declval<const T&>()))>;
3149 std::transform(t.
data().begin(), t.
data().end(), out.begin(), f);
3159 template <
typename T>
3163 if constexpr (!std::is_integral_v<T>) {
3165 const T y {b.
data()[0]};
3167 return detail::parallel_map(a, [](
const T& x) {
return detail::square(x); });
3170 return detail::parallel_map(a, [](
const T& x) {
return detail::root(x); });
3173 return detail::parallel_map(a, [](
const T& x) {
return static_cast<T
>(T(1) / x); });
3177 return broadcast_apply(a, b, [](
const T& x,
const T& y) {
return static_cast<T
>(std::pow(x, y)); });
3183 template <std::
signed_
integral T>
3185 T exponent)
noexcept
3189 while (exponent != 0) {
3190 if ((exponent & 1) != 0) {
3191 result =
times(result, base);
3193 base =
times(base, base);
3216 template <std::
signed_
integral T>
3217 [[nodiscard]] tensor<T> pow(
const tensor<T>& a,
3220 if (std::ranges::any_of(b.data(), [](T exponent) { return exponent < 0; })) {
3221 throw negative_power_error(
"Integers to negative integer powers are not allowed.");
3223 return broadcast_apply(a, b, [](T x, T y) {
return detail::integer_power(x, y); });
3228 template <
typename T>
3231 return detail::parallel_pairwise_sum(t.
data().data(), t.
size());
3236 template <
typename T>
3240 if (
auto laid {detail::as_laid_reduced(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return sum(elements, along); })}) {
3241 return std::move(*laid);
3245 detail::pairwise_sums(t.
data().data(), walk.outer, walk.extent, walk.inner, out.data());
3246 return tensor<T> {detail::without_axis(t.
shape(), axis), std::move(out)};
3252 template <std::
signed_
integral T>
3253 [[nodiscard]] T sum(
const tensor<T>& t)
3255 if (
const auto laid {t.laid_out()}) {
3256 return sum(laid->first);
3258 using unsigned_type =
typename detail::arithmetic<T>::unsigned_type;
3259 const T*
const data {t.data().data()};
3260 const std::size_t count {t.size()};
3261 const std::size_t chunks {(count + detail::elementwise_chunk - 1) / detail::elementwise_chunk};
3262 std::vector<unsigned_type> partial(chunks, 0);
3263 detail::parallel_for(chunks, detail::elementwise_chunk, [&](std::size_t c) {
3264 const std::size_t last {std::min((c + 1) * detail::elementwise_chunk, count)};
3265 unsigned_type total {0};
3266 for (std::size_t i {c * detail::elementwise_chunk}; i < last; ++i) {
3267 total +=
static_cast<unsigned_type
>(data[i]);
3271 unsigned_type total {0};
3272 for (
const unsigned_type part : partial) {
3275 return static_cast<T
>(total);
3280 template <std::
signed_
integral T>
3281 [[nodiscard]] tensor<T>
sum(
const tensor<T>& t,
3284 if (
auto laid {detail::as_laid_reduced(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return sum(elements, along); })}) {
3285 return std::move(*laid);
3287 return detail::reduce_along(t, axis, [](
const T* first, std::size_t count, std::size_t stride) {
3288 const typename detail::arithmetic<T>::plus plus {};
3290 for (std::size_t i {0}; i < count; ++i) {
3291 total = plus(total, first[i * stride]);
3300 template <
typename T>
3303 return detail::divided_by_count(
sum(t), t.
size());
3308 template <
typename T>
3312 if (
auto laid {detail::as_laid_reduced(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return mean(elements, along); })}) {
3313 return std::move(*laid);
3317 detail::pairwise_sums(t.
data().data(), walk.outer, walk.extent, walk.inner, out.data());
3319 x = detail::divided_by_count(x, walk.extent);
3321 return tensor<T> {detail::without_axis(t.
shape(), axis), std::move(out)};
3326 template <std::totally_ordered T>
3329 if (
const auto laid {t.
laid_out()}) {
3330 return max(laid->first);
3332 return detail::extreme(t.
data().data(), t.
size(), 1, std::greater<T> {});
3337 template <std::totally_ordered T>
3340 if (
const auto laid {t.
laid_out()}) {
3341 return min(laid->first);
3343 return detail::extreme(t.
data().data(), t.
size(), 1, std::less<T> {});
3350 template <
typename T>
3355 for (
const T& x : t.
data()) {
3356 product = times(product, x);
3363 template <
typename T>
3367 if (
auto laid {detail::as_laid_reduced(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return prod(elements, along); })}) {
3368 return std::move(*laid);
3376 template <
typename T,
3384 const T*
const in {t.
data().data()};
3385 if (walk.inner == 1) {
3386 parallel_for(walk.outer, walk.extent, [&](std::size_t o) {
3387 const std::size_t first {o * walk.extent};
3388 for (std::size_t k {0}; k < walk.extent; ++k) {
3389 out[first + k] = k == 0 ? in[first] : step(out[first + k - 1], in[first + k]);
3395 const std::size_t bands {(walk.inner + band_columns - 1) / band_columns};
3396 parallel_for(walk.outer * bands, walk.extent * band_columns, [&](std::size_t at) {
3397 const std::size_t o {at / bands};
3398 const std::size_t first {(at % bands) * band_columns};
3399 const std::size_t width {std::min(band_columns, walk.inner - first)};
3400 const std::size_t base {(o * walk.extent * walk.inner) + first};
3401 for (std::size_t k {0}; k < walk.extent; ++k) {
3402 const std::size_t row {base + (k * walk.inner)};
3403 for (std::size_t w {0}; w < width; ++w) {
3404 out[row + w] = k == 0 ? in[row + w] : step(out[row - walk.inner + w], in[row + w]);
3408 return tensor<T> {t.shape(), std::move(out)};
3417 template <detail::split_complex C>
3423 const auto multiply {[
vector, &loops](
const C& running,
const C& x) {
3427 return scan_along(elements, along, multiply);
3429 return std::move(*laid);
3435 template <std::
floating_po
int T>
3439 return std::isnan(b) ? !std::isnan(a) : a < b;
3443 template <std::
integral T>
3444 [[nodiscard]]
bool sorts_before(T a,
3455 template <split_complex C>
3457 const C& b)
noexcept
3459 const bool a_imaginary_nan {std::isnan(a.imag())};
3460 const bool b_imaginary_nan {std::isnan(b.imag())};
3461 if (a.real() < b.real()) {
3462 return !a_imaginary_nan || b_imaginary_nan;
3464 if (a.real() > b.real()) {
3465 return b_imaginary_nan && !a_imaginary_nan;
3467 if (a.real() == b.real() || (std::isnan(a.real()) && std::isnan(b.real()))) {
3468 return a.imag() < b.imag() || (b_imaginary_nan && !a_imaginary_nan);
3470 return std::isnan(b.real());
3476 template <
typename T>
3480 if (
auto laid {detail::as_laid_along(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return cumsum(elements, along); })}) {
3481 return std::move(*laid);
3488 template <
typename T>
3492 if (
auto laid {detail::as_laid_along(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return cumprod(elements, along); })}) {
3493 return std::move(*laid);
3503 template <
typename T>
3504 using order_key_t = std::conditional_t<
sizeof(T) == 8, std::uint64_t, std::uint32_t>;
3507 template <
typename U>
3508 inline constexpr U
sign_bit {U {1} << ((
sizeof(U) * 8) - 1)};
3512 template <std::
floating_po
int T>
3516 const auto bits {std::bit_cast<U>(x)};
3517 return (bits & sign_bit<U>) != 0 ?
static_cast<U
>(~bits) :
static_cast<U
>(bits | sign_bit<U>);
3521 template <std::
integral T>
3522 [[nodiscard]] order_key_t<T> order_key(T x)
noexcept
3524 using U = order_key_t<T>;
3525 return static_cast<U
>(std::bit_cast<U>(x) ^ sign_bit<U>);
3529 template <std::
floating_po
int T>
3533 return std::bit_cast<T>((key & sign_bit<U>) != 0 ?
static_cast<U
>(key & ~sign_bit<U>) :
static_cast<U
>(~key));
3537 template <std::
integral T>
3538 [[nodiscard]] T from_order_key(order_key_t<T> key)
noexcept
3540 return std::bit_cast<T>(
static_cast<order_key_t<T>
>(key ^ sign_bit<order_key_t<T>>));
3544 template <
typename U>
3545 using byte_counts = std::array<std::array<std::size_t, 256>,
sizeof(U)>;
3549 template <
typename U>
3553 for (std::size_t b {0}; b <
sizeof(U); ++b) {
3554 ++counts[b][(key >> (8 * b)) & 0xffU];
3561 std::size_t p)
noexcept
3563 return (count / parts * p) + std::min(p, count % parts);
3575 template <
typename U>
3578 std::int64_t* indices,
3579 std::int64_t* index_spare,
3583 const std::size_t parts {counts.size()};
3584 const std::size_t work {count / parts * 16};
3586 U* key_to {key_spare};
3587 std::int64_t* index_from {indices};
3588 std::int64_t* index_to {index_spare};
3590 for (std::size_t b {0}; b <
sizeof(U); ++b) {
3591 const std::size_t shift {8 * b};
3592 std::array<std::size_t, 256> total {};
3594 std::transform(total.begin(), total.end(), part[b].begin(), total.begin(), std::plus {});
3596 if (std::ranges::find(total, count) != total.end()) {
3599 if (moved && parts > 1) {
3601 std::array<std::size_t, 256> part {};
3602 const std::size_t end {
part_begin(count, parts, p + 1)};
3603 for (std::size_t i {
part_begin(count, parts, p)}; i < end; ++i) {
3604 ++part[(key_from[i] >> shift) & 0xffU];
3606 counts[p][b] = part;
3609 std::size_t place {0};
3610 for (std::size_t v {0}; v < 256; ++v) {
3612 const std::size_t held {part[b][v]};
3621 std::array<std::size_t, 256> at {counts[p][b]};
3622 const std::size_t end {
part_begin(count, parts, p + 1)};
3623 if (index_from ==
nullptr) {
3624 for (std::size_t i {
part_begin(count, parts, p)}; i < end; ++i) {
3625 key_to[at[(key_from[i] >> shift) & 0xffU]++] = key_from[i];
3629 for (std::size_t i {
part_begin(count, parts, p)}; i < end; ++i) {
3630 const std::size_t k {at[(key_from[i] >> shift) & 0xffU]++};
3631 key_to[k] = key_from[i];
3632 index_to[k] = index_from[i];
3635 std::swap(key_from, key_to);
3636 std::swap(index_from, index_to);
3639 return key_from != keys;
3647 template <
typename T>
3650 std::size_t parts = 1)
3653 const auto count {
static_cast<std::size_t
>(last - first)};
3654 const std::size_t shares {std::clamp<std::size_t>(parts, 1, std::max<std::size_t>(count, 1))};
3657 std::vector<byte_counts<U>> counts(shares);
3658 parallel_for(shares, count / shares * 16, [&](std::size_t p) {
3660 const std::size_t end {
part_begin(count, shares, p + 1)};
3661 for (std::size_t i {
part_begin(count, shares, p)}; i < end; ++i) {
3667 const U*
const sorted {
radix_passes(keys.data(), spare.data(),
nullptr,
nullptr, count, counts) ? spare.data() : keys.data()};
3668 parallel_for(shares, count / shares * 16, [&](std::size_t p) {
3669 for (std::size_t i {
part_begin(count, shares, p)}; i <
part_begin(count, shares, p + 1); ++i) {
3670 first[i] = from_order_key<T>(sorted[i]);
3682 template <std::
floating_po
int T>
3685 std::size_t parts = 1)
3687 T*
const numbers_end {std::stable_partition(first, last, [](T x) {
return !std::isnan(x); })};
3689 std::stable_sort(first,
numbers_end, [](T a, T b) {
return a < b; });
3692 std::vector<bool> negative_zeros;
3695 negative_zeros.push_back(std::signbit(*x));
3699 T*
zero {std::lower_bound(first,
numbers_end, T {0}, [](T a, T b) {
return a < b; })};
3700 for (
const bool negative : negative_zeros) {
3701 *
zero++ = negative ? -T {0} : T {0};
3706 template <
typename T>
3707 concept radix_orderable = std::same_as<T, double> || std::same_as<T, float> || std::same_as<T, std::int64_t>;
3711 template <std::
floating_po
int T>
3714 return order_key(x == T {0} ? T {0} : x);
3724 template <std::
floating_po
int T>
3730 return std::stable_partition(first, last, [&](std::int64_t k) {
return !std::isnan(values[
static_cast<std::size_t
>(k) * stride]); });
3736 const std::int64_t* ,
3749 template <radix_orderable T,
3756 std::size_t parts = 1)
3759 std::stable_sort(first, last, less);
3763 std::int64_t*
const numbers_last {
numbers_end(first, last, values, stride)};
3764 const auto count {
static_cast<std::size_t
>(numbers_last - first)};
3765 const std::size_t shares {std::clamp<std::size_t>(parts, 1, std::max<std::size_t>(count, 1))};
3769 std::vector<byte_counts<U>> counts(shares);
3770 parallel_for(shares, count / shares * 16, [&](std::size_t p) {
3772 const std::size_t end {
part_begin(count, shares, p + 1)};
3773 for (std::size_t i {
part_begin(count, shares, p)}; i < end; ++i) {
3774 keys[i] =
argsort_key(values[
static_cast<std::size_t
>(first[i]) * stride]);
3779 if (
radix_passes(keys.data(), key_spare.data(), first, index_spare.data(), count, counts)) {
3780 std::copy_n(index_spare.data(), count, first);
3786 template <
typename T,
3788 void order_run(std::int64_t* first,
3795 std::stable_sort(first, last, less);
3799 template <std::
integral T>
3803 std::stable_sort(first, last);
3810 std::size_t parts = 1)
3813 std::stable_sort(first, last);
3820 template <split_complex C>
3824 std::stable_sort(first, last, [](
const C& a,
const C& b) {
return sorts_before(a, b); });
3830 std::size_t pieces {4};
3831 while (pieces < threads) {
3844 template <
typename T,
3850 const SortRun& sort_piece,
3853 std::vector<std::size_t> bounds(pieces + 1);
3854 for (std::size_t k {0}; k <= pieces; ++k) {
3857 parallel_for(pieces, count / pieces * 16, [&](std::size_t k) { sort_piece(data + bounds[k], data + bounds[k + 1]); });
3860 T* to {scratch.data()};
3861 for (std::size_t width {1}; width < pieces; width *= 2) {
3862 const std::size_t pairs {(pieces + (2 * width) - 1) / (2 * width)};
3863 parallel_for(pairs, count / pairs, [&](std::size_t pair) {
3864 const std::size_t first {bounds[pair * 2 * width]};
3865 const std::size_t middle {bounds[std::min((pair * 2 * width) + width, pieces)]};
3866 const std::size_t last {bounds[std::min((pair + 1) * 2 * width, pieces)]};
3867 std::merge(from + first, from + middle, from + middle, from + last, to + first, less);
3869 std::swap(from, to);
3875 template <radix_orderable T>
3878 std::size_t threads)
3880 sort_run(run, run + count, threads);
3885 template <
typename T>
3886 void sort_shared(T* run,
3888 std::size_t threads)
3890 parallel_sort_run(run, count, merged_pieces(threads), [](T* first, T* last) { sort_run(first, last); },
3891 [](
const T& a,
const T& b) {
return sorts_before(a, b); });
3895 template <radix_orderable T,
3902 std::size_t threads)
3904 order_run(order, order + count, values, stride, less, threads);
3908 template <
typename T,
3910 void order_shared(std::int64_t* order,
3915 std::size_t threads)
3917 parallel_sort_run(order, count, merged_pieces(threads),
3918 [&](std::int64_t* first, std::int64_t* last) { order_run(first, last, values, stride, less); }, less);
3930 template <
typename T>
3934 if (
auto laid {detail::as_laid_along(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return sort(elements, along); })}) {
3935 return std::move(*laid);
3938 const std::size_t lines {walk.
outer * walk.inner};
3940 const T*
const in {t.
data().data()};
3941 const auto gather {[&](std::size_t line, T* into) {
3942 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
3943 for (std::size_t k {0}; k < walk.extent; ++k) {
3944 into[k] = in[first + (k * walk.inner)];
3947 const auto scatter {[&](std::size_t line,
const T* from) {
3948 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
3949 for (std::size_t k {0}; k < walk.extent; ++k) {
3950 out[first + (k * walk.inner)] = from[k];
3953 const std::size_t threads {detail::parallel_threads(walk.extent, 16)};
3954 if (threads >= 2 && detail::parallel_threads(lines, walk.extent * 16) * 2 <= threads) {
3955 buffer<T> line_buffer(walk.inner == 1 ? 0 : walk.extent);
3956 for (std::size_t line {0}; line < lines; ++line) {
3957 T*
const run {walk.inner == 1 ? out.data() + (line * walk.extent) : line_buffer.data()};
3959 detail::sort_shared(run, walk.extent, threads);
3960 if (walk.inner != 1) {
3966 detail::parallel_for(lines, walk.extent * 16, [&](std::size_t line) {
3967 if (walk.inner == 1) {
3968 T* const run {out.data() + (line * walk.extent)};
3970 detail::sort_run(run, run + walk.extent);
3974 gather(line, run.data());
3975 detail::sort_run(run.data(), run.data() + walk.extent);
3976 scatter(line, run.data());
3978 return tensor<T> {t.shape(), std::move(out)};
3986 template <
typename T>
3990 if (
auto laid {detail::as_laid_along(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return argsort(elements, along); })}) {
3991 return std::move(*laid);
3994 const std::size_t lines {walk.
outer * walk.inner};
3996 const T*
const in {t.
data().data()};
3997 const std::size_t threads {detail::parallel_threads(walk.extent, 16)};
3998 const bool shared {threads >= 2 && detail::parallel_threads(lines, walk.extent * 16) * 2 <= threads};
3999 const auto order_line {[&](std::size_t line,
bool among_threads) {
4000 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
4001 const auto value {[&](std::int64_t k) ->
const T& {
4002 return in[first + (
static_cast<std::size_t
>(k) * walk.inner)];
4004 const auto less {[&](std::int64_t a, std::int64_t b) {
return detail::sorts_before(value(a), value(b)); }};
4006 std::iota(order.begin(), order.end(), std::int64_t {0});
4007 const T*
const line_values {in + first};
4008 if (among_threads) {
4009 detail::order_shared(order.data(), walk.extent, line_values, walk.inner, less, threads);
4012 detail::order_run(order.data(), order.data() + walk.extent, line_values, walk.inner, less);
4014 for (std::size_t k {0}; k < walk.extent; ++k) {
4015 out[first + (k * walk.inner)] = order[k];
4019 for (std::size_t line {0}; line < lines; ++line) {
4020 order_line(line,
true);
4024 detail::parallel_for(lines, walk.extent * 16, [&](std::size_t line) { order_line(line, false); });
4035 std::sort(kths.begin(), kths.end());
4036 kths.erase(std::unique(kths.begin(), kths.end()), kths.end());
4037 if (!kths.empty() && kths.back() >= extent) {
4038 throw index_error(
"partition position " + std::to_string(kths.back()) +
" is out of bounds for an extent of "
4039 + std::to_string(extent));
4046 template <
typename It,
4050 const std::vector<std::size_t>& kths,
4053 std::size_t from {0};
4054 for (
const std::size_t k : kths) {
4055 std::nth_element(first +
static_cast<std::ptrdiff_t
>(from), first +
static_cast<std::ptrdiff_t
>(k),
4056 first +
static_cast<std::ptrdiff_t
>(count), less);
4069 template <
typename T>
4071 const std::vector<std::size_t>& kths,
4074 if (
auto laid {detail::as_laid_along(t, axis, [&kths](
const tensor<T>& elements, std::size_t along) {
return partition(elements, kths, along); })}) {
4075 return std::move(*laid);
4078 const std::vector<std::size_t> positions {detail::partition_positions(kths, walk.extent)};
4080 const T*
const in {t.
data().data()};
4081 detail::parallel_for(walk.outer * walk.inner, walk.extent * 4, [&](std::size_t line) {
4082 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
4084 for (std::size_t k {0}; k < walk.extent; ++k) {
4085 run[k] = in[first + (k * walk.inner)];
4087 detail::select_positions(run.begin(), walk.extent, positions,
4088 [](
const T& a,
const T& b) { return detail::sorts_before(a, b); });
4089 for (std::size_t k {0}; k < walk.extent; ++k) {
4090 out[first + (k * walk.inner)] = run[k];
4093 return tensor<T> {t.shape(), std::move(out)};
4101 template <
typename T>
4103 const std::vector<std::size_t>& kths,
4106 if (
auto laid {detail::as_laid_along(t, axis, [&kths](
const tensor<T>& elements, std::size_t along) {
return argpartition(elements, kths, along); })}) {
4107 return std::move(*laid);
4110 const std::vector<std::size_t> positions {detail::partition_positions(kths, walk.extent)};
4112 const T*
const in {t.
data().data()};
4113 detail::parallel_for(walk.outer * walk.inner, walk.extent * 4, [&](std::size_t line) {
4114 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
4116 std::iota(order.begin(), order.end(), std::int64_t {0});
4117 const auto value {[&](std::int64_t k) ->
const T& {
4118 return in[first + (
static_cast<std::size_t
>(k) * walk.inner)];
4120 detail::select_positions(order.begin(), walk.extent, positions, [&](std::int64_t a, std::int64_t b) {
4121 return detail::sorts_before(value(a), value(b));
4123 for (std::size_t k {0}; k < walk.extent; ++k) {
4124 out[first + (k * walk.inner)] = order[k];
4127 return tensor<std::int64_t> {t.shape(), std::move(out)};
4132 template <
typename T>
4141 template <split_complex C>
4156 template <
typename T>
4161 throw dimension_error(
"vdot of " + std::to_string(a.
size()) +
" and " + std::to_string(b.
size()) +
" elements");
4163 return detail::conjugated_chunks(a.
data().data(), b.
data().data(), a.
size());
4173 template <
typename T>
4179 throw dimension_error(
"argsort_by: the order and the key of one shape");
4182 const auto extent {
static_cast<std::int64_t
>(walk.extent)};
4183 const std::int64_t*
const from {order.
data().data()};
4184 if (std::any_of(from, from + order.
size(), [extent](std::int64_t k) { return k < 0 || k >= extent; })) {
4185 throw dimension_error(
"argsort_by: an index out of the axis's range");
4187 const std::size_t lines {walk.outer * walk.inner};
4189 const T*
const in {key.
data().data()};
4190 const std::size_t threads {detail::parallel_threads(walk.extent, 16)};
4191 const bool shared {threads >= 2 && detail::parallel_threads(lines, walk.extent * 16) * 2 <= threads};
4192 const auto order_line {[&](std::size_t line,
bool among_threads) {
4193 const std::size_t first {((line / walk.inner) * walk.extent * walk.inner) + (line % walk.inner)};
4194 const auto value {[&](std::int64_t k) ->
const T& {
4195 return in[first + (
static_cast<std::size_t
>(k) * walk.inner)];
4197 const auto less {[&](std::int64_t a, std::int64_t b) {
return detail::sorts_before(value(a), value(b)); }};
4199 for (std::size_t k {0}; k < walk.extent; ++k) {
4200 run[k] = from[first + (k * walk.inner)];
4202 if (among_threads) {
4203 detail::order_shared(run.data(), walk.extent, in + first, walk.inner, less, threads);
4206 detail::order_run(run.data(), run.data() + walk.extent, in + first, walk.inner, less);
4208 for (std::size_t k {0}; k < walk.extent; ++k) {
4209 out[first + (k * walk.inner)] = run[k];
4213 for (std::size_t line {0}; line < lines; ++line) {
4214 order_line(line,
true);
4218 detail::parallel_for(lines, walk.extent * 16, [&](std::size_t line) { order_line(line, false); });
4225 template <std::
floating_po
int T>
4228 return std::isnan(x);
4232 template <std::
integral T>
4233 [[nodiscard]]
bool wins_any_order(T )
noexcept
4240 template <
typename T,
4242 [[nodiscard]]
bool holds(
const T* first,
4244 const Match& match)
noexcept
4247 for (std::size_t i {0}; i < count; ++i) {
4248 hit |=
static_cast<unsigned>(match(first[i]));
4255 template <
typename T,
4259 const Match& match)
noexcept
4261 constexpr std::size_t block {64};
4262 std::size_t start {0};
4263 while (!
holds(first + start, std::min(block, count - start), match)) {
4266 std::size_t at {start};
4267 while (!match(first[at])) {
4277 template <std::floating_point T,
4281 const Order& larger)
4284 if (std::isnan(value)) {
4285 return static_cast<std::int64_t
>(
first_held(first, count, [](T x) {
return std::isnan(x); }));
4287 return static_cast<std::int64_t
>(
first_held(first, count, [value](T x) {
return x == value; }));
4291 template <std::integral T,
4293 [[nodiscard]] std::int64_t extreme_index(
const T* first,
4295 const Order& larger)
4297 std::size_t best {0};
4298 for (std::size_t i {1}; i < count; ++i) {
4299 if (larger(first[i], first[best])) {
4303 return static_cast<std::int64_t
>(best);
4307 template <
typename T,
4311 const Order& larger)
4314 if (walk.extent == 0) {
4315 throw dimension_error(
"attempt to get argmax or argmin of an empty sequence");
4318 const T*
const in {t.
data().data()};
4319 if (walk.inner == 1) {
4320 parallel_for(walk.outer, walk.extent, [&](std::size_t line) {
4321 out[line] = extreme_index(in + (line * walk.extent), walk.extent, larger);
4329 const std::size_t o {at / bands};
4331 const std::size_t width {std::min(
band_columns, walk.inner - first)};
4332 const T*
const base {in + (o * walk.extent * walk.inner) + first};
4333 std::array<std::size_t, band_columns> best {};
4334 std::array<bool, band_columns> won {};
4335 for (std::size_t w {0}; w < width; ++w) {
4338 for (std::size_t k {1}; k < walk.extent; ++k) {
4339 const T*
const row {base + (k * walk.inner)};
4340 for (std::size_t w {0}; w < width; ++w) {
4348 else if (larger(row[w], base[(best[w] * walk.inner) + w])) {
4353 for (std::size_t w {0}; w < width; ++w) {
4354 out[(o * walk.inner) + first + w] =
static_cast<std::int64_t
>(best[w]);
4362 template <
typename T,
4366 const Order& larger)
4369 std::vector<std::int64_t> best(chunks);
4374 std::int64_t winner {best[0]};
4375 for (
const std::int64_t candidate : best) {
4376 const T& x {first[candidate]};
4380 if (larger(x, first[winner])) {
4390 template <
typename T>
4393 if (t.
size() == 0) {
4396 return detail::parallel_extreme_index(t.
data().data(), t.
size(), std::greater<T> {});
4401 template <
typename T>
4404 if (t.
size() == 0) {
4407 return detail::parallel_extreme_index(t.
data().data(), t.
size(), std::less<T> {});
4412 template <
typename T>
4416 if (
auto laid {detail::as_laid_reduced(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return argmax(elements, along); })}) {
4417 return std::move(*laid);
4419 return detail::extreme_indices(t, axis, std::greater<T> {});
4424 template <
typename T>
4428 if (
auto laid {detail::as_laid_reduced(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return argmin(elements, along); })}) {
4429 return std::move(*laid);
4431 return detail::extreme_indices(t, axis, std::less<T> {});
4441 template <
typename T>
4444 const std::vector<std::size_t>& shape {t.
shape()};
4445 const T*
const data {t.
data().data()};
4446 const std::size_t count {t.
size()};
4447 const std::size_t rank {shape.size()};
4448 const std::size_t chunk {detail::elementwise_chunk};
4449 const std::size_t chunks {(count + chunk - 1) / chunk};
4450 std::vector<std::size_t> offsets(chunks + 1, 0);
4451 detail::parallel_for(chunks, chunk, [&](std::size_t c) {
4452 const std::size_t last {std::min((c + 1) * chunk, count)};
4453 offsets[c + 1] =
static_cast<std::size_t
>(std::count_if(data + (c * chunk), data + last, [](
const T& x) {
return x != T {}; }));
4455 std::partial_sum(offsets.begin(), offsets.end(), offsets.begin());
4456 const std::size_t found {offsets.back()};
4457 std::vector<buffer<std::int64_t>> out(rank);
4461 detail::parallel_for(chunks, chunk * rank, [&](std::size_t c) {
4462 const std::size_t first {c * chunk};
4463 const std::size_t last {std::min(first + chunk, count)};
4464 const std::size_t span {last - first + 1};
4466 std::vector<std::size_t> coordinate(rank);
4467 std::size_t rest {first};
4468 for (std::size_t a {rank}; a-- > 0;) {
4469 coordinate[a] = rest % shape[a];
4473 for (std::size_t k {first}; k < last; ++k) {
4474 for (std::size_t a {0}; a < rank; ++a) {
4475 scratch[(a * span) + at] =
static_cast<std::int64_t
>(coordinate[a]);
4477 at +=
static_cast<std::size_t
>(data[k] != T {});
4478 for (std::size_t a {rank}; a-- > 0;) {
4479 if (++coordinate[a] < shape[a]) {
4485 for (std::size_t a {0}; a < rank; ++a) {
4486 std::copy_n(scratch.data() + (a * span), at, out[a].data() + offsets[c]);
4489 std::vector<tensor<std::int64_t>> indices;
4490 indices.reserve(rank);
4492 indices.emplace_back(std::vector<std::size_t> {found}, std::move(axis_out));
4500 template <
typename T>
4503 if (
const auto laid {t.
laid_out()}) {
4506 const auto& values {t.
data()};
4507 const std::size_t count {t.
size()};
4508 const std::size_t chunk {detail::elementwise_chunk};
4509 const std::size_t chunks {(count + chunk - 1) / chunk};
4510 std::vector<std::size_t> found(chunks, 0);
4511 detail::parallel_for(chunks, chunk, [&](std::size_t c) {
4512 const std::size_t last {std::min((c + 1) * chunk, count)};
4513 std::size_t total {0};
4514 for (std::size_t k {c * chunk}; k < last; ++k) {
4515 total +=
static_cast<std::size_t
>(values[k] != T {});
4519 return std::accumulate(found.begin(), found.end(), std::size_t {0});
4524 template <
typename T>
4528 if (
auto laid {detail::as_laid_reduced(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return count_nonzero(elements, along); })}) {
4529 return std::move(*laid);
4531 return detail::fold_along(t, axis, std::int64_t {0},
4532 [](std::int64_t total,
const T& x) {
return total +
static_cast<std::int64_t
>(x != T {}); });
4541 template <
typename T>
4547 bool matched {indices.
rank() == t.
rank()};
4548 for (std::size_t k {0}; matched && k < t.
rank(); ++k) {
4549 matched = k == axis || indices.
shape()[k] == t.
shape()[k];
4552 throw dimension_error(
"take_along_axis: the indices' shape is the tensor's but along the axis");
4554 const auto extent {
static_cast<std::int64_t
>(t.
shape()[axis])};
4555 if (std::ranges::any_of(indices.
data(), [extent](std::int64_t i) { return i < 0 || i >= extent; })) {
4556 throw dimension_error(
"take_along_axis: an index out of the axis's range");
4558 const std::size_t count {indices.
shape()[axis]};
4559 const std::size_t inner {walk.inner};
4561 const T*
const in {t.
data().data()};
4562 const std::int64_t* at {indices.
data().data()};
4563 detail::parallel_for(indices.
size() / std::max<std::size_t>(inner, 1), inner, [&](std::size_t row) {
4564 const std::size_t o {row / std::max<std::size_t>(count, 1)};
4565 for (std::size_t i {0}; i < inner; ++i) {
4566 const auto index {
static_cast<std::size_t
>(at[(row * inner) + i])};
4567 out[(row * inner) + i] = in[(((o * walk.extent) + index) * inner) + i];
4570 return tensor<T> {indices.shape(), std::move(out)};
4592 template <
typename T>
4598 if (axis >= t.
rank()) {
4599 throw dimension_error(
"take: axis " + std::to_string(axis) +
" is out of bounds for a tensor of rank "
4600 + std::to_string(t.
rank()));
4602 const std::vector<std::size_t>& shape {t.
shape()};
4603 const auto at {shape.begin() +
static_cast<std::ptrdiff_t
>(axis)};
4604 std::vector<std::size_t> out_shape(shape.begin(), at);
4605 out_shape.insert(out_shape.end(), indices.
shape().begin(), indices.
shape().end());
4606 out_shape.insert(out_shape.end(), at + 1, shape.end());
4607 const std::size_t outer {detail::element_count(shape.begin(), at)};
4608 const std::size_t inner {detail::element_count(at + 1, shape.end())};
4609 const std::size_t count {indices.
size()};
4610 const std::size_t total {detail::element_count(out_shape.begin(), out_shape.end())};
4611 const std::size_t extent {shape[axis]};
4612 if (extent == 0 && (total != 0 || (mode == take_mode::wrap && count != 0))) {
4613 throw index_error(
"cannot do a non-empty take from an empty axes.");
4615 const auto signed_extent {
static_cast<std::int64_t
>(extent)};
4617 const auto outside {[&](std::int64_t value) {
4618 return index_error(
"index " + std::to_string(value) +
" is out of bounds for axis " + std::to_string(axis)
4619 +
" with size " + std::to_string(extent));
4622 for (std::size_t k {0}; mode == take_mode::raise && outer != 0 && k < count; ++k) {
4623 if (index[k] < -signed_extent || index[k] >= signed_extent) {
4624 throw outside(index[k]);
4629 std::vector<std::size_t> offsets(count);
4630 for (std::size_t k {0}; k < count; ++k) {
4631 std::int64_t i {index[k]};
4632 if (mode == take_mode::wrap) {
4634 i += i < 0 ? signed_extent : 0;
4636 else if (mode == take_mode::clip) {
4637 i = std::clamp<std::int64_t>(i, 0, signed_extent - 1);
4639 else if (i < -signed_extent || i >= signed_extent) {
4643 i += i < 0 ? signed_extent : 0;
4645 offsets[k] =
static_cast<std::size_t
>(i);
4648 const T*
const in {t.
data().data()};
4649 detail::parallel_for(outer * count, inner, [&](std::size_t row) {
4650 const std::size_t o {row / count};
4651 const T*
const from {in + (((o * extent) + offsets[row % count]) * inner)};
4652 for (std::size_t i {0}; i < inner; ++i) {
4653 out[(row * inner) + i] = from[i];
4656 return tensor<T> {std::move(out_shape), std::move(out)};
4669 template <std::
floating_po
int T>
4675 const bool nan {std::isnan(chosen)};
4676 if (!nan && chosen != T {0}) {
4679 std::size_t first {0};
4680 for (std::size_t i {0}; i < count; ++i) {
4681 first +=
static_cast<std::size_t
>(nan ? !std::isnan(row[i]) : row[i] < T {0});
4683 std::size_t rank {k - first};
4686 if (nan ? std::isnan(row[i]) : row[i] == T {0}) {
4697 template <std::
integral T>
4698 [[nodiscard]] T stable_choice(
const T* ,
4713 template <
typename T>
4714 requires std::floating_point<T> || std::integral<T>
4717 const std::int64_t* at,
4722 std::copy(row, row + count, work);
4725 for (std::size_t j {0}; j < m; ++j) {
4726 out[j] = work[
static_cast<std::size_t
>(at[j])];
4730 std::array<std::size_t, select_positions_sorted> order {};
4731 for (std::size_t j {0}; j < m; ++j) {
4732 const auto k {
static_cast<std::size_t
>(at[j])};
4734 for (; i > 0 && order[i - 1] > k; --i) {
4735 order[i] = order[i - 1];
4739 const auto before {[](T a, T b) {
return sorts_before(a, b); }};
4740 std::size_t from {0};
4741 for (std::size_t j {0}; j < m; ++j) {
4742 const std::size_t k {order[j]};
4746 if (k == from || k + 1 == count) {
4747 std::iter_swap(work + k, k == from ? std::min_element(work + from, work + count, before)
4748 : std::max_element(work + from, work + count, before));
4751 std::nth_element(work + from, work + k, work + count, before);
4755 for (std::size_t j {0}; j < m; ++j) {
4756 const auto k {
static_cast<std::size_t
>(at[j])};
4762 template <split_complex C>
4765 const std::int64_t* at,
4770 std::copy(row, row + count, work);
4772 for (std::size_t j {0}; j < m; ++j) {
4773 out[j] = work[
static_cast<std::size_t
>(at[j])];
4785 template <
typename T>
4789 if (t.
rank() == 0 || positions.
rank() == 0) {
4790 throw dimension_error(
"select_along_last: a tensor and positions of rank 1 or more");
4792 const std::vector<std::size_t> lead(t.
shape().begin(), t.
shape().end() - 1);
4793 const std::size_t count {t.
shape().back()};
4794 const std::size_t m {positions.
shape().back()};
4795 const bool every {positions.
rank() == 1};
4796 if (!every && !std::equal(lead.begin(), lead.end(), positions.
shape().begin(), positions.
shape().end() - 1)) {
4797 throw dimension_error(
"select_along_last: positions of shape (m,) or the lines' shape and m");
4799 const auto extent {
static_cast<std::int64_t
>(count)};
4800 if (std::ranges::any_of(positions.
data(), [extent](std::int64_t k) { return k < 0 || k >= extent; })) {
4801 throw dimension_error(
"select_along_last: a position out of the lines");
4803 std::vector<std::size_t> shape {lead};
4805 const std::size_t lines {std::accumulate(lead.begin(), lead.end(), std::size_t {1}, std::multiplies {})};
4807 const T*
const in {t.
data().data()};
4808 const std::int64_t* at {positions.
data().data()};
4809 const std::size_t block {std::max<std::size_t>(1, 4096 / std::max<std::size_t>(count, 1))};
4810 const std::size_t blocks {(lines + block - 1) / block};
4811 detail::parallel_for(blocks, block * count * 16, [&](std::size_t b) {
4813 for (std::size_t line {b * block}; line < std::min(lines, (b + 1) * block); ++line) {
4814 detail::select_row(in + (line * count), count, at + (every ? 0 : line * m), m, work.data(),
4815 out.data() + (line * m));
4818 return tensor<T> {std::move(shape), std::move(out)};
4827 template <
typename T>
4836 template <split_complex C>
4853 template <
typename T>
4856 std::ptrdiff_t first,
4859 constexpr std::size_t block {detail::correlation_block};
4860 const auto n {
static_cast<std::ptrdiff_t
>(a.
size())};
4861 const auto m {
static_cast<std::ptrdiff_t
>(v.
size())};
4862 const T*
const x {a.
data().data()};
4863 const T*
const y {v.
data().data()};
4865 detail::parallel_for((count + block - 1) / block, block *
static_cast<std::size_t
>(m), [&](std::size_t b) {
4866 const std::size_t lags {std::min(block, count - (b * block))};
4867 const std::ptrdiff_t k {first +
static_cast<std::ptrdiff_t
>(b * block)};
4868 if (lags == block && k >= 0 && k +
static_cast<std::ptrdiff_t
>(block) + m <= n + 1) {
4869 std::array<T, block>
sum {};
4870 for (std::ptrdiff_t j {0}; j < m; ++j) {
4871 for (std::size_t l {0}; l < block; ++l) {
4872 sum[l] = detail::correlation_step(
sum[l], x[j + k +
static_cast<std::ptrdiff_t
>(l)], y[j]);
4875 std::copy(
sum.begin(),
sum.end(), out.data() + (b * block));
4878 for (std::size_t l {0}; l < lags; ++l) {
4879 const std::ptrdiff_t lag {k +
static_cast<std::ptrdiff_t
>(l)};
4881 for (std::ptrdiff_t j {std::max<std::ptrdiff_t>(0, -lag)}; j < std::min(m, n - lag); ++j) {
4882 sum = detail::correlation_step(
sum, x[j + lag], y[j]);
4884 out[(b * block) + l] =
sum;
4887 return tensor<T> {{count}, std::move(out)};
4895 template <
typename T>
4900 if (a.
rank() != 1) {
4903 const T*
const first {a.
data().data()};
4904 const T*
const last {first + a.
size()};
4905 const auto less {[](
const T& x,
const T& y) {
return detail::sorts_before(x, y); }};
4906 return detail::parallel_map(v, [&](
const T& x) {
4907 const T*
const at {right ? std::upper_bound(first, last, x, less) : std::lower_bound(first, last, x, less)};
4908 return static_cast<std::int64_t
>(at - first);
4914 template <std::totally_ordered T>
4918 if (
auto laid {detail::as_laid_reduced(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return max(elements, along); })}) {
4919 return std::move(*laid);
4921 return detail::extremes_along(t, axis, std::greater<T> {});
4926 template <std::totally_ordered T>
4930 if (
auto laid {detail::as_laid_reduced(t, axis, [](
const tensor<T>& elements, std::size_t along) {
return min(elements, along); })}) {
4931 return std::move(*laid);
4933 return detail::extremes_along(t, axis, std::less<T> {});
4942 template <
typename T>
4944 std::vector<std::size_t> shape)
4946 const std::vector<std::size_t>& from {t.
shape()};
4947 bool fits {from.size() <= shape.size()};
4948 for (std::size_t i {0}; fits && i < from.size(); ++i) {
4949 const std::size_t have {from[from.size() - 1 - i]};
4950 fits = have == 1 || have == shape[shape.size() - 1 - i];
4953 throw dimension_error(
"broadcast_to: the shape " + detail::shape_text(from) +
" does not broadcast to "
4954 + detail::shape_text(shape));
4956 const std::size_t total {detail::element_count(shape.begin(), shape.end())};
4959 return tensor<T> {std::move(shape), std::move(out)};
4961 const std::vector<std::size_t> strides {detail::broadcast_strides(from, shape.size())};
4962 const std::size_t inner {shape.empty() ? 1 : shape.back()};
4963 const bool spread {!shape.empty() && strides.back() == 0};
4964 const T*
const in {t.
data().data()};
4965 detail::parallel_for(total / inner, inner, [&](std::size_t row) {
4966 std::size_t rest {row};
4968 for (std::size_t axis {shape.size() - (shape.empty() ? 0 : 1)}; axis-- > 0;) {
4969 at += (rest % shape[axis]) * strides[axis];
4970 rest /= shape[axis];
4972 T*
const into {out.data() + (row * inner)};
4974 std::fill_n(into, inner, in[at]);
4977 std::copy_n(in + at, inner, into);
4980 return tensor<T> {std::move(shape), std::move(out)};
4987 template <
typename T>
4989 std::vector<std::size_t> shape)
4991 std::size_t count {1};
4992 for (
const std::size_t extent : shape) {
4993 count = detail::checked_mul(count, extent);
4995 if (count != t.
size()) {
4996 throw dimension_error(
"cannot reshape a tensor of size " + std::to_string(t.
size()) +
" into shape "
4997 + detail::shape_text(shape));
4999 return t.
reshaped(std::move(shape));
5008 template <
typename T>
5010 std::ptrdiff_t offset)
5012 const std::vector<std::size_t>& shape {t.
shape()};
5013 if (shape.size() < 2) {
5014 throw dimension_error(
"diagonal: a matrix or a stack of matrices (rank >= 2)");
5016 const std::size_t rows {shape[shape.size() - 2]};
5017 const std::size_t cols {shape[shape.size() - 1]};
5018 const std::size_t magnitude {offset < 0 ? std::size_t {0} -
static_cast<std::size_t
>(offset) :
static_cast<std::size_t
>(offset)};
5019 const std::size_t r0 {offset < 0 ? magnitude : 0};
5020 const std::size_t c0 {offset < 0 ? 0 : magnitude};
5021 const std::size_t length {r0 < rows && c0 < cols ? std::min(rows - r0, cols - c0) : 0};
5034 const auto r {
static_cast<std::ptrdiff_t
>(rows)};
5035 const auto c {
static_cast<std::ptrdiff_t
>(cols)};
5036 const auto band {std::clamp<std::ptrdiff_t>(k, -r - 1, c + 1)};
5037 const auto edge {
static_cast<std::ptrdiff_t
>(i) + band +
static_cast<std::ptrdiff_t
>(lower)};
5038 return static_cast<std::size_t
>(std::clamp<std::ptrdiff_t>(edge, 0, c));
5047 template <
typename T>
5052 const std::vector<std::size_t>& shape {t.
shape()};
5053 if (shape.size() < 2) {
5054 throw dimension_error(
"triangle: a matrix or a stack of matrices (rank >= 2)");
5056 if (
const auto laid {t.
laid_out()}) {
5057 const std::vector<std::size_t>& order {laid->second};
5058 const std::size_t rank {order.size()};
5059 const bool kept {order[rank - 2] == rank - 2 && order[rank - 1] == rank - 1};
5060 if (kept || (order[rank - 2] == rank - 1 && order[rank - 1] == rank - 2)) {
5061 const auto reach {
static_cast<std::ptrdiff_t
>(shape[rank - 2] + shape[rank - 1])};
5062 const std::ptrdiff_t within {std::clamp(k, -reach, reach)};
5063 return triangle(laid->first, kept ? within : -within, kept == lower).transposed(detail::view_axes(order));
5066 const std::size_t rows {shape[shape.size() - 2]};
5067 const std::size_t cols {shape[shape.size() - 1]};
5068 const std::size_t count {t.
size()};
5071 return tensor<T> {shape, std::move(out)};
5073 const T*
const in {t.
data().data()};
5074 detail::parallel_for(count / cols, cols * detail::memory_element_weight, [&](std::size_t line) {
5075 const std::size_t split {detail::triangle_split(line % rows, rows, cols, k, lower)};
5076 const std::size_t first {lower ? 0 : split};
5077 const std::size_t last {lower ? split : cols};
5078 T*
const row {out.data() + (line * cols)};
5079 std::fill_n(row, first, T {});
5080 std::copy_n(in + (line * cols) + first, last - first, row + first);
5081 std::fill_n(row + last, cols - last, T {});
5083 return tensor<T> {shape, std::move(out)};
5096 std::vector<std::size_t> offsets(rows + 1, 0);
5097 for (std::size_t i {0}; i < rows; ++i) {
5098 const std::size_t split {detail::triangle_split(i, rows, cols, k, lower)};
5099 offsets[i + 1] = offsets[i] + (lower ? split : cols - split);
5101 const std::size_t found {offsets.back()};
5104 detail::parallel_for(rows, cols * detail::memory_element_weight, [&](std::size_t i) {
5105 const std::size_t split {detail::triangle_split(i, rows, cols, k, lower)};
5106 const std::size_t at {offsets[i]};
5107 auto column {
static_cast<std::int64_t
>(lower ? 0 : split)};
5108 for (std::size_t j {at}; j < offsets[i + 1]; ++j) {
5109 row_of[j] =
static_cast<std::int64_t
>(i);
5110 col_of[j] = column++;
5117 template <
typename T>
5121 std::size_t count {1};
5122 for (
const std::size_t extent : shape) {
5123 count = detail::checked_mul(count, extent);
5126 detail::parallel_fill(values.data(), count, value);
5127 return tensor<T> {std::move(shape), std::move(values)};
5131 template <
typename T>
5134 std::size_t count {1};
5135 for (
const std::size_t extent : shape) {
5136 count = detail::checked_mul(count, extent);
5139 detail::parallel_zero(values.data(), count);
5140 return tensor<T> {std::move(shape), std::move(values)};
5148 template <
typename T>
5152 if (v.
rank() != 1) {
5155 const std::size_t length {v.
size()};
5156 const std::size_t magnitude {k < 0 ? std::size_t {0} -
static_cast<std::size_t
>(k) :
static_cast<std::size_t
>(k)};
5157 const std::size_t n {length + magnitude};
5158 const std::size_t count {detail::checked_mul(n, n)};
5160 detail::parallel_zero(values.data(), count);
5161 const std::size_t r0 {k < 0 ? magnitude : 0};
5162 const std::size_t c0 {k < 0 ? 0 : magnitude};
5163 const T*
const from {v.
data().data()};
5164 for (std::size_t i {0}; i < length; ++i) {
5165 values[((r0 + i) * n) + c0 + i] = from[i];
5167 return tensor<T> {{n, n}, std::move(values)};
5171 template <
typename T>
5174 return full<T>(std::move(shape),
static_cast<T
>(1));
5178 template <
typename T>
5181 buffer<T> out(detail::checked_mul(n, n), T {});
5182 for (std::size_t i {0}; i < n; ++i) {
5183 out[(i * n) + i] =
static_cast<T
>(1);
5185 return tensor<T> {{n, n}, std::move(out)};
5202 if (step == 0.0 || !std::isfinite(step) || !std::isfinite(start) || !std::isfinite(stop)) {
5203 throw dimension_error(
"arange needs finite bounds and a finite, non-zero step");
5205 const double length {std::ceil((stop - start) / step)};
5206 if (!(length < 0x1p64)) {
5207 throw dimension_error(
"arange: the length does not fit in std::size_t");
5209 const std::size_t count {length > 0.0 ?
static_cast<std::size_t
>(length) : 0};
5211 const double second {start + step};
5212 const double delta {second - start};
5220 for (std::size_t i {2}; i < count; ++i) {
5221 out[i] = std::fma(
static_cast<double>(i), delta, start);
5225 for (std::size_t i {2}; i < count; ++i) {
5227 out[i] = start + std::fma(
static_cast<double>(i), delta, -0.0);
5243 const double step {count > 1 ? (stop - start) /
static_cast<double>(count - 1) : 0.0};
5244 for (std::size_t i {0}; i < count; ++i) {
5249 const double offset {std::fma(
static_cast<double>(i), step, -0.0)};
5250 out[i] = start + offset;
5253 out[count - 1] = stop;
A standard exception Std that numerical::error tags too, its one what answering for both bases.
Thrown when an operation's operands have incompatible shapes.
Thrown when an index or a range leaves its axis.
Thrown when an integer is raised to a negative integer power, as NumPy refuses it: no integer is the ...
negative_power_error(const std::string &message)
Builds the error.
A dense N-dimensional tensor of T with value semantics, read row-major. Its elements are immutable on...
const std::vector< std::size_t > & shape() const noexcept
The size of each axis.
std::optional< std::pair< tensor, std::vector< std::size_t > > > laid_out() const
Where this view reaches each of its elements once from the first (a transpose's, transposed): its axe...
tensor reshaped(std::vector< std::size_t > shape) const
This tensor's elements in shape, without a copy: the new tensor shares the buffer,...
const buffer< T > & data() const
The elements, row-major. A view's (transposed) are copied out by the first call, one thread copying w...
std::size_t rank() const noexcept
The number of axes (0 for a scalar).
const T * origin() const noexcept
The first element where the elements are held (index 0 along every axis): a view (transposed,...
const std::vector< std::ptrdiff_t > & strides() const noexcept
Each axis's step, in elements, from the first element (origin), row-major unless a view,...
std::size_t size() const noexcept
The total number of elements.
bool contiguous() const noexcept
Whether the elements lie row-major where they are held: data then copies nothing.
tensor diagonal_view(std::size_t row, std::size_t column, std::size_t length) const
The length elements from (row, column) down the diagonal of each matrix (the last two axes),...
A dense vector of T with value semantics.
The elementary functions of a complex number of float or double as the GNU C library computes them (N...
The types whose indices order_run sorts by the elements' bits.
exp, exp2, expm1, tanh, sinh and cosh of double by a table of 2^(j/128) and a polynomial,...
bool numpy_fuses_reductions() noexcept
Whether NumPy's complex product fuses its multiply-adds in its reduction loop (the product's loop who...
void parallel_sort_run(T *data, std::size_t count, std::size_t pieces, const SortRun &sort_piece, const Less &less)
Sorts the count elements at data stably in pieces, a power of four, among the threads: each piece sor...
bool is_nan(const T &)
A NaN? Never for a scalar without one.
T elementary_exp2(const T &x)
exp2 of a scalar, as std::exp2 gives it.
T divided_by_count(T value, std::size_t n)
A mean: value divided by the count n, in the wider of T and double and back to T, the count exact (to...
void double_sines_into(const double *in, double *out, std::size_t count)
sin (Cosine false) or cos of the count doubles at in into out, each double_sine's: table_sine to sine...
void exp_into(const double *in, double *out, std::size_t count)
exp of the count doubles at in into out, each elementary_exp's: table_exp in vectors (with the FMA in...
constexpr std::size_t band_columns
The columns of a band that one task of fold_along and scan_along walks row by row.
tensor< T > cos_viewed(const tensor< T > &a)
cos of every element of a: of a contiguous copy (cos_of).
auto broadcast_rows(const T *a, const std::vector< std::size_t > &a_shape, const T *b, const std::vector< std::size_t > &b_shape, const Op &op)
op(a, b) element by element of the row-major a (a_shape) and b (b_shape), broadcast against each othe...
std::int64_t parallel_extreme_index(const T *first, std::size_t count, const Order &larger)
extreme_index of the count contiguous elements at first, by chunks on the threads: each chunk's own,...
void select_row(const T *row, std::size_t count, const std::int64_t *at, std::size_t m, T *work, T *out)
Into out, the elements of the count at row that a stable sort puts at the m positions at,...
std::complex< R > complex_log(const std::complex< R > &z)
log(z): log|z| + i arg(z), log|z| as log1p(|z|² − 1) / 2 near the unit circle (|z|² − 1 exact or almo...
double float_sine(double x)
sin (Cosine false) or cos of a double that a float holds: reduced_sine to sine_reduction_bound,...
std::size_t part_begin(std::size_t count, std::size_t parts, std::size_t p) noexcept
Where part p of parts sharing count elements begins: the first count % parts one longer.
double double_sine(double x)
sin (Cosine false) or cos of a double, as double_sines_into gives it: table_sine to sine_reduction_bo...
axis_walk walk_along(const std::vector< std::size_t > &shape, std::size_t axis)
axis_walk of shape along axis; outer and inner 0 where the result is empty.
tensor< R > pairs_into(const T *a, const T *b, const std::vector< std::size_t > &shape, const Op &op)
op(a, b) element by element of the row-major a and b, both of shape shape, in op's result type R,...
tensor< T > log1p_viewed(const tensor< T > &a)
log1p of every element of a: of a contiguous copy (log1p_of).
T elementary_atan(const T &x)
atan of a scalar, as std::atan gives it.
constexpr std::size_t radix_sort_least
Runs this long or longer are sorted by their bits (radix_sort); shorter ones by comparisons.
C numpy_reciprocal(const C &x, bool fused)
NumPy's complex reciprocal 1 / x (its reciprocal, the loop x ** -1 takes): the ratio of the smaller p...
T clipped(T x, T low, T high) noexcept
ieee_minimum of ieee_maximum of x and low, and high, the bounds no NaN: the same bits (a NaN x kept,...
constexpr double sine_reduction_bound
The magnitude to which reduced_sine reduces its argument: k below 2²⁰, k·π/2's parts exact enough.
bool lies_close(const tensor< T > &t) noexcept
Whether an element-wise operation reads the view t's elements where they lie: stepping within a cache...
bool numpy_loop_is_scalar(const std::vector< std::size_t > &a_shape, const std::vector< std::size_t > &b_shape) noexcept
Whether NumPy multiplies an operand of shape a_shape by one of b_shape in its scalar loop rather than...
void sinh_into(const double *in, double *out, std::size_t count)
sinh of the count doubles at in into out, each elementary_sinh's: table_sinh in vectors (with the FMA...
constexpr U sign_bit
The highest bit of U, a real's or an integer's sign.
T elementary_cosh(const T &x)
cosh of a scalar, as std::cosh gives it.
double table_expm1(double x) noexcept
expm1(x) by table_exp's reduction, x = (128 e + j) ln 2/128 + r: in units of 2^e, 2^(j/128) − 2^−e
std::complex< R > complex_sin(const std::complex< R > &z)
sin(z): sin(re) cosh(im) + i cos(re) sinh(im) of |re|, the sign of re carried by the sine; an imagina...
tensor< T > sin_of(const T *data, const std::vector< std::size_t > &shape)
sin of every one of the row-major scalars at data, of shape shape: std::sin of each.
bool numpy_fuses_scalar_products() noexcept
Whether NumPy's complex product fuses its multiply-adds in its scalar loop (where numpy_loop_is_scala...
void log1p_into(const double *in, double *out, std::size_t count)
log1p of the count doubles at in into out, each elementary_log1p's: table_log1p in vectors (with the ...
void run_fused(const Body &body)
body(0), where std::fma is the instruction (with_fma_instruction) on an x86-64 processor with AVX2 an...
double table_exp2(double x) noexcept
2^x for an x whose result is normal or infinite (−1022 or above): x = (128 e + j)/128 + f,...
void count_bytes(byte_counts< U > &counts, U key) noexcept
Counts key's bytes in counts. Every byte is counted in the one read that makes the keys,...
constexpr std::size_t correlation_block
The lags numerical::correlate sums together: a block's sums held in registers (four AVX2 vectors of d...
C numpy_power(const C &a, const C &b, bool fused)
NumPy's complex power a ** b (npymath's npy_cpow): 1 for a zero exponent, 0 or NaN for a zero base; a...
tensor< R > broadcast_into(const T *a, const std::vector< std::size_t > &a_shape, const std::vector< std::ptrdiff_t > &a_strides, const T *b, const std::vector< std::size_t > &b_shape, const std::vector< std::ptrdiff_t > &b_strides, const Op &op)
op(a, b) element by element of a, of shape a_shape, its elements a_strides apart along each axis from...
void float_sines_into(const double *in, double *out, std::size_t count)
sin (Cosine false) or cos of the count doubles at in, each a float's, into out, each float_sine's: re...
carried_t< T > real_part(const T &x)
The real part of a real (or general) scalar, in carried_t.
T square(const T &x)
x · x, by the type's own product (a complex one's C Annex G, as the library's *).
std::complex< R > complex_tan(const std::complex< R > &z)
tan(z): (sin(re) cos(re) + i sinh(im) cosh(im)) / (cos(re)² + sinh(im)²); past half_exp_bound the ima...
T settled_zero(T value, const T *first, std::size_t count, std::size_t stride, const Order &larger)
The extreme value of count elements stride apart from first, its zero's sign settled as IEEE 754-2019...
T elementary_tanh(const T &x)
tanh of a scalar, as std::tanh gives it.
R divided_by_zero(R x) noexcept
x divided by +0, as IEEE divides it, spelled out: a NaN for a zero or a NaN, else an infinity of x's ...
T stable_choice(const T *row, std::size_t count, std::size_t k, T chosen)
Of the count elements at row, the one a stable sort (sort_run) puts at k, from the one chosen that a ...
tensor< T > cosh_viewed(const tensor< T > &a)
cosh of every element of a: of a contiguous copy (cosh_of).
tensor< T > cosh_of(const T *data, const std::vector< std::size_t > &shape)
cosh of every one of the row-major scalars at data, of shape shape: std::cosh of each.
T elementary_cos(const T &x)
cos of a scalar, as std::cos gives it.
bool radix_passes(U *keys, U *key_spare, std::int64_t *indices, std::int64_t *index_spare, std::size_t count, std::vector< byte_counts< U > > &counts)
Sorts the count keys at keys stably by their bytes from the least significant, the indices at indices...
std::int64_t extreme_index(const T *first, std::size_t count, const Order &larger)
The index of the first largest (larger std::greater) or smallest of count contiguous elements,...
bool numpy_fuses_products() noexcept
Whether NumPy's complex product fuses its multiply-adds on this processor: on arm64,...
tensor< R > fold_along(const tensor< T > &t, std::size_t axis, const R &init, const Step &step)
Folds each line of t along axis in order, step(running, x) from init: a tensor of the result type R,...
C numpy_multiply(const C &a, const C &b, bool fused)
a · b of complex numbers as NumPy's array loop computes it: ai·bi and ai·br rounded,...
T elementary_expm1(const T &x)
expm1 of a scalar, as std::expm1 gives it.
T chunked_conjugate_dot(const T *a, const T *b, std::size_t count)
Σ conj(a[i]) · b[i] (zdotc): chunked_dot of each chunk by conjugate_dot_product, without a conjugated...
constexpr std::size_t elementwise_chunk
The elements one task of an element-wise operation takes, a quarter of a thread's work: a tensor of a...
tensor< T > sinh_of(const T *data, const std::vector< std::size_t > &shape)
sinh of every one of the row-major scalars at data, of shape shape: std::sinh of each.
bool numpy_fuses_quotients() noexcept
Whether NumPy's complex quotient fuses its multiply-adds, as its PyPI wheels are built: on macOS arm6...
void exp2_into(const double *in, double *out, std::size_t count)
exp2 of the count doubles at in into out, each elementary_exp2's: table_exp2 in vectors (with the FMA...
tensor< T > log1p_of(const T *data, const std::vector< std::size_t > &shape)
log1p of every one of the row-major scalars at data, of shape shape: std::log1p of each.
void sort_run(T *first, T *last, std::size_t parts=1)
Sorts [first, last) stably as sorts_before orders reals: the NaNs moved behind the rest in their orde...
double table_atan(double x) noexcept
atan(x): y = |x|, or 1/|x| past 1 (its rounding error carried), c = k/64 the nearest point of atan_ta...
auto with_decimal_rounding(int decimals, const F &f)
f(rounding) with the decimal_rounding of R to decimals places: in R, as NumPy's, but in double for a ...
auto times(const A &a, const B &b)
a · b.
T walked_extreme(const T *first, std::size_t count, std::size_t stride, const Order &larger)
extreme of a line walked element by element: a NaN wins, as in NumPy (every comparison with it is fal...
C numpy_multiply_late(const C &a, const C &b)
a · b of complex numbers as NumPy's reduction loop computes it where MSVC contracts it (its Windows w...
tensor< T > expm1_viewed(const tensor< T > &a)
expm1 of every element of a: of a contiguous copy (expm1_of).
tensor< C > numpy_square(const tensor< C > &a, const numpy_loops &loops=numpy_wheel_loops())
x ** 2 of a complex tensor as NumPy's square computes it (the loop of an array to the Python integer ...
tensor< T > runs_into(const tensor< T > &t, const Kernel &kernel)
kernel (kernel(in, out, count), of contiguous elements) of every element of t: a contiguous tensor at...
tensor< C > numpy_prod(const tensor< C > &t, std::size_t axis, const numpy_loops &loops=numpy_wheel_loops())
The products of a complex t along axis, each line in order from 1 (NumPy's identity,...
tensor< T > exp2_of(const T *data, const std::vector< std::size_t > &shape)
exp2 of every one of the row-major scalars at data, of shape shape: std::exp2 of each.
std::complex< R > complex_tanh(const std::complex< R > &z)
tanh(z): (sinh(re) cosh(re) + i sin(im) cos(im)) / (sinh(re)² + cos(im)²); past half_exp_bound the re...
constexpr bool by_reduction(std::size_t inner) noexcept
Whether extremes_along takes the reduction's path: an exact type, or an axis of single elements....
bool sorts_before(T a, T b) noexcept
Whether a sorts before b as NumPy sorts reals: ascending, NaNs last.
auto parallel_map(const T *data, const std::vector< std::size_t > &shape, const F &f) -> tensor< std::decay_t< decltype(f(std::declval< const T & >()))> >
f(x) for every one of the row-major elements at data, of shape shape (another array's storage,...
std::size_t triangle_split(std::size_t i, std::size_t rows, std::size_t cols, std::ptrdiff_t k, bool lower)
The first column of row i past the part to and below its k diagonal (lower), or the first column of t...
T contiguous_extreme(const T *first, std::size_t count, const Order &larger)
The largest (larger is std::greater) or smallest of count contiguous elements, or a NaN if one is amo...
double table_sinh(double x) noexcept
sinh(x) = (d − u)/2 (hyperbolic_parts), the sign of x: within 0.503 ulp of the exact value up to the ...
tensor< T > sin_viewed(const tensor< T > &a)
sin of every element of a: of a contiguous copy (sin_of).
tensor< T > atan_of(const T *data, const std::vector< std::size_t > &shape)
atan of every one of the row-major scalars at data, of shape shape: std::atan of each.
std::vector< std::size_t > partition_positions(std::vector< std::size_t > kths, std::size_t extent)
kths ascending and each once, every one checked against extent.
void atan_into(const double *in, double *out, std::size_t count)
atan of the count doubles at in into out, each elementary_atan's: table_atan in vectors (with the FMA...
T correlation_step(const T &sum, const T &x, const T &y)
sum + x · y of a real or an integer (wrapped) correlation's terms.
double table_exp(double x) noexcept
exp(x) for an x whose result is normal or infinite (exp_normal_bound or above): x = (128 e + j) ln 2/...
tensor< T > cos_of(const T *data, const std::vector< std::size_t > &shape)
cos of every one of the row-major scalars at data, of shape shape: std::cos of each.
tensor< T > exp2_viewed(const tensor< T > &a)
exp2 of every element of a: of a contiguous copy (exp2_of).
T extreme(const T *first, std::size_t count, std::size_t stride, const Order &larger)
The largest (larger is std::greater) or smallest element of a line of reals: a contiguous one by the ...
void parallel_for(std::size_t count, std::size_t work_per_index, const Body &body)
Runs body(i) for every i in [0, count).
T from_order_key(order_key_t< T > key) noexcept
The real whose order_key is key.
auto as_laid_reduced(const tensor< T > &t, std::size_t axis, const F &f) -> std::optional< decltype(f(t, axis))>
as_laid_along of an f whose result takes the axis out, along any axis: a reduction of columns runs ac...
T conjugated_chunks(const T *a, const T *b, std::size_t count)
Σ a[i] · b[i] of reals, their own conjugates: chunked_dot.
std::conditional_t< sizeof(T)==8, std::uint64_t, std::uint32_t > order_key_t
The unsigned integer of T's width, whose order order_key maps T's onto.
T elementary_tan(const T &x)
tan of a scalar, as std::tan gives it.
double table_tanh(double x) noexcept
tanh(x) = t / (t + 2), t = expm1(2|x|) as a pair (expm1_parts, its two parts summed exactly),...
gemm_path gemm_path_current() noexcept
The path the next double product takes: the most capable one under the cap that runs here (each path'...
constexpr double exp_normal_bound
ln 2^-1022 toward zero: below it, exp is subnormal, where table_exp's two scalings would round twice.
std::size_t merged_pieces(std::size_t threads) noexcept
The pieces parallel_sort_run sorts a line in for threads: the least power of four that many.
bool wins_any_order(T x) noexcept
A real's NaN, which wins numpy.argmax and numpy.argmin.
T elementary_log(const T &x)
log of a scalar, as std::log gives it.
T ieee_maximum(T a, T b) noexcept
The larger of a and b as IEEE 754-2019's maximum orders them: a NaN wins (a's where both are),...
tensor< double > by_runs(const tensor< double > &a, doubles_into into)
A function of every element of a, its kernel of doubles (runs_into).
void tanh_into(const double *in, double *out, std::size_t count)
tanh of the count doubles at in into out, each elementary_tanh's: table_tanh in vectors (with the FMA...
std::pair< T, T > floor_divmod(T a, T b) noexcept
a // b and a % b of reals as NumPy's divmod computes them (Python's: the quotient floored,...
constexpr double exp2_normal_bound
2^x below which exp2 is subnormal, where table_exp2's two scalings would round twice.
std::complex< R > complex_exp(const std::complex< R > &z)
exp(z): exp(re) times cos(im) + i sin(im), a real part past exp_bound taken off in steps of it (each ...
tensor< T > exp_viewed(const tensor< T > &a)
exp of every element of a: of a contiguous copy (exp_of).
tensor< T > expm1_of(const T *data, const std::vector< std::size_t > &shape)
expm1 of every one of the row-major scalars at data, of shape shape: std::expm1 of each.
void radix_sort(T *first, T *last, std::size_t parts=1)
Sorts [first, last) by order_key, a byte at a time from the least significant (by radix_passes,...
double table_log1p(double x) noexcept
ln(1 + x) by log_parts of 1 + x as a pair (its two-sum): near 0, where c = 1, r is x exactly....
T parallel_extreme(const T *first, std::size_t count, const Order &larger)
contiguous_extreme of count elements on the threads: runs of pairwise_leaf each its own,...
bool holds(const T *first, std::size_t count, const Match &match) noexcept
Whether one of the count contiguous elements at first passes match: tested without a branch,...
double table_cosh(double x) noexcept
cosh(x) = (d + u)/2 (hyperbolic_parts): within 0.502 ulp of the exact value up to the overflow; NaN f...
C library_power(const C &a, const C &b)
a ** b by std::pow: libstdc++'s calls the C library's cpow, which NumPy calls (the same bits under gl...
T elementary_exp(const T &x)
exp of a scalar, as std::exp gives it.
T elementary_sin(const T &x)
sin of a scalar, as std::sin gives it.
std::complex< R > complex_sqrt(const std::complex< R > &z)
The square root of a complex number, as the GNU C library's csqrt computes it (NumPy's on Linux,...
tensor< float > through_doubles(const float *data, const std::vector< std::size_t > &shape, Into into)
The function into (one of the *_into of doubles) of every one of the row-major floats at data,...
T masked_select(bool condition, T if_true, T if_false) noexcept
if_true where condition holds, else if_false, by a mask of their bits: no branch, so a loop of it vec...
std::size_t checked_mul(std::size_t a, std::size_t b)
a * b, throwing numerical::dimension_error if the product overflows std::size_t. Guards dimension/sha...
std::vector< std::size_t > without_axis(std::vector< std::size_t > shape, std::size_t axis)
The shape with axis axis removed.
tensor< T > atan_viewed(const tensor< T > &a)
atan of every element of a: of a contiguous copy (atan_of).
C numpy_divide(const C &a, const C &b, bool fused)
a / b of complex numbers as NumPy's array loop computes it: Smith's division by the larger part of b ...
void(*)(const double *, double *, std::size_t) doubles_into
A function of count contiguous doubles into as many (the *_into kernels).
T root(const T &x)
The square root of a scalar, as std::sqrt gives it.
void expm1_into(const double *in, double *out, std::size_t count)
expm1 of the count doubles at in into out, each elementary_expm1's: table_expm1 in vectors (with the ...
@ zero
C = A·B: C's elements before the product are not read.
void select_positions(It first, std::size_t count, const std::vector< std::size_t > &kths, const Less &less)
Rearranges [first, first + count) by less so that each of the ascending kths holds the element a sort...
T elementary_log1p(const T &x)
log1p of a scalar, as std::log1p gives it.
tensor< T > tanh_of(const T *data, const std::vector< std::size_t > &shape)
tanh of every one of the row-major scalars at data, of shape shape: elementary_tanh of each.
std::vector< std::size_t > view_axes(const std::vector< std::size_t > &order)
The axes that put the elements of a tensor laid out in order (tensor::laid_out, its view's axes in th...
void cosh_into(const double *in, double *out, std::size_t count)
cosh of the count doubles at in into out, each elementary_cosh's: table_cosh in vectors (with the FMA...
std::size_t element_count(It first, It last)
The elements a shape's extents [first, last) hold: 0 where one of them is 0, whatever the others,...
auto as_laid_along(const tensor< T > &t, std::size_t axis, const F &f) -> std::optional< decltype(f(t, axis))>
Along axis of a transpose (laid_along): f (f(elements, axis)) of its elements as they lie along the m...
matrix< T > integer_power(const matrix< T > &a, double e)
a to the integer power e (a finite whole number) by repeated squaring, as numpy.linalg....
tensor< T > reduce_along(const tensor< T > &t, std::size_t axis, const Reduce &reduce)
Reduces each line of t along axis by reduce, which takes the line's first element,...
std::size_t first_held(const T *first, std::size_t count, const Match &match) noexcept
The index of the first of the contiguous elements at first that passes match, which one of them does:...
tensor< T > extremes_along(const tensor< T > &t, std::size_t axis, const Order &larger)
The largest (larger is std::greater) or smallest elements along axis. Along an inner axis each line i...
order_key_t< T > order_key(T x) noexcept
x's bits as an unsigned integer ordered as x is by <: the sign bit set for a positive real,...
std::optional< laid_axis< T > > laid_along(const tensor< T > &t, std::size_t axis)
t laid out and the axis matching axis: nothing for any tensor but a transpose, or an axis past the ra...
order_key_t< T > argsort_key(T x) noexcept
A real's order_key for an argsort: both zeros the same key, as < holds them equal (their indices then...
tensor< T > scan_along(const tensor< T > &t, std::size_t axis, const Step &step)
Each line of t along axis replaced by its running step (x₀, step(x₀, x₁), …), in order.
void sort_shared(T *run, std::size_t count, std::size_t threads)
Sorts one long line of count elements at run among threads: each pass of its radix sort shared out (r...
numpy_loops numpy_wheel_loops() noexcept
NumPy's loops as its current PyPI wheels build them on this processor (detail::numpy_fuses_products,...
std::complex< R > complex_cos(const std::complex< R > &z)
cos(z), as cosh(i z): complex_cosh of −im + i re.
std::vector< std::size_t > reduced_axes(const std::vector< std::size_t > &order, std::size_t axis)
The axes that put a result along axis of a tensor laid out in order (view_axes), that axis taken out,...
std::array< std::array< std::size_t, 256 >, sizeof(U)> byte_counts
How many keys of U hold each value of each byte: the byte at bit 8·b counted in [b].
tensor< std::int64_t > extreme_indices(const tensor< T > &t, std::size_t axis, const Order &larger)
argmax or argmin along axis by larger.
constexpr std::size_t pairwise_leaf
The longest subtree of pairwise_sum's tree parallel_pairwise_sum sums on one thread: 256 KiB of doubl...
T elementary_sinh(const T &x)
sinh of a scalar, as std::sinh gives it.
C::value_type numpy_absolute(const C &x, bool fused)
NumPy's complex magnitude (its absolute's vector loop): the parts' magnitudes, an infinite one making...
T ieee_minimum(T a, T b) noexcept
The smaller of a and b as IEEE 754-2019's minimum orders them: a NaN wins (a's where both are),...
T chunked_dot(const T *a, const T *b, std::size_t count, const Dot &dot)
Σ a[i] · b[i] for a plain dot product: chunks of dot_chunk elements in parallel, each by dot,...
std::int64_t * numbers_end(std::int64_t *first, std::int64_t *last, const T *values, std::size_t stride)
Where a real's NaNs begin among the indices [first, last), moved behind the rest in their order.
tensor< T > tanh_viewed(const tensor< T > &a)
tanh of every element of a: of a contiguous copy (tanh_of).
void order_run(std::int64_t *first, std::int64_t *last, const T *values, std::size_t stride, const Less &less, std::size_t parts=1)
Sorts the indices [first, last) stably by their elements (values[k · stride]), as sorts_before orders...
tensor< T > exp_of(const T *data, const std::vector< std::size_t > &shape)
exp of every one of the row-major elements at data, of shape shape: elementary_exp of each.
tensor< T > sinh_viewed(const tensor< T > &a)
sinh of every element of a: of a contiguous copy (sinh_of).
void order_shared(std::int64_t *order, std::size_t count, const T *values, std::size_t stride, const Less &less, std::size_t threads)
Sorts the indices of one long line among threads: each pass of its radix sort shared out.
tensor< C > numpy_cumprod(const tensor< C > &t, std::size_t axis, const numpy_loops &loops=numpy_wheel_loops())
The running products of a complex t along axis, in order, each as NumPy's loops compute it (detail::n...
@ fused
std::fma, where every target has the instruction.
constexpr std::size_t select_positions_sorted
A line with more positions than these is sorted whole rather than selected position by position.
R numpy_muladd(R x, R y, R z, bool fused)
x · y + z: one fused multiply-add where fused, else the product rounded first (statements apart,...
C numpy_reduction_multiply(const C &a, const C &b, const numpy_loops &loops)
a · b as NumPy's reduction loop computes it, fused as loops says.
The NumericAL public API.
std::vector< T, detail::default_init_allocator< T > > buffer
The storage of vector, matrix and tensor, namely a std::vector whose buffer<T>(n) and resize(n) leave...
tensor< T > broadcast_to(const tensor< T > &t, std::vector< std::size_t > shape)
t repeated along its axes of extent 1 and new leading ones to shape, as numpy.broadcast_to (a copy: a...
T sum(const tensor< T > &t)
The sum of every element, by pairwise halving (0 for an empty tensor), on the threads where they are ...
std::array< tensor< std::int64_t >, 2 > triangle_indices(std::size_t rows, std::size_t cols, std::ptrdiff_t k, bool lower)
The row and the column indices of the rows × cols matrix's part to and below its k diagonal (lower) o...
matrix< T > operator+(const matrix< T > &a, const matrix< T > &b)
Element-wise sum.
tensor< std::int64_t > argsort(const tensor< T > &t, std::size_t axis)
The indices that sort the elements along axis, as numpy.argsort (its kind="stable"): of each line,...
tensor< T > log1p(const tensor< T > &a)
log(1 + x) of every real element, exact near 0: detail::log1p_into of a double one (within 0....
tensor< T > select_along_last(const tensor< T > &t, const tensor< std::int64_t > &positions)
Of each line along the last axis of t, the elements its stable sort (sort) puts at the positions posi...
tensor< T > log(const tensor< T > &a)
log of every element: std::log of a real one, detail::complex_log of a complex one.
tensor< double > linspace(double start, double stop, std::size_t count)
count values evenly spaced from start to stop, both included, as NumPy's linspace: element i is start...
tensor< T > clip(const tensor< T > &a, T low, T high)
a limited to [low, high] element by element, as numpy.clip of two numbers: minimum(maximum(a,...
matrix< T > operator*(const T &scalar, const matrix< T > &m)
Scalar times matrix.
tensor< T > exp(const tensor< T > &a)
exp of every element: detail::exp_into of a double one (within 0.506 ulp of the exact value),...
tensor< T > diag(const tensor< T > &v, std::ptrdiff_t k)
The square matrix with the vector v on its k diagonal (above the main one for a positive k) and zeros...
tensor< T > cosh(const tensor< T > &a)
cosh of every real element: detail::cosh_into of a double one (within 0.501 ulp of the exact value,...
tensor< T > operator/(const tensor< T > &a, const tensor< T > &b)
Element-wise quotient, broadcast.
tensor< std::int64_t > argpartition(const tensor< T > &t, const std::vector< std::size_t > &kths, std::size_t axis)
The indices that partition each line along axis, as numpy.argpartition: of the elements partition wou...
tensor< T > ones(std::vector< std::size_t > shape)
A tensor of shape full of ones.
tensor< T > correlate(const tensor< T > &a, const tensor< T > &v, std::ptrdiff_t first, std::size_t count)
The cross-correlation c[k] = Σₙ a[n + k] · conj(v[n]) of two vectors over the lags first … first + co...
tensor< T > triangle(const tensor< T > &t, std::ptrdiff_t k, bool lower)
The matrices of t (its last two axes) zero above their k diagonal (lower) or below it (else),...
tensor< T > take_along_axis(const tensor< T > &t, const tensor< std::int64_t > &indices, std::size_t axis)
The elements of t at indices along axis, the other axes matched, as numpy.take_along_axis: indices of...
tensor< std::int64_t > searchsorted(const tensor< T > &a, const tensor< T > &v, bool right=false)
Where each element of v would go in the sorted line a to keep it sorted, as numpy....
tensor< T > cumsum(const tensor< T > &t, std::size_t axis)
The running sums along axis, in order, as numpy.cumsum.
tensor< T > expm1(const tensor< T > &a)
exp(x) − 1 of every real element, exact near 0: detail::expm1_into of a double one (within 0....
tensor< T > maximum(const tensor< T > &a, const tensor< T > &b)
The larger element of a and b, broadcast, as numpy.maximum (detail::ieee_maximum; a NaN wins,...
tensor< T > take(const tensor< T > &t, const tensor< std::int64_t > &indices, std::size_t axis, take_mode mode)
The elements of t at indices along axis, as numpy.take: the result's shape t's with the axis replaced...
tensor< T > minimum(const tensor< T > &a, const tensor< T > &b)
The smaller element of a and b, broadcast, as numpy.minimum (detail::ieee_minimum; a NaN wins,...
T prod(const tensor< T > &t)
The product of every element, in order (numpy.prod's, whose multiplications are sequential: only its ...
gemm_path
The kernels a double matrix product can run on (a float one too on arm64), from the most portable....
@ amx
The Apple AMX coprocessor, fused multiply-adds; the products it does not suit on the NEON path's tile...
T max(const tensor< T > &t)
The largest element (a NaN, if there is one), of a transpose's elements as they lie (in any order the...
auto broadcast_apply(const tensor< T > &a, const tensor< T > &b, const Op &op)
op(a, b) element by element, the operands broadcast against each other, in op's result type (a compar...
tensor< T > sqrt(const tensor< T > &a)
The square root of every element: std::sqrt of a real one, detail::complex_sqrt of a complex one.
std::vector< tensor< std::int64_t > > nonzero(const tensor< T > &t)
The indices of the nonzero elements (x != T {}: a NaN is one), one tensor per axis,...
tensor< T > exp2(const tensor< T > &a)
2^x of every real element: detail::exp2_into of a double one (within 0.506 ulp of the exact value whe...
tensor< T > cumprod(const tensor< T > &t, std::size_t axis)
The running products along axis, in order, as numpy.cumprod.
tensor< T > zeros(std::vector< std::size_t > shape)
A tensor of shape full of zeros (each thread's share zeroed as the C library zeroes).
matrix< T > operator-(const matrix< T > &a, const matrix< T > &b)
Element-wise difference.
auto map(const tensor< T > &t, const F &f) -> tensor< std::decay_t< decltype(f(std::declval< const T & >()))> >
f(x) for every element, in a tensor of the same shape and of f's result type.
tensor< T > cos(const tensor< T > &a)
cos of every element: std::cos of a real one (a float the double's, detail::cos_of),...
tensor< double > arange(double start, double stop, double step, bool fused=true)
start, start + step, … below stop (above it for a negative step), as NumPy's arange: its length is ce...
tensor< T > eye(std::size_t n)
The n × n identity.
std::size_t count_nonzero(const tensor< T > &t)
The number of nonzero elements (x != T {}: a NaN is one), as numpy.count_nonzero without an axis: by ...
tensor< T > sinh(const tensor< T > &a)
sinh of every real element: detail::sinh_into of a double one (within 0.503 ulp of the exact value,...
tensor< T > round(const tensor< T > &a, int decimals=0)
Every element rounded to decimals places (tens, hundreds for a negative one), half to even,...
tensor< T > sort(const tensor< T > &t, std::size_t axis)
The elements sorted along axis, as numpy.sort: ascending, NaNs last, equal elements in their order (N...
tensor< std::int64_t > argsort_by(const tensor< std::int64_t > &order, const tensor< T > &key, std::size_t axis)
order, indices along axis into key of its shape, each line's sorted stably by the elements of key the...
tensor< T > atan(const tensor< T > &a)
atan of every real element: detail::atan_into of a double one (within 0.501 ulp of the exact value),...
tensor< T > reshape(const tensor< T > &t, std::vector< std::size_t > shape)
The same elements in the same order under another shape, shared with t (no copy).
tensor< T > diagonal(const tensor< T > &t, std::ptrdiff_t offset)
The offset diagonal of each matrix of t (its last two axes), as numpy.linalg.diagonal: above the main...
take_mode
What take does with an index outside its axis, as numpy.take's mode: refuse it, take it modulo the ex...
T mean(const tensor< T > &t)
The mean of every element. An empty tensor's mean is NaN, as NumPy's (which also warns).
tensor< T > tanh(const tensor< T > &a)
tanh of every element: detail::tanh_into of a double one (within 0.502 ulp of the exact value),...
tensor< T > floor_divide(const tensor< T > &a, const tensor< T > &b)
a // b element by element, broadcast, as numpy.floor_divide (detail::floor_divmod).
T min(const tensor< T > &t)
The smallest element (a NaN, if there is one), of a transpose's elements as they lie (in any order th...
std::int64_t argmax(const tensor< T > &t)
The flat index of the first largest element, as numpy.argmax (the first NaN where there is one).
std::int64_t argmin(const tensor< T > &t)
The flat index of the first smallest element, as numpy.argmin (the first NaN where there is one).
tensor< T > tan(const tensor< T > &a)
tan of every element: std::tan of a real one, detail::complex_tan of a complex one.
tensor< T > remainder(const tensor< T > &a, const tensor< T > &b)
a % b element by element, broadcast, as numpy.remainder (Python's: of b's sign; a real by zero fmod's...
T vdot(const tensor< T > &a, const tensor< T > &b)
Σ conj(a[i]) · b[i] over every element of a and b in their row-major order, as numpy....
tensor< T > sin(const tensor< T > &a)
sin of every element: std::sin of a real one (a float the double's, detail::sin_of),...
tensor< T > full(std::vector< std::size_t > shape, const T &value)
A tensor of shape with every element value.
tensor< T > partition(const tensor< T > &t, const std::vector< std::size_t > &kths, std::size_t axis)
The elements of each line along axis rearranged as numpy.partition: at each of kths the element sort ...
tensor< T > pow(const tensor< T > &a, const tensor< T > &b)
a to the power b element by element, broadcast, as std::pow; a single exponent of 2,...
The native vector of float and double the kernels hold their lanes in (GNU vector extensions,...
#define NUMERICAL_INLINE_LAMBDA
T operator()(T a, T b) const noexcept
T operator()(T a, T b) const noexcept
T operator()(T a) const noexcept
T operator()(T a, T b) const noexcept
std::make_unsigned_t< decltype(T {}+0U)> unsigned_type
Wide enough that no promotion is signed.
The element operations of T the tensor operators apply: the standard ones.
std::multiplies< T > multiplies
a · b.
std::plus< T > plus
a + b.
std::negate< T > negate
-a.
std::minus< T > minus
a - b.
The extents and strides that walk a tensor of shape with axis axis taken out: outer positions before ...
std::size_t outer
The product of the extents before the axis.
std::size_t inner
The product of the extents after the axis.
std::size_t extent
The axis' extent.
A real of type R rounded to decimals places, half to even, as numpy.round: nearbyint(x · 10ᵈ) / 10ᵈ (...
decimal_rounding(int places) noexcept
R operator()(R x) const noexcept
C exact_from
The magnitude from which a value is its own rounding (decimals above 0).
C scale
10 to the magnitude of decimals, in C.
int decimals
The places; fewer than none round to tens, hundreds and so on.
A transpose laid out (tensor::laid_out) and the axis matching one of its view's.
tensor< T > elements
Its elements row-major as they lie.
std::vector< std::size_t > order
Its view's axes in the order they lie.
static double call(double x)
std::sin(x).
static double call(double x)
std::cos(x).
The C library's sin (Cosine false) or cos, call(x): one class each, so that each is covered whole.
Which of NumPy's complex loops fuse their multiply-adds: its product's vector loop,...
bool late_reductions
Where it fuses, as detail::numpy_multiply_late does (MSVC's contraction).
bool reductions
The product's reduction loop (detail::numpy_fuses_reductions).
bool scalar_products
The product's scalar loop.
bool quotients
The quotient (Smith's algorithm, a scalar loop).
bool products
The product's vector loop.
A *_into kernel of doubles as a kernel of floats: each piece of 512 converted to doubles on the stack...
doubles_into into
The kernel of doubles.
void operator()(const float *in, float *out, std::size_t count) const
The kernel of the count floats at in into out.
A dense, row-major, N-dimensional tensor and Einstein summation.