6#ifndef NUMERICAL_BUFFER_HPP
7#define NUMERICAL_BUFFER_HPP
26#if !defined(NUMERICAL_OUT_OF_LINE)
27# if defined(_MSC_VER) && !defined(__clang__)
28# define NUMERICAL_OUT_OF_LINE __declspec(noinline)
30# define NUMERICAL_OUT_OF_LINE [[gnu::noinline]]
80 for (std::size_t i {0}; i <
registered.count; ++i) {
119 if (b.start !=
nullptr) {
132 std::size_t&
got)
noexcept
134 const std::lock_guard<std::mutex>
held {
lock_};
137 if (b.start !=
nullptr && b.bytes >= bytes && b.bytes / 2 < bytes && (
best ==
nullptr || b.bytes <
best->bytes)) {
141 if (
best ==
nullptr) {
145 return std::exchange(
best->start,
nullptr);
154 std::size_t bytes)
noexcept
161 const std::lock_guard<std::mutex>
held {
lock_};
164 if (b.start ==
nullptr || b.order <
place->order) {
166 if (b.start ==
nullptr) {
184 std::array<block, Places>
freed {};
186 const std::lock_guard<std::mutex>
held {
lock_};
187 freed = std::exchange(
kept_, std::array<block, Places> {});
190 if (b.start !=
nullptr) {
210#if defined(__linux__)
212 using kept_huge_blocks = kept_blocks<std::size_t {1} << 21U, 2, std::size_t {1} << 26U>;
238 template <
typename T>
242 template <std::
floating_po
int R>
252 template <
typename T>
258 template <
typename U>
272#if defined(__linux__)
274 static constexpr std::size_t
huge_page {kept_huge_blocks::alignment};
314#if defined(__linux__)
316 static std::atomic<std::size_t>
colour {0};
317 const std::size_t offset {((
colour.fetch_add(1) %
huge_colours) * 4096) +
alignof(std::max_align_t) * 4};
319 std::byte* start {kept_huge_blocks::shared().take(bytes, bytes)};
320 if (start ==
nullptr) {
324 return placed(start, offset, bytes);
333 static std::atomic<std::size_t>
colour {0};
337 if (start ==
nullptr) {
338 start =
static_cast<std::byte*
>(
::operator new(bytes, std::align_val_t {
page}));
340 return placed(start, offset, bytes);
342 return std::allocator<T>::allocate(n);
347 std::size_t n)
noexcept
349#if defined(__linux__)
359 std::allocator<T>::deallocate(at, n);
365 std::size_t bytes)
noexcept
367 std::byte*
const data {start + offset};
368 std::memcpy(data -
sizeof(std::size_t), &offset,
sizeof(std::size_t));
369 std::memcpy(data - (2 *
sizeof(std::size_t)), &bytes,
sizeof(std::size_t));
370 return reinterpret_cast<T*
>(data);
378 template <
typename Kept>
381 auto*
const data {
reinterpret_cast<std::byte*
>(at)};
382 std::size_t offset {};
383 std::size_t bytes {};
384 std::memcpy(&offset, data -
sizeof(std::size_t),
sizeof(std::size_t));
385 std::memcpy(&bytes, data - (2 *
sizeof(std::size_t)),
sizeof(std::size_t));
386 if (!Kept::shared().keep(data - offset, bytes)) {
388 std::align_val_t {Kept::alignment});
393 template <
typename U>
395 void construct(
U* at)
noexcept(std::is_nothrow_default_constructible_v<U>)
397 ::new (
static_cast<void*
>(at))
U;
404 template <
typename U>
409 const typename U::value_type
nan {std::numeric_limits<typename U::value_type>::quiet_NaN()};
410 const std::array<typename U::value_type, 2> parts {
nan,
nan};
411 std::memcpy(
static_cast<void*
>(at), parts.data(),
sizeof(parts));
416 template <
typename U,
421 ::new (
static_cast<void*
>(at))
U(std::forward<Args>(
args)...);
429 template <
typename T>
431 const std::vector<T>& other)
433 return std::equal(
buffer.begin(),
buffer.end(), other.begin(), other.end());
442 template <
typename T>
443 using buffer = std::vector<T, detail::default_init_allocator<T>>;
#define NUMERICAL_OUT_OF_LINE
Blocks freed and kept for the next buffer that fits one: a fresh block's pages are each faulted in an...
static kept_blocks & shared()
The process's blocks, never destroyed: a buffer of static storage may be freed after any destructor.
static constexpr std::size_t alignment
The alignment of a block.
bool keep(std::byte *start, std::size_t bytes) noexcept
Keeps the block at start of bytes if it is under largest, in place of the block kept longest when eve...
std::array< block, Places > kept_
kept_blocks(kept_blocks &&)=delete
kept_blocks & operator=(const kept_blocks &)=delete
void clear() noexcept
Frees every block kept (numerical::release_memory).
static constexpr std::size_t largest
The bytes from which a freed block is freed at once, not kept.
std::byte * take(std::size_t bytes, std::size_t &got) noexcept
A kept block of at least bytes and under twice as many, its size in got; nullptr if none.
kept_blocks & operator=(kept_blocks &&)=delete
kept_blocks(const kept_blocks &)=delete
~kept_blocks()
Frees the blocks still kept.
bool on_release(void(*step)()) noexcept
Registers step among the calling thread's thread_releases, while a place is left (a workspace past th...
constexpr std::size_t kept_pages_least
The fewest bytes a buffer takes a kept block of 4 KiB pages for (kept_page_blocks): below,...
constexpr std::size_t thread_release_places
The most workspaces a thread registers (on_release): one per kind, scalar type and slot,...
void release_thread_memory()
Empties every workspace the calling thread keeps between calls: its thread_releases,...
constexpr std::size_t kept_pages_largest
The bytes under which a buffer's block is a kept one, no huge pages asked: 32 MiB,...
constexpr bool is_complex_number
Whether T is a std::complex of a floating-point type.
bool operator==(const std::vector< T, default_init_allocator< T > > &buffer, const std::vector< T > &other)
Element-wise equality of a numerical::buffer and a std::vector, found by argument-dependent lookup th...
thread_release_steps & thread_releases() noexcept
The calling thread's thread_release_steps.
The NumericAL public API.
std::vector< T, detail::default_init_allocator< T > > buffer
The storage of vector, matrix and tensor, namely a std::vector whose buffer<T>(n) and resize(n) leave...
The allocator of another element type, as std::allocator_traits rebinds it.
The allocator of numerical::buffer. An element constructed with no value is default-initialised,...
static void deallocate_kept(T *at) noexcept
Frees the block allocate gave from at, kept by Kept where it takes it. Out of line: inlined where a s...
void construct(U *at) noexcept
construct of a complex number, whose default constructor zeroes: left as its memory holds it,...
void construct(U *at, Args &&... args)
Constructs the element at at from args.
static constexpr std::size_t page_colours
The data's offset past a kept block's start goes round 16 multiples of 256 bytes past a cache line th...
static T * placed(std::byte *start, std::size_t offset, std::size_t bytes) noexcept
The data at offset into the block at start of bytes, the offset and the size kept before it.
T value_type
The element type.
void deallocate(T *at, std::size_t n) noexcept
Frees what allocate gave for n elements.
void construct(U *at) noexcept(std::is_nothrow_default_constructible_v< U >)
Default-initialises the element at at.
default_init_allocator() noexcept=default
static constexpr bool on_kept_pages(std::size_t n) noexcept
Whether n elements take a block of 4 KiB pages kept when freed (kept_page_blocks): from kept_pages_le...
T * allocate(std::size_t n)
n elements: on huge pages from their least bytes on Linux, in a kept block of 4 KiB pages from kept_p...
std::size_t order
When it was kept, the least the longest.
std::byte * start
The block's start, nullptr for a free place.
std::size_t bytes
Its size.
The calling thread's steps that empty the workspaces it keeps between calls, each registered by the w...
std::size_t count
How many.
std::array< void(*)(), thread_release_places > steps
The steps, in their order of registration.