14#ifndef NUMERICAL_PARALLEL_HPP
15#define NUMERICAL_PARALLEL_HPP
21#include <condition_variable>
30#include <system_error>
46#elif defined(__APPLE__)
47# include <sys/sysctl.h>
57 return static_cast<long>(
_getpid());
59 return static_cast<long>(
getpid());
64 inline constexpr std::size_t
no_core {std::numeric_limits<std::size_t>::max()};
75 for (std::size_t c {0}; c <
cores; ++c) {
84 std::size_t
caller)
noexcept
106 const std::string&
cpu_root)
noexcept
109 std::map<std::pair<long, long>, std::size_t> index;
110 std::vector<cpu_set_t>
sets;
117 const long package {std::stol(first_line(topology + "physical_package_id"))};
148 for (std::size_t c {0}; c <
cores.size(); ++c) {
172 std::size_t index)
noexcept
174 if (index <
cores.size()) {
187#if defined(__linux__)
189#elif defined(__APPLE__)
191 std::size_t size {
sizeof(
cores)};
193 return static_cast<std::size_t
>(std::max(
cores, 0));
213 long long period)
noexcept
222 const auto number {[](std::string_view
digits,
long long& value) {
226 while (!
text.empty() && (
text.back() ==
'\n' ||
text.back() ==
' ')) {
227 text.remove_suffix(1);
229 const std::size_t
space {
text.find(
' ')};
238#if defined(__linux__)
248 std::size_t
least {0};
254 std::ifstream
self {
"/proc/self/cgroup"};
258 if (
line.rfind(
"0::", 0) == 0) {
267 for (
const char*
const v1 : {
"/sys/fs/cgroup/cpu",
"/sys/fs/cgroup/cpu,cpuacct"}) {
291#if defined(__linux__)
317 inline constexpr std::chrono::microseconds
spin_time {500};
320 template <
typename Ready>
323 const std::chrono::steady_clock::time_point
until {std::chrono::steady_clock::now() +
spin_time};
324 while (!
ready() && std::chrono::steady_clock::now() <
until) {
325 std::this_thread::yield();
349 std::atomic<std::size_t>
next {0};
358 for (std::size_t c {
next.fetch_add(1)}; c <
chunks; c =
next.fetch_add(1)) {
363 failed[c] = std::current_exception();
366 const std::lock_guard<std::mutex>
held {
lock};
389 std::size_t helpers)
noexcept
398 return rank < helpers;
450 static std::mutex lock;
466 std::chrono::milliseconds
wait)
474 const std::lock_guard<std::mutex>
held {
lock_};
478 for (
const std::unique_ptr<worker_bell>&
bell :
bells_) {
479 {
const std::lock_guard<std::mutex>
held {
bell->lock}; }
480 bell->rung.notify_one();
482 const std::chrono::steady_clock::time_point
until {std::chrono::steady_clock::now() +
wait};
484 std::this_thread::yield();
511 std::atomic<bool>& busy;
513 explicit release(std::atomic<bool>& flag) : busy {flag}
516 release(
const release&) =
delete;
517 release& operator=(
const release&) =
delete;
518 release(release&&) =
delete;
519 release& operator=(release&&) =
delete;
525 const auto job {std::make_shared<pool_job>()};
530 job->chunks = chunks;
531 job->helpers = helpers;
532#if defined(__linux__)
535 job->failed.resize(chunks);
537 const std::lock_guard<std::mutex>
held {
lock_};
541 for (std::size_t w {0}; w <
bells_.size(); ++w) {
543#if defined(__linux__)
548 if (
bell.asleep.load()) {
550 bell.rung.notify_one();
554 if (
side !=
nullptr) {
566 std::unique_lock<std::mutex>
held {
job->lock};
572 for (
const std::exception_ptr&
error :
job->failed) {
574 std::rethrow_exception(
error);
602 static std::atomic<thread_pool*>
pool {
nullptr};
615#if defined(__linux__)
620 for (std::size_t w {
bells_.size()}; w <
wanted.size(); ++w) {
621 const std::size_t core {
wanted[w]};
622 auto bell {std::make_unique<worker_bell>(core)};
624 std::thread {[
this,
cores, core, rung] {
634 for (std::size_t w {
bells_.size()}; w <
wanted.size(); ++w) {
635 auto bell {std::make_unique<worker_bell>(
wanted[w])};
637 std::thread {[
this, rung] {
serve(*rung); }}.detach();
649 std::size_t
seen {0};
651#if defined(__linux__)
656#if defined(__linux__)
664 bell.asleep.store(
true);
665 std::unique_lock<std::mutex>
held {
bell.lock};
667 bell.asleep.store(
false);
677 std::shared_ptr<pool_job>
job;
679 const std::lock_guard<std::mutex>
held {
lock_};
683#if defined(__linux__)
689#if defined(__linux__)
697 if (
wanted &&
job->joined.fetch_add(1) <
job->helpers) {
704 std::vector<std::unique_ptr<worker_bell>>
bells_;
713#if defined(__linux__)
722 static std::atomic<std::size_t>
cap {0};
753 ? std::numeric_limits<std::size_t>::max()
789 template <
typename Body>
795 const auto runner {[](
const void*
erased, std::size_t begin, std::size_t end) {
797 for (std::size_t i {begin}; i < end; ++i) {
806 const std::size_t chunks {std::max<std::size_t>(1, std::min(count,
threads * 4))};
807 const std::size_t chunk {((count + chunks) - 1) / chunks};
825 template <
typename Body,
834 const auto runner {[](
const void*
erased, std::size_t begin, std::size_t end) {
836 for (std::size_t i {begin}; i < end; ++i) {
842 const std::size_t chunks {std::max<std::size_t>(1, std::min(count,
threads * 4))};
843 const std::size_t chunk {((count + chunks) - 1) / chunks};
853 inline constexpr std::size_t
zero_chunk {std::size_t {1} << 12};
862 template <
typename T>
865#if defined(__linux__)
866 constexpr std::size_t
huge_page {std::size_t {1} << 21U};
867 constexpr std::size_t
huge_least {std::size_t {1} << 24U};
890 template <
typename T>
895 std::fill(out, out + count,
T {});
900 const std::size_t begin {chunk *
piece};
901 std::fill(out + begin, out + std::min(count, begin +
piece),
T {});
907 template <
typename T>
913 std::fill(out, out + count, value);
918 const std::size_t begin {chunk *
piece};
919 std::fill(out + begin, out + std::min(count, begin +
piece), value);
929 template <
typename T>
935 std::copy(
in,
in + count, out);
940 const std::size_t begin {chunk *
piece};
941 std::copy(
in + begin,
in + std::min(count, begin +
piece), out + begin);
955 detail::thread_cap().store(threads);
965 const char* omp)
noexcept
967 const auto trimmed = [](std::string_view text)
noexcept {
968 while (!text.empty() && (text.front() ==
' ' || text.front() ==
'\t')) {
969 text.remove_prefix(1);
971 while (!text.empty() && (text.back() ==
' ' || text.back() ==
'\t')) {
972 text.remove_suffix(1);
976 const auto count = [](std::string_view digits)
noexcept {
977 std::size_t value {0};
978 for (
const char c : digits) {
979 if (c <
'0' || c >
'9') {
980 return std::size_t {0};
982 value = std::min<std::size_t>((value * 10) +
static_cast<std::size_t
>(c -
'0'), std::size_t {1} << 31U);
990 const std::string_view list {omp ==
nullptr ?
"" : omp};
991 return count(trimmed(std::string_view {list.data(), std::min(list.size(), list.find(
','))}));
997 const std::size_t cap {detail::thread_cap().load()};
998 const std::size_t hardware {detail::hardware_threads()};
999 return cap == 0 ? hardware : std::min(cap, hardware);
The threads every kernel shares, started once: a parallel_for costs a wake-up, not a thread start.
std::mutex lock_
Guards current_ and generation_.
thread_pool & operator=(thread_pool &&)=delete
bool release_on_workers(void(*release)(), std::chrono::milliseconds wait)
Runs release on every worker, each on its own thread between two runs, never inside one,...
static void unlock_after_fork() noexcept
pthread_atfork's handler after a fork, in the parent and in the child: the start lock released.
static thread_pool * started() noexcept
The pool if shared has started it, else nullptr (asking starts nothing).
thread_pool & operator=(const thread_pool &)=delete
std::shared_ptr< pool_job > current_
The latest run.
static thread_pool & shared()
The pool, started on first use with a worker per hardware thread beside the caller.
thread_pool(thread_pool &&)=delete
std::size_t workers_
The threads started.
thread_pool(const thread_pool &)=delete
std::mutex release_lock_
One release request at a time.
static void lock_for_fork() noexcept
The started pool, nullptr before.
std::atomic< void(*)()> release_
What a release request runs on each worker.
std::atomic< bool > busy_
Whether a run holds the pool.
void start()
Starts the workers (after construction: each uses the pool), worker_cores: on Linux one per core,...
bool try_run(chunk_runner run, const void *body, std::size_t count, std::size_t chunk, std::size_t chunks, std::size_t helpers, side_runner side=nullptr, const void *side_body=nullptr)
Runs chunks chunks of chunk indices up to count on the caller and at most helpers workers,...
std::vector< std::unique_ptr< worker_bell > > bells_
Each worker's, added once it runs; read once the pool is published.
std::atomic< std::size_t > releases_
Counts the release requests.
std::atomic< std::size_t > released_
Counts the workers' acknowledgements.
void serve(worker_bell &bell)
A worker on core: waits for a run (polling a while, then asleep), works on it unless it sits it out,...
static std::atomic< thread_pool * > & instance() noexcept
static std::mutex & start_lock() noexcept
The lock under which shared makes and starts the pool, and which a fork takes before it forks.
long owner_
The process whose threads they are.
std::atomic< std::size_t > generation_
Counts the runs (polled without the lock).
What every exception NumericAL throws is, beside its standard base (all but std::bad_alloc and std::s...
constexpr std::size_t no_core
No core: a worker free to run anywhere, or a caller on a processor the pool does not know.
constexpr std::size_t zero_chunk
The elements one index of parallel_zero writes: eight 4 KiB pages of double.
void parallel_zero(T *out, std::size_t count)
Sets the count elements at out to zero, split across the threads the work pays for: a fresh result's ...
void parallel_for_beside(std::size_t count, std::size_t work_per_index, const Body &body, const Side &side)
Runs side on the caller while the pool's workers run body(i) for every i in [0, count),...
constexpr bool sits_out(std::size_t worker, std::size_t caller) noexcept
Whether the worker pinned to core worker sits out a run whose caller is on core caller: one thread pe...
constexpr std::size_t zero_work
The work of zeroing one element, in multiply-adds: a second thread from 1 MiB of double....
constexpr std::size_t quota_processors(long long quota, long long period) noexcept
The processors a CPU quota of quota microseconds a period allows, rounded up (cgroup v1's cpu....
std::vector< std::size_t > worker_cores(std::size_t cores, std::size_t threads)
The core of each worker the pool starts beside its callers, from the cores known (0: none) and its th...
std::size_t parallel_threads(std::size_t count, std::size_t work_per_index)
The threads parallel_for shares count indices of work_per_index out to (threads_for): the hardware's,...
std::size_t cpu_max_processors(std::string_view text) noexcept
quota_processors of cgroup v2's cpu.max text, "<quota> <period>" ("max" for none): 0 for none or a te...
constexpr std::size_t vector_speedup
The multiply-adds a vectorised inner loop does in the time of one in work_per_thread's dependent chai...
constexpr bool wants(std::size_t core, std::size_t caller, std::size_t helpers) noexcept
Whether the worker on core joins a run of helpers workers whose caller is on caller: by rank among th...
void(*)(const void *side) side_runner
A task the caller of a run does before it takes chunks (parallel_for_beside), type-erased.
constexpr std::size_t work_per_thread
The work one thread must have to pay for starting it: about 60 µs of multiply-adds,...
void parallel_copy(const T *in, std::size_t count, T *out)
Copies the count elements at in to out, split across the threads the work pays for,...
constexpr std::size_t elementwise_chunk
The elements one task of an element-wise operation takes, a quarter of a thread's work: a tensor of a...
constexpr std::chrono::microseconds spin_time
How long a thread polls before it sleeps: kernels run in quick succession (a matrix product packs,...
std::size_t threads_for(std::size_t count, std::size_t work_per_index, std::size_t hardware)
How many threads count indices of work_per_index each pay for: one per work_per_thread,...
void(*)(const void *body, std::size_t begin, std::size_t end) chunk_runner
A chunk of a parallel_for, type-erased: runs body(i) for i in [begin, end).
constexpr std::size_t pool_threads(std::size_t logical, std::size_t physical, std::size_t quota=0) noexcept
The pool's threads from the logical hardware threads, the physical cores and the processors a CPU quo...
void spin_until(const Ready &ready)
Polls ready, yielding the core between polls, until it holds or spin_time has passed.
void parallel_for(std::size_t count, std::size_t work_per_index, const Body &body)
Runs body(i) for every i in [0, count).
long current_process() noexcept
This process's identifier: a pool started before a fork belongs to the parent, whose threads the chil...
void parallel_fill(T *out, std::size_t count, const T &value)
Sets the count elements at out to value, split across the threads as parallel_zero is,...
std::atomic< std::size_t > & thread_cap() noexcept
The most threads a parallel_for uses (0: as many as the pool has), set by numerical::set_max_threads.
std::size_t quota_limit() noexcept
The processors the CPU quotas over this process allow (cgroup_processors on Linux),...
std::size_t physical_cores() noexcept
The physical cores this process may run on, or 0 where the system does not say. Linux: core_sets; mac...
std::size_t hardware_threads() noexcept
The threads the pool runs, the caller's included (at least 1), known without starting the pool: one p...
constexpr std::size_t fault_chunk(std::size_t count) noexcept
The elements one index of parallel_zero or parallel_copy takes of count: zero_chunk,...
constexpr std::size_t memory_element_weight
The weight of an element a pass bound by the memory writes or moves, against a multiply-add,...
The NumericAL public API.
std::size_t max_threads()
The threads a kernel may use: the cap if one is set, else the hardware threads (starting none).
std::size_t environment_threads(const char *numerical, const char *omp) noexcept
The cap of threads the environment asks for, as the Python package and OpenBLAS read it: numerical (N...
void set_max_threads(std::size_t threads) noexcept
Caps the threads a kernel uses at threads, the caller's included (1: none beside it,...
T min(const tensor< T > &t)
The smallest element (a NaN, if there is one), of a transpose's elements as they lie (in any order th...
One parallel_for handed to the pool: its chunks, the next one to take, and how many are done....
chunk_runner run
The chunk runner.
std::atomic< std::size_t > finished
The chunks done.
std::size_t chunk
Indices per chunk.
std::size_t count
The number of iterations.
std::mutex lock
Orders the last chunk's signal against the caller's sleep.
std::size_t core
The caller's core: its worker sits it out (sits_out).
std::atomic< std::size_t > joined
The workers that asked to.
std::size_t helpers
The workers that may join it beside the caller.
std::size_t chunks
Chunks.
std::atomic< std::size_t > next
The next chunk to take.
void work() noexcept
Takes chunks until none is left, keeping the exception each throws.
std::condition_variable done
Signalled when the last chunk finishes.
const void * body
The body it is handed.
std::vector< std::exception_ptr > failed
The exception of each chunk, if it threw.
A worker's own wake-up: its core, whether it sleeps, and what wakes it.
const std::size_t core
The core it is pinned to, or no_core.
std::condition_variable rung
Signalled when a run that wants it starts.
std::atomic< bool > asleep
Set before it checks for a run and sleeps.
std::mutex lock
Orders a ring against its sleep.
worker_bell(std::size_t on)