buffer.hpp

Go to the documentation of this file.

 
#ifndef NUMERICAL_BUFFER_HPP
#define NUMERICAL_BUFFER_HPP
 
#include <algorithm>
#include <atomic>
#include <cstddef>
#include <cstring>
#include <new>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>
 
#if defined(__linux__)
#  include <sys/mman.h>
#endif
 
namespace numerical {
 
  namespace detail {
 
    template <typename T>
    struct default_init_allocator : std::allocator<T>
    {
      using value_type = T; 
 
      template <typename U>
      struct rebind
      {
        using other = default_init_allocator<U>; 
      };
 
      default_init_allocator() noexcept = default;
 
      template <typename U>
      // NOLINTNEXTLINE(hicpp-explicit-conversions,google-explicit-constructor): the containers rebind an allocator by this conversion
      explicit (false) default_init_allocator(const default_init_allocator<U>& /*other*/) noexcept
      {}
 
#if defined(__linux__)
      static constexpr std::size_t huge_page {std::size_t {1} << 21U};
 
      static constexpr std::size_t huge_least {std::size_t {1} << 24U};
 
      static constexpr std::size_t huge_colours {64};
 
      [[nodiscard]] static constexpr bool on_huge_pages(std::size_t n) noexcept
      {
        return n <= (static_cast<std::size_t>(-1) - huge_page) / sizeof(T) && n * sizeof(T) >= huge_least;
      }
 
      [[nodiscard]] T* allocate(std::size_t n)
      {
        if (!on_huge_pages(n)) {
          return std::allocator<T>::allocate(n);
        }
        static std::atomic<std::size_t> colour {0};
        const std::size_t               offset {((colour.fetch_add(1) % huge_colours) * 4096) + alignof(std::max_align_t) * 4};
        const std::size_t               bytes  {((offset + (n * sizeof(T))) + huge_page - 1) / huge_page * huge_page};
        auto* const                     start  {static_cast<std::byte*>(::operator new(bytes, std::align_val_t {huge_page}))};
        (void)madvise(start, bytes, MADV_HUGEPAGE);                            // a hint: the pages stay 4 KiB where the system declines
        std::byte* const data                  {start + offset};
        std::memcpy(data - sizeof(std::size_t), &offset, sizeof(std::size_t)); // the start, as the data's distance from it
        return reinterpret_cast<T*>(data);                                     // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): storage for T, aligned past max_align_t
      }
 
      void deallocate(T         * at,
                      std::size_t n) noexcept
      {
        if (!on_huge_pages(n)) {
          std::allocator<T>::deallocate(at, n);
          return;
        }
        deallocate_huge(at);
      }
 
      [[gnu::noinline]] static void deallocate_huge(T* at) noexcept
      {
        auto* const data   {reinterpret_cast<std::byte*>(at)}; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast): the storage's bytes
        std::size_t offset {};
        std::memcpy(&offset, data - sizeof(std::size_t), sizeof(std::size_t));
        ::operator delete(data - offset,
                          std::align_val_t {huge_page});
      }
 
#endif
 
      template <typename U>
      void construct(U* at) noexcept(std::is_nothrow_default_constructible_v<U>)
      {
        ::new (static_cast<void*>(at)) U;
      }
 
      template <typename U,
                typename ... Args>
      void construct(U   *     at,
                     Args&&... args)
      {
        ::new (static_cast<void*>(at)) U(std::forward<Args>(args)...);
      }
    };
 
    template <typename T>
    [[nodiscard]] bool operator==(const std::vector<T, default_init_allocator<T>>& buffer,
                                  const std::vector<T>&                            other)
    {
      return std::equal(buffer.begin(), buffer.end(), other.begin(), other.end());
    }
  } // namespace detail
 
  template <typename T>
  using buffer = std::vector<T, detail::default_init_allocator<T>>;
} // namespace numerical
 
#endif // NUMERICAL_BUFFER_HPP