TLA Line data Source code
1 : //
2 : // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3 : //
4 : // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 : // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 : //
7 : // Official repository: https://github.com/cppalliance/capy
8 : //
9 :
10 : #ifndef BOOST_CAPY_RECYCLING_MEMORY_RESOURCE_HPP
11 : #define BOOST_CAPY_RECYCLING_MEMORY_RESOURCE_HPP
12 :
13 : #include <boost/capy/detail/config.hpp>
14 :
15 : #include <bit>
16 : #include <cstddef>
17 : #include <memory_resource>
18 : #include <mutex>
19 :
20 : namespace boost {
21 : namespace capy {
22 :
23 : /** Recycling memory resource with size-class buckets.
24 :
25 : This memory resource recycles memory blocks using power-of-two
26 : size classes for O(1) allocation lookup. It maintains a thread-local
27 : pool for fast lock-free access and a global pool for cross-thread
28 : block sharing.
29 :
30 : Size classes: 64, 128, 256, 512, 1024, 2048 bytes.
31 : Allocations larger than 2048 bytes bypass the pools entirely.
32 :
33 : This is the default allocator used by run_async when no allocator
34 : is specified.
35 :
36 : @par Thread Safety
37 : Thread-safe. The thread-local pool requires no synchronization.
38 : The global pool uses a mutex for cross-thread access.
39 :
40 : @par Example
41 : @code
42 : auto* mr = get_recycling_memory_resource();
43 : run_async(ex, mr)(my_task());
44 : @endcode
45 :
46 : @see get_recycling_memory_resource
47 : @see run_async
48 : */
49 : BOOST_CAPY_MSVC_WARNING_PUSH
50 : BOOST_CAPY_MSVC_WARNING_DISABLE(4275) // non dll-interface base class
51 : class BOOST_CAPY_DECL recycling_memory_resource : public std::pmr::memory_resource
52 : {
53 : static constexpr std::size_t num_classes = 6;
54 : static constexpr std::size_t min_class_size = 64; // 2^6
55 : static constexpr std::size_t max_class_size = 2048; // 2^11
56 : static constexpr std::size_t bucket_capacity = 16;
57 :
58 : static std::size_t
59 HIT 29876 : round_up_pow2(std::size_t n) noexcept
60 : {
61 29876 : return n <= min_class_size ? min_class_size : std::bit_ceil(n);
62 : }
63 :
64 : static std::size_t
65 29876 : get_class_index(std::size_t rounded) noexcept
66 : {
67 29876 : std::size_t idx = std::countr_zero(rounded) - 6; // 64 = 2^6
68 29876 : return idx < num_classes ? idx : num_classes;
69 : }
70 :
71 : struct bucket
72 : {
73 : std::size_t count = 0;
74 : void* ptrs[bucket_capacity] = {};
75 :
76 19233 : void* pop() noexcept
77 : {
78 19233 : if(count == 0)
79 7030 : return nullptr;
80 12203 : return ptrs[--count];
81 : }
82 :
83 : // Peter Dimov's idea
84 7030 : void* pop(bucket& b) noexcept
85 : {
86 7030 : if(count == 0)
87 6516 : return nullptr;
88 4602 : for(std::size_t i = 0; i < count; ++i)
89 4088 : b.ptrs[i] = ptrs[i];
90 514 : b.count = count - 1;
91 514 : count = 0;
92 514 : return b.ptrs[b.count];
93 : }
94 :
95 21288 : bool push(void* p) noexcept
96 : {
97 21288 : if(count >= bucket_capacity)
98 8571 : return false;
99 12717 : ptrs[count++] = p;
100 12717 : return true;
101 : }
102 : };
103 :
104 : struct pool
105 : {
106 : bucket buckets[num_classes];
107 :
108 : // No destructor: a non-trivial dtor forces a guard variable on the
109 : // thread_local in local(), checked on every alloc/free. Constant
110 : // initialization plus a trivial dtor makes that access a bare TLS
111 : // load. Cached blocks are instead reclaimed explicitly: per-thread
112 : // by arm_thread_cleanup() at thread exit, and the global pool by
113 : // global()'s holder destructor at process exit.
114 : };
115 :
116 37203 : static pool& local() noexcept
117 : {
118 : static thread_local pool p;
119 37203 : return p;
120 : }
121 :
122 : static pool& global() noexcept;
123 : static std::mutex& global_mutex() noexcept;
124 :
125 : void* allocate_slow(std::size_t rounded, std::size_t idx);
126 : void deallocate_slow(void* p, std::size_t idx);
127 :
128 : // Register a thread-exit callback that drains this thread's local
129 : // pool back to the OS. Called only off the hot path: unconditionally
130 : // from the slow paths, and once per thread from deallocate_fast
131 : // behind a guard-free flag.
132 : static void arm_thread_cleanup() noexcept;
133 :
134 : public:
135 : ~recycling_memory_resource();
136 :
137 : /** Allocate without virtual dispatch.
138 :
139 : Handles the fast path inline (thread-local bucket pop)
140 : and falls through to the slow path for global pool or
141 : heap allocation.
142 : */
143 : void*
144 14938 : allocate_fast(std::size_t bytes, std::size_t)
145 : {
146 14938 : std::size_t rounded = round_up_pow2(bytes);
147 14938 : std::size_t idx = get_class_index(rounded);
148 14938 : if(idx >= num_classes)
149 MIS 0 : return ::operator new(bytes);
150 HIT 14938 : auto& lp = local();
151 14938 : if(auto* p = lp.buckets[idx].pop())
152 7908 : return p;
153 7030 : return allocate_slow(rounded, idx);
154 : }
155 :
156 : /** Deallocate without virtual dispatch.
157 :
158 : Handles the fast path inline (thread-local bucket push)
159 : and falls through to the slow path for global pool or
160 : heap deallocation.
161 : */
162 : void
163 14938 : deallocate_fast(void* p, std::size_t bytes, std::size_t)
164 : {
165 14938 : std::size_t rounded = round_up_pow2(bytes);
166 14938 : std::size_t idx = get_class_index(rounded);
167 14938 : if(idx >= num_classes)
168 : {
169 MIS 0 : ::operator delete(p);
170 0 : return;
171 : }
172 : // Guard-free flag (constinit bool, trivial dtor): arms thread-exit
173 : // cleanup exactly once for any thread that caches via deallocate,
174 : // including consumer threads that never hit a slow path.
175 : static thread_local bool armed = false;
176 HIT 14938 : if(!armed)
177 : {
178 291 : armed = true;
179 291 : arm_thread_cleanup();
180 : }
181 14938 : auto& lp = local();
182 14938 : if(lp.buckets[idx].push(p))
183 8588 : return;
184 6350 : deallocate_slow(p, idx);
185 : }
186 :
187 : protected:
188 : void*
189 : do_allocate(std::size_t bytes, std::size_t) override;
190 :
191 : void
192 : do_deallocate(void* p, std::size_t bytes, std::size_t) override;
193 :
194 : bool
195 2 : do_is_equal(const memory_resource& other) const noexcept override
196 : {
197 2 : return this == &other;
198 : }
199 : };
200 : BOOST_CAPY_MSVC_WARNING_POP
201 :
202 : /** Returns pointer to the default recycling memory resource.
203 :
204 : The returned pointer is valid for the lifetime of the program.
205 : This is the default allocator used by run_async.
206 :
207 : @return Pointer to the recycling memory resource.
208 :
209 : @see recycling_memory_resource
210 : @see run_async
211 : */
212 : BOOST_CAPY_DECL
213 : std::pmr::memory_resource*
214 : get_recycling_memory_resource() noexcept;
215 :
216 : } // namespace capy
217 : } // namespace boost
218 :
219 : #endif
|
