17#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
18#include <Kokkos_Macros.hpp>
20 "Including non-public Kokkos header files is not allowed.");
22#ifndef KOKKOS_HBWSPACE_HPP
23#define KOKKOS_HBWSPACE_HPP
25#include <Kokkos_Macros.hpp>
26#ifdef KOKKOS_ENABLE_HBWSPACE
28#include <Kokkos_HostSpace.hpp>
42void init_lock_array_hbw_space();
49bool lock_address_hbw_space(
void* ptr);
57void unlock_address_hbw_space(
void* ptr);
77 using memory_space = HBWSpace;
78 using size_type = size_t;
86 using execution_space = Kokkos::DefaultHostExecutionSpace;
89 using device_type = Kokkos::Device<execution_space, memory_space>;
93 HBWSpace(
const HBWSpace& rhs) =
default;
94 HBWSpace& operator=(
const HBWSpace&) =
default;
95 ~HBWSpace() =
default;
100 enum AllocationMechanism {
107 explicit HBWSpace(
const AllocationMechanism&);
110 void* allocate(
const size_t arg_alloc_size)
const;
111 void* allocate(
const char* arg_label,
const size_t arg_alloc_size,
112 const size_t arg_logical_size = 0)
const;
115 void deallocate(
void*
const arg_alloc_ptr,
const size_t arg_alloc_size)
const;
116 void deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
117 const size_t arg_alloc_size,
118 const size_t arg_logical_size = 0)
const;
121 template <
class,
class,
class,
class>
122 friend class LogicalMemorySpace;
124 void* impl_allocate(
const char* arg_label,
const size_t arg_alloc_size,
125 const size_t arg_logical_size = 0,
126 const Kokkos::Tools::SpaceHandle =
127 Kokkos::Tools::make_space_handle(name()))
const;
128 void impl_deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
129 const size_t arg_alloc_size,
130 const size_t arg_logical_size = 0,
131 const Kokkos::Tools::SpaceHandle =
132 Kokkos::Tools::make_space_handle(name()))
const;
136 static constexpr const char* name() {
return "HBW"; }
139 AllocationMechanism m_alloc_mech;
140 friend class Kokkos::Impl::SharedAllocationRecord<
141 Kokkos::Experimental::HBWSpace, void>;
155class SharedAllocationRecord<Kokkos::Experimental::HBWSpace, void>
156 :
public SharedAllocationRecord<void, void> {
158 friend Kokkos::Experimental::HBWSpace;
160 using RecordBase = SharedAllocationRecord<void, void>;
162 SharedAllocationRecord(
const SharedAllocationRecord&) =
delete;
163 SharedAllocationRecord& operator=(
const SharedAllocationRecord&) =
delete;
165 static void deallocate(RecordBase*);
167#ifdef KOKKOS_ENABLE_DEBUG
169 static RecordBase s_root_record;
172 const Kokkos::Experimental::HBWSpace m_space;
175 ~SharedAllocationRecord();
176 SharedAllocationRecord() =
default;
178 SharedAllocationRecord(
179 const Kokkos::Experimental::HBWSpace& arg_space,
180 const std::string& arg_label,
const size_t arg_alloc_size,
181 const RecordBase::function_type arg_dealloc = &deallocate);
184 inline std::string get_label()
const {
185 return std::string(RecordBase::head()->m_label);
188 KOKKOS_INLINE_FUNCTION
static SharedAllocationRecord* allocate(
189 const Kokkos::Experimental::HBWSpace& arg_space,
190 const std::string& arg_label,
const size_t arg_alloc_size) {
191 KOKKOS_IF_ON_HOST((
return new SharedAllocationRecord(arg_space, arg_label,
193 KOKKOS_IF_ON_DEVICE(((
void)arg_space; (void)arg_label; (void)arg_alloc_size;
198 static void* allocate_tracked(
const Kokkos::Experimental::HBWSpace& arg_space,
199 const std::string& arg_label,
200 const size_t arg_alloc_size);
203 static void* reallocate_tracked(
void*
const arg_alloc_ptr,
204 const size_t arg_alloc_size);
207 static void deallocate_tracked(
void*
const arg_alloc_ptr);
209 static SharedAllocationRecord* get_record(
void* arg_alloc_ptr);
211 static void print_records(std::ostream&,
212 const Kokkos::Experimental::HBWSpace&,
213 bool detail =
false);
228 Kokkos::Experimental::HBWSpace>::assignable,
232struct MemorySpaceAccess<Kokkos::HostSpace, Kokkos::Experimental::HBWSpace> {
233 enum :
bool { assignable =
true };
234 enum :
bool { accessible =
true };
235 enum :
bool { deepcopy =
true };
240 enum :
bool { assignable =
false };
241 enum :
bool { accessible =
true };
242 enum :
bool { deepcopy =
true };
256struct DeepCopy<Kokkos::Experimental::HBWSpace, Kokkos::Experimental::HBWSpace,
257 DefaultHostExecutionSpace> {
258 DeepCopy(
void* dst,
const void* src,
size_t n) {
259 hostspace_parallel_deepcopy(dst, src, n);
262 DeepCopy(
const DefaultHostExecutionSpace& exec,
void* dst,
const void* src,
264 hostspace_parallel_deepcopy(exec, dst, src, n);
268template <
class ExecutionSpace>
269struct DeepCopy<Kokkos::Experimental::HBWSpace, Kokkos::Experimental::HBWSpace,
271 DeepCopy(
void* dst,
const void* src,
size_t n) {
272 hostspace_parallel_deepcopy(dst, src, n);
275 DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
size_t n) {
277 "Kokkos::Impl::DeepCopy<Kokkos::Experimental::HBWSpace, "
278 "Kokkos::Experimental::HBWSpace,ExecutionSpace::DeepCopy: fence "
280 hostspace_parallel_deepcopy_async(dst, src, n);
285struct DeepCopy<HostSpace, Kokkos::Experimental::HBWSpace,
286 DefaultHostExecutionSpace> {
287 DeepCopy(
void* dst,
const void* src,
size_t n) {
288 hostspace_parallel_deepcopy(dst, src, n);
291 DeepCopy(
const DefaultHostExecutionSpace& exec,
void* dst,
const void* src,
293 hostspace_parallel_deepcopy(exec, dst, src, n);
297template <
class ExecutionSpace>
298struct DeepCopy<HostSpace, Kokkos::Experimental::HBWSpace, ExecutionSpace> {
299 DeepCopy(
void* dst,
const void* src,
size_t n) {
300 hostspace_parallel_deepcopy(dst, src, n);
303 DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
size_t n) {
305 "Kokkos::Impl::DeepCopy<HostSpace, Kokkos::Experimental::HBWSpace, "
306 "ExecutionSpace>::DeepCopy: fence before copy");
307 hostspace_parallel_deepcopy_async(copy_space, dst, src, n);
312struct DeepCopy<Kokkos::Experimental::HBWSpace, HostSpace,
313 DefaultHostExecutionSpace> {
314 DeepCopy(
void* dst,
const void* src,
size_t n) {
315 hostspace_parallel_deepcopy(dst, src, n);
318 DeepCopy(
const DefaultHostExecutionSpace& exec,
void* dst,
const void* src,
320 hostspace_parallel_deepcopy(exec, dst, src, n);
324template <
class ExecutionSpace>
325struct DeepCopy<Kokkos::Experimental::HBWSpace, HostSpace, ExecutionSpace> {
326 DeepCopy(
void* dst,
const void* src,
size_t n) {
327 hostspace_parallel_deepcopy(dst, src, n);
330 DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
size_t n) {
332 "Kokkos::Impl::DeepCopy<Kokkos::Experimental::HBWSpace, HostSpace, "
333 "ExecutionSpace>::DeepCopy: fence before copy");
334 hostspace_parallel_deepcopy_async(dst, src, n);
Memory management for host memory.
Access relationship between DstMemorySpace and SrcMemorySpace.