40#ifndef TPETRA_DETAILS_WRAPPEDDUALVIEW_HPP
41#define TPETRA_DETAILS_WRAPPEDDUALVIEW_HPP
43#include <Tpetra_Access.hpp>
44#include <Tpetra_Details_temporaryViewUtils.hpp>
45#include <Kokkos_DualView.hpp>
46#include "Teuchos_TestForException.hpp"
51#ifdef DEBUG_UVM_REMOVAL
53#define DEBUG_UVM_REMOVAL_ARGUMENT ,const char* callerstr = __builtin_FUNCTION(),const char * filestr=__builtin_FILE(),const int linnum = __builtin_LINE()
55#define DEBUG_UVM_REMOVAL_PRINT_CALLER(fn) \
57 auto envVarSet = std::getenv("TPETRA_UVM_REMOVAL"); \
58 if (envVarSet && (std::strcmp(envVarSet,"1") == 0)) \
59 std::cout << (fn) << " called from " << callerstr \
60 << " at " << filestr << ":"<<linnum \
61 << " host cnt " << dualView.h_view.use_count() \
62 << " device cnt " << dualView.d_view.use_count() \
68#define DEBUG_UVM_REMOVAL_ARGUMENT
69#define DEBUG_UVM_REMOVAL_PRINT_CALLER(fn)
77 template<
typename SC,
typename LO,
typename GO,
typename NO>
87template <
typename DualViewType>
89 using valueType =
typename DualViewType::value_type;
90 using constValueType =
typename DualViewType::const_value_type;
91 static constexpr bool value = std::is_same<valueType, constValueType>::value;
94template <
typename DualViewType>
95using enableIfConstData = std::enable_if_t<hasConstData<DualViewType>::value>;
97template <
typename DualViewType>
98using enableIfNonConstData = std::enable_if_t<!hasConstData<DualViewType>::value>;
100template <
typename DualViewType>
101enableIfNonConstData<DualViewType>
102sync_host(DualViewType dualView) {
103 dualView.sync_host();
106template <
typename DualViewType>
107enableIfConstData<DualViewType>
108sync_host(DualViewType dualView) { }
110template <
typename DualViewType>
111enableIfNonConstData<DualViewType>
112sync_device(DualViewType dualView) {
113 dualView.sync_device();
116template <
typename DualViewType>
117enableIfConstData<DualViewType>
118sync_device(DualViewType dualView) { }
123template <
typename DualViewType>
124class WrappedDualView {
127 using DVT = DualViewType;
128 using t_host =
typename DualViewType::t_host;
129 using t_dev =
typename DualViewType::t_dev;
131 using HostType =
typename t_host::device_type;
132 using DeviceType =
typename t_dev::device_type;
135 static constexpr bool dualViewHasNonConstData = !impl::hasConstData<DualViewType>::value;
136 static constexpr bool deviceMemoryIsHostAccessible =
137 Kokkos::SpaceAccessibility<Kokkos::DefaultHostExecutionSpace, typename t_dev::memory_space>::accessible;
142 WrappedDualView(DualViewType dualV)
143 : originalDualView(dualV),
144 dualView(originalDualView)
152 WrappedDualView(DualViewType dualV,DualViewType origDualV)
153 : originalDualView(origDualV),
158 WrappedDualView(
const t_dev deviceView) {
159 TEUCHOS_TEST_FOR_EXCEPTION(
160 deviceView.data() !=
nullptr && deviceView.use_count() == 0,
161 std::invalid_argument,
162 "Tpetra::Details::WrappedDualView: cannot construct with a device view that\n"
163 "does not own its memory (i.e. constructed with a raw pointer and dimensions)\n"
164 "because the WrappedDualView needs to assume ownership of the memory.");
168 if(deviceView.use_count() != 0)
170 hostView = Kokkos::create_mirror_view(
171 Kokkos::WithoutInitializing,
172 typename t_host::memory_space(),
175 originalDualView = DualViewType(deviceView, hostView);
176 originalDualView.clear_sync_state();
177 originalDualView.modify_device();
178 dualView = originalDualView;
182 WrappedDualView(
const WrappedDualView parent,
int offset,
int numEntries) {
183 originalDualView = parent.originalDualView;
184 dualView = getSubview(parent.dualView, offset, numEntries);
189 WrappedDualView(
const WrappedDualView parent,
const Kokkos::pair<size_t,size_t>& rowRng,
const Kokkos::Impl::ALL_t& colRng) {
190 originalDualView = parent.originalDualView;
191 dualView = getSubview2D(parent.dualView,rowRng,colRng);
194 WrappedDualView(
const WrappedDualView parent,
const Kokkos::Impl::ALL_t &rowRng,
const Kokkos::pair<size_t,size_t>& colRng) {
195 originalDualView = parent.originalDualView;
196 dualView = getSubview2D(parent.dualView,rowRng,colRng);
199 WrappedDualView(
const WrappedDualView parent,
const Kokkos::pair<size_t,size_t>& rowRng,
const Kokkos::pair<size_t,size_t>& colRng) {
200 originalDualView = parent.originalDualView;
201 dualView = getSubview2D(parent.dualView,rowRng,colRng);
204 size_t extent(
const int i)
const {
205 return dualView.h_view.extent(i);
208 void stride(
size_t * stride_)
const {
209 dualView.stride(stride_);
213 size_t origExtent(
const int i)
const {
214 return originalDualView.h_view.extent(i);
217 const char * label()
const {
218 return dualView.d_view.label();
222 typename t_host::const_type
223 getHostView(Access::ReadOnlyStruct
224 DEBUG_UVM_REMOVAL_ARGUMENT
227 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostViewReadOnly");
228 if(needsSyncPath()) {
229 throwIfDeviceViewAlive();
230 impl::sync_host(originalDualView);
232 return dualView.view_host();
236 getHostView(Access::ReadWriteStruct
237 DEBUG_UVM_REMOVAL_ARGUMENT
240 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostViewReadWrite");
241 static_assert(dualViewHasNonConstData,
242 "ReadWrite views are not available for DualView with const data");
243 if(needsSyncPath()) {
244 throwIfDeviceViewAlive();
245 impl::sync_host(originalDualView);
246 originalDualView.modify_host();
249 return dualView.view_host();
253 getHostView(Access::OverwriteAllStruct
254 DEBUG_UVM_REMOVAL_ARGUMENT
257 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostViewOverwriteAll");
258 static_assert(dualViewHasNonConstData,
259 "OverwriteAll views are not available for DualView with const data");
261 return getHostView(Access::ReadWrite);
263 if(needsSyncPath()) {
264 throwIfDeviceViewAlive();
265 if (deviceMemoryIsHostAccessible) Kokkos::fence();
266 dualView.clear_sync_state();
267 dualView.modify_host();
269 return dualView.view_host();
272 typename t_dev::const_type
273 getDeviceView(Access::ReadOnlyStruct
274 DEBUG_UVM_REMOVAL_ARGUMENT
277 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceViewReadOnly");
278 if(needsSyncPath()) {
279 throwIfHostViewAlive();
280 impl::sync_device(originalDualView);
282 return dualView.view_device();
286 getDeviceView(Access::ReadWriteStruct
287 DEBUG_UVM_REMOVAL_ARGUMENT
290 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceViewReadWrite");
291 static_assert(dualViewHasNonConstData,
292 "ReadWrite views are not available for DualView with const data");
293 if(needsSyncPath()) {
294 throwIfHostViewAlive();
295 impl::sync_device(originalDualView);
296 originalDualView.modify_device();
298 return dualView.view_device();
302 getDeviceView(Access::OverwriteAllStruct
303 DEBUG_UVM_REMOVAL_ARGUMENT
306 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceViewOverwriteAll");
307 static_assert(dualViewHasNonConstData,
308 "OverwriteAll views are not available for DualView with const data");
310 return getDeviceView(Access::ReadWrite);
312 if(needsSyncPath()) {
313 throwIfHostViewAlive();
314 if (deviceMemoryIsHostAccessible) Kokkos::fence();
315 dualView.clear_sync_state();
316 dualView.modify_device();
318 return dualView.view_device();
321 template<
class TargetDeviceType>
322 typename std::remove_reference<decltype(std::declval<DualViewType>().template view<TargetDeviceType>())>::type::const_type
323 getView (Access::ReadOnlyStruct s DEBUG_UVM_REMOVAL_ARGUMENT)
const {
324 using ReturnViewType =
typename std::remove_reference<decltype(std::declval<DualViewType>().template view<TargetDeviceType>())>::type::const_type;
325 using ReturnDeviceType =
typename ReturnViewType::device_type;
326 constexpr bool returnDevice = std::is_same<ReturnDeviceType, DeviceType>::value;
328 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Device>ReadOnly");
329 if(needsSyncPath()) {
330 throwIfHostViewAlive();
331 impl::sync_device(originalDualView);
335 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Host>ReadOnly");
336 if(needsSyncPath()) {
337 throwIfDeviceViewAlive();
338 impl::sync_host(originalDualView);
342 return dualView.template view<TargetDeviceType>();
346 template<
class TargetDeviceType>
347 typename std::remove_reference<decltype(std::declval<DualViewType>().template view<TargetDeviceType>())>::type
348 getView (Access::ReadWriteStruct s DEBUG_UVM_REMOVAL_ARGUMENT)
const {
349 using ReturnViewType =
typename std::remove_reference<decltype(std::declval<DualViewType>().template view<TargetDeviceType>())>::type;
350 using ReturnDeviceType =
typename ReturnViewType::device_type;
351 constexpr bool returnDevice = std::is_same<ReturnDeviceType, DeviceType>::value;
354 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Device>ReadWrite");
355 static_assert(dualViewHasNonConstData,
356 "ReadWrite views are not available for DualView with const data");
357 if(needsSyncPath()) {
358 throwIfHostViewAlive();
359 impl::sync_device(originalDualView);
360 originalDualView.modify_device();
364 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Host>ReadWrite");
365 static_assert(dualViewHasNonConstData,
366 "ReadWrite views are not available for DualView with const data");
367 if(needsSyncPath()) {
368 throwIfDeviceViewAlive();
369 impl::sync_host(originalDualView);
370 originalDualView.modify_host();
374 return dualView.template view<TargetDeviceType>();
378 template<
class TargetDeviceType>
379 typename std::remove_reference<decltype(std::declval<DualViewType>().template view<TargetDeviceType>())>::type
380 getView (Access::OverwriteAllStruct s DEBUG_UVM_REMOVAL_ARGUMENT)
const {
381 using ReturnViewType =
typename std::remove_reference<decltype(std::declval<DualViewType>().template view<TargetDeviceType>())>::type;
382 using ReturnDeviceType =
typename ReturnViewType::device_type;
385 return getView<TargetDeviceType>(Access::ReadWrite);
387 constexpr bool returnDevice = std::is_same<ReturnDeviceType, DeviceType>::value;
390 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Device>OverwriteAll");
391 static_assert(dualViewHasNonConstData,
392 "OverwriteAll views are not available for DualView with const data");
393 if(needsSyncPath()) {
394 throwIfHostViewAlive();
395 dualView.clear_sync_state();
396 dualView.modify_host();
400 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getView<Host>OverwriteAll");
401 static_assert(dualViewHasNonConstData,
402 "OverwriteAll views are not available for DualView with const data");
403 if(needsSyncPath()) {
404 throwIfDeviceViewAlive();
405 dualView.clear_sync_state();
406 dualView.modify_device();
410 return dualView.template view<TargetDeviceType>();
414 typename t_host::const_type
415 getHostSubview(
int offset,
int numEntries, Access::ReadOnlyStruct
416 DEBUG_UVM_REMOVAL_ARGUMENT
419 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostSubviewReadOnly");
420 if(needsSyncPath()) {
421 throwIfDeviceViewAlive();
422 impl::sync_host(originalDualView);
424 return getSubview(dualView.view_host(), offset, numEntries);
428 getHostSubview(
int offset,
int numEntries, Access::ReadWriteStruct
429 DEBUG_UVM_REMOVAL_ARGUMENT
432 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostSubviewReadWrite");
433 static_assert(dualViewHasNonConstData,
434 "ReadWrite views are not available for DualView with const data");
435 if(needsSyncPath()) {
436 throwIfDeviceViewAlive();
437 impl::sync_host(originalDualView);
438 originalDualView.modify_host();
440 return getSubview(dualView.view_host(), offset, numEntries);
444 getHostSubview(
int offset,
int numEntries, Access::OverwriteAllStruct
445 DEBUG_UVM_REMOVAL_ARGUMENT
448 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getHostSubviewOverwriteAll");
449 static_assert(dualViewHasNonConstData,
450 "OverwriteAll views are not available for DualView with const data");
451 return getHostSubview(offset, numEntries, Access::ReadWrite);
454 typename t_dev::const_type
455 getDeviceSubview(
int offset,
int numEntries, Access::ReadOnlyStruct
456 DEBUG_UVM_REMOVAL_ARGUMENT
459 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceSubviewReadOnly");
460 if(needsSyncPath()) {
461 throwIfHostViewAlive();
462 impl::sync_device(originalDualView);
464 return getSubview(dualView.view_device(), offset, numEntries);
468 getDeviceSubview(
int offset,
int numEntries, Access::ReadWriteStruct
469 DEBUG_UVM_REMOVAL_ARGUMENT
472 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceSubviewReadWrite");
473 static_assert(dualViewHasNonConstData,
474 "ReadWrite views are not available for DualView with const data");
475 if(needsSyncPath()) {
476 throwIfHostViewAlive();
477 impl::sync_device(originalDualView);
478 originalDualView.modify_device();
480 return getSubview(dualView.view_device(), offset, numEntries);
484 getDeviceSubview(
int offset,
int numEntries, Access::OverwriteAllStruct
485 DEBUG_UVM_REMOVAL_ARGUMENT
488 DEBUG_UVM_REMOVAL_PRINT_CALLER(
"getDeviceSubviewOverwriteAll");
489 static_assert(dualViewHasNonConstData,
490 "OverwriteAll views are not available for DualView with const data");
491 return getDeviceSubview(offset, numEntries, Access::ReadWrite);
496 typename t_host::HostMirror getHostCopy()
const {
497 auto X_dev = dualView.view_host();
498 if(X_dev.span_is_contiguous()) {
499 auto mirror = Kokkos::create_mirror_view(X_dev);
500 Kokkos::deep_copy(mirror,X_dev);
504 auto X_contig = Tpetra::Details::TempView::toLayout<decltype(X_dev), Kokkos::LayoutLeft>(X_dev);
505 auto mirror = Kokkos::create_mirror_view(X_contig);
506 Kokkos::deep_copy(mirror,X_contig);
511 typename t_dev::HostMirror getDeviceCopy()
const {
512 auto X_dev = dualView.view_device();
513 if(X_dev.span_is_contiguous()) {
514 auto mirror = Kokkos::create_mirror_view(X_dev);
515 Kokkos::deep_copy(mirror,X_dev);
519 auto X_contig = Tpetra::Details::TempView::toLayout<decltype(X_dev), Kokkos::LayoutLeft>(X_dev);
520 auto mirror = Kokkos::create_mirror_view(X_contig);
521 Kokkos::deep_copy(mirror,X_contig);
527 bool is_valid_host()
const {
528 return dualView.view_host().size() == 0 || dualView.view_host().data();
531 bool is_valid_device()
const {
532 return dualView.view_device().size() == 0 || dualView.view_device().data();
536 bool need_sync_host()
const {
537 return originalDualView.need_sync_host();
540 bool need_sync_device()
const {
541 return originalDualView.need_sync_device();
544 int host_view_use_count()
const {
545 return originalDualView.h_view.use_count();
548 int device_view_use_count()
const {
549 return originalDualView.d_view.use_count();
555 template<
typename SC,
typename LO,
typename GO,
typename NO>
556 friend class ::Tpetra::MultiVector;
562 DualViewType getOriginalDualView()
const {
563 return originalDualView;
566 DualViewType getDualView()
const {
570 template <
typename ViewType>
571 ViewType getSubview(ViewType view,
int offset,
int numEntries)
const {
572 return Kokkos::subview(view, Kokkos::pair<int, int>(offset, offset+numEntries));
575 template <
typename ViewType,
typename int_type>
576 ViewType getSubview2D(ViewType view, Kokkos::pair<int_type,int_type> offset0,
const Kokkos::Impl::ALL_t&)
const {
577 return Kokkos::subview(view,offset0,Kokkos::ALL());
580 template <
typename ViewType,
typename int_type>
581 ViewType getSubview2D(ViewType view,
const Kokkos::Impl::ALL_t&, Kokkos::pair<int_type,int_type> offset1)
const {
582 return Kokkos::subview(view,Kokkos::ALL(),offset1);
585 template <
typename ViewType,
typename int_type>
586 ViewType getSubview2D(ViewType view, Kokkos::pair<int_type,int_type> offset0, Kokkos::pair<int_type,int_type> offset1)
const {
587 return Kokkos::subview(view,offset0,offset1);
591 bool memoryIsAliased()
const {
592 return deviceMemoryIsHostAccessible && dualView.h_view.data() == dualView.d_view.data();
595 bool needsSyncPath()
const {
612#ifdef KOKKOS_ENABLE_CUDA
613 return std::is_same<typename t_dev::memory_space,Kokkos::CudaUVMSpace>::value || !memoryIsAliased();
615 return !memoryIsAliased();
620 void throwIfViewsAreDifferentSizes()
const {
623 if(dualView.d_view.size() != dualView.h_view.size()) {
624 std::ostringstream msg;
625 msg <<
"Tpetra::Details::WrappedDualView (name = " << dualView.d_view.label()
626 <<
"; host and device views are different sizes: "
627 << dualView.h_view.size() <<
" vs " <<dualView.h_view.size();
628 throw std::runtime_error(msg.str());
632 void throwIfHostViewAlive()
const {
633 throwIfViewsAreDifferentSizes();
634 if (dualView.h_view.use_count() > dualView.d_view.use_count()) {
635 std::ostringstream msg;
636 msg <<
"Tpetra::Details::WrappedDualView (name = " << dualView.d_view.label()
637 <<
"; host use_count = " << dualView.h_view.use_count()
638 <<
"; device use_count = " << dualView.d_view.use_count() <<
"): "
639 <<
"Cannot access data on device while a host view is alive";
640 throw std::runtime_error(msg.str());
644 void throwIfDeviceViewAlive()
const {
645 throwIfViewsAreDifferentSizes();
646 if (dualView.d_view.use_count() > dualView.h_view.use_count()) {
647 std::ostringstream msg;
648 msg <<
"Tpetra::Details::WrappedDualView (name = " << dualView.d_view.label()
649 <<
"; host use_count = " << dualView.h_view.use_count()
650 <<
"; device use_count = " << dualView.d_view.use_count() <<
"): "
651 <<
"Cannot access data on host while a device view is alive";
652 throw std::runtime_error(msg.str());
657 return originalDualView.h_view != dualView.h_view;
660 mutable DualViewType originalDualView;
661 mutable DualViewType dualView;
Implementation details of Tpetra.
Namespace Tpetra contains the class and methods constituting the Tpetra library.