Kokkos Core Kernels Package Version of the Day
Loading...
Searching...
No Matches
Kokkos_DualView.hpp
Go to the documentation of this file.
1//@HEADER
2// ************************************************************************
3//
4// Kokkos v. 4.0
5// Copyright (2022) National Technology & Engineering
6// Solutions of Sandia, LLC (NTESS).
7//
8// Under the terms of Contract DE-NA0003525 with NTESS,
9// the U.S. Government retains certain rights in this software.
10//
11// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
12// See https://kokkos.org/LICENSE for license information.
13// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
14//
15//@HEADER
16
22
23#ifndef KOKKOS_DUALVIEW_HPP
24#define KOKKOS_DUALVIEW_HPP
25#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE
26#define KOKKOS_IMPL_PUBLIC_INCLUDE
27#define KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_DUALVIEW
28#endif
29
30#include <Kokkos_Core.hpp>
31#include <impl/Kokkos_Error.hpp>
32
33namespace Kokkos {
34
35/* \class DualView
36 * \brief Container to manage mirroring a Kokkos::View that lives
37 * in device memory with a Kokkos::View that lives in host memory.
38 *
39 * This class provides capabilities to manage data which exists in two
40 * memory spaces at the same time. It keeps views of the same layout
41 * on two memory spaces as well as modified flags for both
42 * allocations. Users are responsible for setting the modified flags
43 * manually if they change the data in either memory space, by calling
44 * the sync() method templated on the device where they modified the
45 * data. Users may synchronize data by calling the modify() function,
46 * templated on the device towards which they want to synchronize
47 * (i.e., the target of the one-way copy operation).
48 *
49 * The DualView class also provides convenience methods such as
50 * realloc, resize and capacity which call the appropriate methods of
51 * the underlying Kokkos::View objects.
52 *
53 * The four template arguments are the same as those of Kokkos::View.
54 * (Please refer to that class' documentation for a detailed
55 * description.)
56 *
57 * \tparam DataType The type of the entries stored in the container.
58 *
59 * \tparam Layout The array's layout in memory.
60 *
61 * \tparam Device The Kokkos Device type. If its memory space is
62 * not the same as the host's memory space, then DualView will
63 * contain two separate Views: one in device memory, and one in
64 * host memory. Otherwise, DualView will only store one View.
65 *
66 * \tparam MemoryTraits (optional) The user's intended memory access
67 * behavior. Please see the documentation of Kokkos::View for
68 * examples. The default suffices for most users.
69 */
70
71namespace Impl {
72
73#ifdef KOKKOS_ENABLE_CUDA
74
75inline const Kokkos::Cuda& get_cuda_space(const Kokkos::Cuda& in) { return in; }
76
77inline const Kokkos::Cuda& get_cuda_space() {
78 return *Kokkos::Impl::cuda_get_deep_copy_space();
79}
80
81template <typename NonCudaExecSpace>
82inline const Kokkos::Cuda& get_cuda_space(const NonCudaExecSpace&) {
83 return get_cuda_space();
84}
85
86#endif // KOKKOS_ENABLE_CUDA
87
88} // namespace Impl
89template <class DataType, class Arg1Type = void, class Arg2Type = void,
90 class Arg3Type = void>
91class DualView : public ViewTraits<DataType, Arg1Type, Arg2Type, Arg3Type> {
92 template <class, class, class, class>
93 friend class DualView;
94
95 public:
97
98 using traits = ViewTraits<DataType, Arg1Type, Arg2Type, Arg3Type>;
99
101 using host_mirror_space = typename traits::host_mirror_space;
102
104 using t_dev = View<typename traits::data_type, Arg1Type, Arg2Type, Arg3Type>;
105
108 using t_host = typename t_dev::HostMirror;
109
112 using t_dev_const =
113 View<typename traits::const_data_type, Arg1Type, Arg2Type, Arg3Type>;
114
117 using t_host_const = typename t_dev_const::HostMirror;
118
120 using t_dev_const_randomread =
121 View<typename traits::const_data_type, typename traits::array_layout,
122 typename traits::device_type,
123 Kokkos::MemoryTraits<Kokkos::RandomAccess>>;
124
128 using t_host_const_randomread = typename t_dev_const_randomread::HostMirror;
129
131 using t_dev_um =
132 View<typename traits::data_type, typename traits::array_layout,
133 typename traits::device_type, MemoryUnmanaged>;
134
136 using t_host_um =
137 View<typename t_host::data_type, typename t_host::array_layout,
138 typename t_host::device_type, MemoryUnmanaged>;
139
141 using t_dev_const_um =
142 View<typename traits::const_data_type, typename traits::array_layout,
143 typename traits::device_type, MemoryUnmanaged>;
144
146 using t_host_const_um =
147 View<typename t_host::const_data_type, typename t_host::array_layout,
148 typename t_host::device_type, MemoryUnmanaged>;
149
151 using t_dev_const_randomread_um =
152 View<typename t_host::const_data_type, typename t_host::array_layout,
153 typename t_host::device_type,
154 Kokkos::MemoryTraits<Kokkos::Unmanaged | Kokkos::RandomAccess>>;
155
159 using t_host_const_randomread_um =
161
163
165
166 protected:
167 // modified_flags[0] -> host
168 // modified_flags[1] -> device
169 using t_modified_flags = View<unsigned int[2], LayoutLeft, Kokkos::HostSpace>;
170 t_modified_flags modified_flags;
171
172 public:
174
175 // Moved this specifically after modified_flags to resolve an alignment issue
176 // on MSVC/NVCC
178
179 t_dev d_view;
180 t_host h_view;
182
184
185
191 DualView() = default;
192
202 DualView(const std::string& label,
203 const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
204 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
205 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
206 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
207 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
208 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
209 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
210 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG)
211 : modified_flags(
212 Kokkos::view_alloc(typename t_modified_flags::execution_space{},
213 "DualView::modified_flags")),
214 d_view(label, n0, n1, n2, n3, n4, n5, n6, n7),
215 h_view(create_mirror_view(d_view)) // without UVM, host View mirrors
216 {}
217
228 template <class... P>
229 DualView(const Impl::ViewCtorProp<P...>& arg_prop,
230 std::enable_if_t<!Impl::ViewCtorProp<P...>::has_pointer,
231 size_t> const n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
232 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
233 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
234 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
235 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
236 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
237 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
238 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG)
239 : modified_flags(t_modified_flags("DualView::modified_flags")),
240 d_view(arg_prop, n0, n1, n2, n3, n4, n5, n6, n7) {
241 // without UVM, host View mirrors
242 if (Kokkos::Impl::has_type<Impl::WithoutInitializing_t, P...>::value)
243 h_view = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, d_view);
244 else
245 h_view = Kokkos::create_mirror_view(d_view);
246 }
247
249 template <class SS, class LS, class DS, class MS>
250 DualView(const DualView<SS, LS, DS, MS>& src)
251 : modified_flags(src.modified_flags),
252 d_view(src.d_view),
253 h_view(src.h_view) {}
254
256 template <class SD, class S1, class S2, class S3, class Arg0, class... Args>
257 DualView(const DualView<SD, S1, S2, S3>& src, const Arg0& arg0, Args... args)
258 : modified_flags(src.modified_flags),
259 d_view(Kokkos::subview(src.d_view, arg0, args...)),
260 h_view(Kokkos::subview(src.h_view, arg0, args...)) {}
261
272 DualView(const t_dev& d_view_, const t_host& h_view_)
273 : modified_flags(t_modified_flags("DualView::modified_flags")),
274 d_view(d_view_),
275 h_view(h_view_) {
276 if (int(d_view.rank) != int(h_view.rank) ||
277 d_view.extent(0) != h_view.extent(0) ||
278 d_view.extent(1) != h_view.extent(1) ||
279 d_view.extent(2) != h_view.extent(2) ||
280 d_view.extent(3) != h_view.extent(3) ||
281 d_view.extent(4) != h_view.extent(4) ||
282 d_view.extent(5) != h_view.extent(5) ||
283 d_view.extent(6) != h_view.extent(6) ||
284 d_view.extent(7) != h_view.extent(7) ||
285 d_view.stride_0() != h_view.stride_0() ||
286 d_view.stride_1() != h_view.stride_1() ||
287 d_view.stride_2() != h_view.stride_2() ||
288 d_view.stride_3() != h_view.stride_3() ||
289 d_view.stride_4() != h_view.stride_4() ||
290 d_view.stride_5() != h_view.stride_5() ||
291 d_view.stride_6() != h_view.stride_6() ||
292 d_view.stride_7() != h_view.stride_7() ||
293 d_view.span() != h_view.span()) {
294 Kokkos::Impl::throw_runtime_exception(
295 "DualView constructed with incompatible views");
296 }
297 }
298 // does the DualView have only one device
299 struct impl_dualview_is_single_device {
300 enum : bool {
301 value = std::is_same<typename t_dev::device_type,
302 typename t_host::device_type>::value
303 };
304 };
305
306 // does the given device match the device of t_dev?
307 template <typename Device>
308 struct impl_device_matches_tdev_device {
309 enum : bool {
310 value = std::is_same<typename t_dev::device_type, Device>::value
311 };
312 };
313 // does the given device match the device of t_host?
314 template <typename Device>
315 struct impl_device_matches_thost_device {
316 enum : bool {
317 value = std::is_same<typename t_host::device_type, Device>::value
318 };
319 };
320
321 // does the given device match the execution space of t_host?
322 template <typename Device>
323 struct impl_device_matches_thost_exec {
324 enum : bool {
325 value = std::is_same<typename t_host::execution_space, Device>::value
326 };
327 };
328
329 // does the given device match the execution space of t_dev?
330 template <typename Device>
331 struct impl_device_matches_tdev_exec {
332 enum : bool {
333 value = std::is_same<typename t_dev::execution_space, Device>::value
334 };
335 };
336
337 // does the given device's memory space match the memory space of t_dev?
338 template <typename Device>
339 struct impl_device_matches_tdev_memory_space {
340 enum : bool {
341 value = std::is_same<typename t_dev::memory_space,
342 typename Device::memory_space>::value
343 };
344 };
345
347
349
372 template <class Device>
373 KOKKOS_INLINE_FUNCTION const typename std::conditional_t<
374 impl_device_matches_tdev_device<Device>::value, t_dev,
375 typename std::conditional_t<
376 impl_device_matches_thost_device<Device>::value, t_host,
377 typename std::conditional_t<
378 impl_device_matches_thost_exec<Device>::value, t_host,
379 typename std::conditional_t<
380 impl_device_matches_tdev_exec<Device>::value, t_dev,
381 typename std::conditional_t<
382 impl_device_matches_tdev_memory_space<Device>::value,
383 t_dev, t_host>>>>>
384 view() const {
385 constexpr bool device_is_memspace =
386 std::is_same<Device, typename Device::memory_space>::value;
387 constexpr bool device_is_execspace =
388 std::is_same<Device, typename Device::execution_space>::value;
389 constexpr bool device_exec_is_t_dev_exec =
390 std::is_same<typename Device::execution_space,
391 typename t_dev::execution_space>::value;
392 constexpr bool device_mem_is_t_dev_mem =
393 std::is_same<typename Device::memory_space,
394 typename t_dev::memory_space>::value;
395 constexpr bool device_exec_is_t_host_exec =
396 std::is_same<typename Device::execution_space,
397 typename t_host::execution_space>::value;
398 constexpr bool device_mem_is_t_host_mem =
399 std::is_same<typename Device::memory_space,
400 typename t_host::memory_space>::value;
401 constexpr bool device_is_t_host_device =
402 std::is_same<typename Device::execution_space,
403 typename t_host::device_type>::value;
404 constexpr bool device_is_t_dev_device =
405 std::is_same<typename Device::memory_space,
406 typename t_host::device_type>::value;
407
408 static_assert(
409 device_is_t_dev_device || device_is_t_host_device ||
410 (device_is_memspace &&
411 (device_mem_is_t_dev_mem || device_mem_is_t_host_mem)) ||
412 (device_is_execspace &&
413 (device_exec_is_t_dev_exec || device_exec_is_t_host_exec)) ||
414 ((!device_is_execspace && !device_is_memspace) &&
415 ((device_mem_is_t_dev_mem || device_mem_is_t_host_mem) ||
416 (device_exec_is_t_dev_exec || device_exec_is_t_host_exec))),
417 "Template parameter to .view() must exactly match one of the "
418 "DualView's device types or one of the execution or memory spaces");
419
420 return Impl::if_c<std::is_same<typename t_dev::memory_space,
421 typename Device::memory_space>::value,
422 t_dev, t_host>::select(d_view, h_view);
423 }
424
425 KOKKOS_INLINE_FUNCTION
426 t_host view_host() const { return h_view; }
427
428 KOKKOS_INLINE_FUNCTION
429 t_dev view_device() const { return d_view; }
430
431 KOKKOS_INLINE_FUNCTION constexpr bool is_allocated() const {
432 return (d_view.is_allocated() && h_view.is_allocated());
433 }
434
435 template <class Device>
436 static int get_device_side() {
437 constexpr bool device_is_memspace =
438 std::is_same<Device, typename Device::memory_space>::value;
439 constexpr bool device_is_execspace =
440 std::is_same<Device, typename Device::execution_space>::value;
441 constexpr bool device_exec_is_t_dev_exec =
442 std::is_same<typename Device::execution_space,
443 typename t_dev::execution_space>::value;
444 constexpr bool device_mem_is_t_dev_mem =
445 std::is_same<typename Device::memory_space,
446 typename t_dev::memory_space>::value;
447 constexpr bool device_exec_is_t_host_exec =
448 std::is_same<typename Device::execution_space,
449 typename t_host::execution_space>::value;
450 constexpr bool device_mem_is_t_host_mem =
451 std::is_same<typename Device::memory_space,
452 typename t_host::memory_space>::value;
453 constexpr bool device_is_t_host_device =
454 std::is_same<typename Device::execution_space,
455 typename t_host::device_type>::value;
456 constexpr bool device_is_t_dev_device =
457 std::is_same<typename Device::memory_space,
458 typename t_host::device_type>::value;
459
460 static_assert(
461 device_is_t_dev_device || device_is_t_host_device ||
462 (device_is_memspace &&
463 (device_mem_is_t_dev_mem || device_mem_is_t_host_mem)) ||
464 (device_is_execspace &&
465 (device_exec_is_t_dev_exec || device_exec_is_t_host_exec)) ||
466 ((!device_is_execspace && !device_is_memspace) &&
467 ((device_mem_is_t_dev_mem || device_mem_is_t_host_mem) ||
468 (device_exec_is_t_dev_exec || device_exec_is_t_host_exec))),
469 "Template parameter to .sync() must exactly match one of the "
470 "DualView's device types or one of the execution or memory spaces");
471
472 int dev = -1;
473 if (device_is_t_dev_device)
474 dev = 1;
475 else if (device_is_t_host_device)
476 dev = 0;
477 else {
478 if (device_is_memspace) {
479 if (device_mem_is_t_dev_mem) dev = 1;
480 if (device_mem_is_t_host_mem) dev = 0;
481 if (device_mem_is_t_host_mem && device_mem_is_t_dev_mem) dev = -1;
482 }
483 if (device_is_execspace) {
484 if (device_exec_is_t_dev_exec) dev = 1;
485 if (device_exec_is_t_host_exec) dev = 0;
486 if (device_exec_is_t_host_exec && device_exec_is_t_dev_exec) dev = -1;
487 }
488 if (!device_is_execspace && !device_is_memspace) {
489 if (device_mem_is_t_dev_mem) dev = 1;
490 if (device_mem_is_t_host_mem) dev = 0;
491 if (device_mem_is_t_host_mem && device_mem_is_t_dev_mem) dev = -1;
492 if (device_exec_is_t_dev_exec) dev = 1;
493 if (device_exec_is_t_host_exec) dev = 0;
494 if (device_exec_is_t_host_exec && device_exec_is_t_dev_exec) dev = -1;
495 }
496 }
497 return dev;
498 }
499 static constexpr const int view_header_size = 128;
500 void impl_report_host_sync() const noexcept {
501 if (Kokkos::Tools::Experimental::get_callbacks().sync_dual_view !=
502 nullptr) {
503 Kokkos::Tools::syncDualView(
504 h_view.label(),
505 reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(h_view.data()) -
506 view_header_size),
507 false);
508 }
509 }
510 void impl_report_device_sync() const noexcept {
511 if (Kokkos::Tools::Experimental::get_callbacks().sync_dual_view !=
512 nullptr) {
513 Kokkos::Tools::syncDualView(
514 d_view.label(),
515 reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(d_view.data()) -
516 view_header_size),
517 true);
518 }
519 }
520
538 // deliberately passing args by cref as they're used multiple times
539 template <class Device, class... Args>
540 void sync_impl(std::true_type, Args const&... args) {
541 if (modified_flags.data() == nullptr) return;
542
543 int dev = get_device_side<Device>();
544
545 if (dev == 1) { // if Device is the same as DualView's device type
546 if ((modified_flags(0) > 0) && (modified_flags(0) >= modified_flags(1))) {
547#ifdef KOKKOS_ENABLE_CUDA
548 if (std::is_same<typename t_dev::memory_space,
549 Kokkos::CudaUVMSpace>::value) {
550 if (d_view.data() == h_view.data())
551 Kokkos::Impl::cuda_prefetch_pointer(
552 Impl::get_cuda_space(args...), d_view.data(),
553 sizeof(typename t_dev::value_type) * d_view.span(), true);
554 }
555#endif
556
557 deep_copy(args..., d_view, h_view);
558 modified_flags(0) = modified_flags(1) = 0;
559 impl_report_device_sync();
560 }
561 }
562 if (dev == 0) { // hopefully Device is the same as DualView's host type
563 if ((modified_flags(1) > 0) && (modified_flags(1) >= modified_flags(0))) {
564#ifdef KOKKOS_ENABLE_CUDA
565 if (std::is_same<typename t_dev::memory_space,
566 Kokkos::CudaUVMSpace>::value) {
567 if (d_view.data() == h_view.data())
568 Kokkos::Impl::cuda_prefetch_pointer(
569 Impl::get_cuda_space(args...), d_view.data(),
570 sizeof(typename t_dev::value_type) * d_view.span(), false);
571 }
572#endif
573
574 deep_copy(args..., h_view, d_view);
575 modified_flags(0) = modified_flags(1) = 0;
576 impl_report_host_sync();
577 }
578 }
579 if (std::is_same<typename t_host::memory_space,
580 typename t_dev::memory_space>::value) {
581 typename t_dev::execution_space().fence(
582 "Kokkos::DualView<>::sync: fence after syncing DualView");
583 typename t_host::execution_space().fence(
584 "Kokkos::DualView<>::sync: fence after syncing DualView");
585 }
586 }
587
588 template <class Device>
589 void sync(const std::enable_if_t<
590 (std::is_same<typename traits::data_type,
591 typename traits::non_const_data_type>::value) ||
592 (std::is_same<Device, int>::value),
593 int>& = 0) {
594 sync_impl<Device>(std::true_type{});
595 }
596
597 template <class Device, class ExecutionSpace>
598 void sync(const ExecutionSpace& exec,
599 const std::enable_if_t<
600 (std::is_same<typename traits::data_type,
601 typename traits::non_const_data_type>::value) ||
602 (std::is_same<Device, int>::value),
603 int>& = 0) {
604 sync_impl<Device>(std::true_type{}, exec);
605 }
606
607 // deliberately passing args by cref as they're used multiple times
608 template <class Device, class... Args>
609 void sync_impl(std::false_type, Args const&...) {
610 if (modified_flags.data() == nullptr) return;
611
612 int dev = get_device_side<Device>();
613
614 if (dev == 1) { // if Device is the same as DualView's device type
615 if ((modified_flags(0) > 0) && (modified_flags(0) >= modified_flags(1))) {
616 Impl::throw_runtime_exception(
617 "Calling sync on a DualView with a const datatype.");
618 }
619 impl_report_device_sync();
620 }
621 if (dev == 0) { // hopefully Device is the same as DualView's host type
622 if ((modified_flags(1) > 0) && (modified_flags(1) >= modified_flags(0))) {
623 Impl::throw_runtime_exception(
624 "Calling sync on a DualView with a const datatype.");
625 }
626 impl_report_host_sync();
627 }
628 }
629
630 template <class Device>
631 void sync(const std::enable_if_t<
632 (!std::is_same<typename traits::data_type,
633 typename traits::non_const_data_type>::value) ||
634 (std::is_same<Device, int>::value),
635 int>& = 0) {
636 sync_impl<Device>(std::false_type{});
637 }
638 template <class Device, class ExecutionSpace>
639 void sync(const ExecutionSpace& exec,
640 const std::enable_if_t<
641 (!std::is_same<typename traits::data_type,
642 typename traits::non_const_data_type>::value) ||
643 (std::is_same<Device, int>::value),
644 int>& = 0) {
645 sync_impl<Device>(std::false_type{}, exec);
646 }
647
648 // deliberately passing args by cref as they're used multiple times
649 template <typename... Args>
650 void sync_host_impl(Args const&... args) {
651 if (!std::is_same<typename traits::data_type,
652 typename traits::non_const_data_type>::value)
653 Impl::throw_runtime_exception(
654 "Calling sync_host on a DualView with a const datatype.");
655 if (modified_flags.data() == nullptr) return;
656 if (modified_flags(1) > modified_flags(0)) {
657#ifdef KOKKOS_ENABLE_CUDA
658 if (std::is_same<typename t_dev::memory_space,
659 Kokkos::CudaUVMSpace>::value) {
660 if (d_view.data() == h_view.data())
661 Kokkos::Impl::cuda_prefetch_pointer(
662 Impl::get_cuda_space(args...), d_view.data(),
663 sizeof(typename t_dev::value_type) * d_view.span(), false);
664 }
665#endif
666
667 deep_copy(args..., h_view, d_view);
668 modified_flags(1) = modified_flags(0) = 0;
669 impl_report_host_sync();
670 }
671 }
672
673 template <class ExecSpace>
674 void sync_host(const ExecSpace& exec) {
675 sync_host_impl(exec);
676 }
677 void sync_host() { sync_host_impl(); }
678
679 // deliberately passing args by cref as they're used multiple times
680 template <typename... Args>
681 void sync_device_impl(Args const&... args) {
682 if (!std::is_same<typename traits::data_type,
683 typename traits::non_const_data_type>::value)
684 Impl::throw_runtime_exception(
685 "Calling sync_device on a DualView with a const datatype.");
686 if (modified_flags.data() == nullptr) return;
687 if (modified_flags(0) > modified_flags(1)) {
688#ifdef KOKKOS_ENABLE_CUDA
689 if (std::is_same<typename t_dev::memory_space,
690 Kokkos::CudaUVMSpace>::value) {
691 if (d_view.data() == h_view.data())
692 Kokkos::Impl::cuda_prefetch_pointer(
693 Impl::get_cuda_space(args...), d_view.data(),
694 sizeof(typename t_dev::value_type) * d_view.span(), true);
695 }
696#endif
697
698 deep_copy(args..., d_view, h_view);
699 modified_flags(1) = modified_flags(0) = 0;
700 impl_report_device_sync();
701 }
702 }
703
704 template <class ExecSpace>
705 void sync_device(const ExecSpace& exec) {
706 sync_device_impl(exec);
707 }
708 void sync_device() { sync_device_impl(); }
709
710 template <class Device>
711 bool need_sync() const {
712 if (modified_flags.data() == nullptr) return false;
713 int dev = get_device_side<Device>();
714
715 if (dev == 1) { // if Device is the same as DualView's device type
716 if ((modified_flags(0) > 0) && (modified_flags(0) >= modified_flags(1))) {
717 return true;
718 }
719 }
720 if (dev == 0) { // hopefully Device is the same as DualView's host type
721 if ((modified_flags(1) > 0) && (modified_flags(1) >= modified_flags(0))) {
722 return true;
723 }
724 }
725 return false;
726 }
727
728 inline bool need_sync_host() const {
729 if (modified_flags.data() == nullptr) return false;
730 return modified_flags(0) < modified_flags(1);
731 }
732
733 inline bool need_sync_device() const {
734 if (modified_flags.data() == nullptr) return false;
735 return modified_flags(1) < modified_flags(0);
736 }
737 void impl_report_device_modification() {
738 if (Kokkos::Tools::Experimental::get_callbacks().modify_dual_view !=
739 nullptr) {
740 Kokkos::Tools::modifyDualView(
741 d_view.label(),
742 reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(d_view.data()) -
743 view_header_size),
744 true);
745 }
746 }
747 void impl_report_host_modification() {
748 if (Kokkos::Tools::Experimental::get_callbacks().modify_dual_view !=
749 nullptr) {
750 Kokkos::Tools::modifyDualView(
751 h_view.label(),
752 reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(h_view.data()) -
753 view_header_size),
754 false);
755 }
756 }
762 template <class Device, class Dummy = DualView,
763 std::enable_if_t<!Dummy::impl_dualview_is_single_device::value>* =
764 nullptr>
765 void modify() {
766 if (modified_flags.data() == nullptr) {
767 modified_flags = t_modified_flags("DualView::modified_flags");
768 }
769
770 int dev = get_device_side<Device>();
771
772 if (dev == 1) { // if Device is the same as DualView's device type
773 // Increment the device's modified count.
774 modified_flags(1) =
775 (modified_flags(1) > modified_flags(0) ? modified_flags(1)
776 : modified_flags(0)) +
777 1;
778 impl_report_device_modification();
779 }
780 if (dev == 0) { // hopefully Device is the same as DualView's host type
781 // Increment the host's modified count.
782 modified_flags(0) =
783 (modified_flags(1) > modified_flags(0) ? modified_flags(1)
784 : modified_flags(0)) +
785 1;
786 impl_report_host_modification();
787 }
788
789#ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK
790 if (modified_flags(0) && modified_flags(1)) {
791 std::string msg = "Kokkos::DualView::modify ERROR: ";
792 msg += "Concurrent modification of host and device views ";
793 msg += "in DualView \"";
794 msg += d_view.label();
795 msg += "\"\n";
796 Kokkos::abort(msg.c_str());
797 }
798#endif
799 }
800
801 template <
802 class Device, class Dummy = DualView,
803 std::enable_if_t<Dummy::impl_dualview_is_single_device::value>* = nullptr>
804 void modify() {
805 return;
806 }
807
808 template <class Dummy = DualView,
809 std::enable_if_t<!Dummy::impl_dualview_is_single_device::value>* =
810 nullptr>
811 inline void modify_host() {
812 if (modified_flags.data() != nullptr) {
813 modified_flags(0) =
814 (modified_flags(1) > modified_flags(0) ? modified_flags(1)
815 : modified_flags(0)) +
816 1;
817 impl_report_host_modification();
818#ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK
819 if (modified_flags(0) && modified_flags(1)) {
820 std::string msg = "Kokkos::DualView::modify_host ERROR: ";
821 msg += "Concurrent modification of host and device views ";
822 msg += "in DualView \"";
823 msg += d_view.label();
824 msg += "\"\n";
825 Kokkos::abort(msg.c_str());
826 }
827#endif
828 }
829 }
830
831 template <
832 class Dummy = DualView,
833 std::enable_if_t<Dummy::impl_dualview_is_single_device::value>* = nullptr>
834 inline void modify_host() {
835 return;
836 }
837
838 template <class Dummy = DualView,
839 std::enable_if_t<!Dummy::impl_dualview_is_single_device::value>* =
840 nullptr>
841 inline void modify_device() {
842 if (modified_flags.data() != nullptr) {
843 modified_flags(1) =
844 (modified_flags(1) > modified_flags(0) ? modified_flags(1)
845 : modified_flags(0)) +
846 1;
847 impl_report_device_modification();
848#ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK
849 if (modified_flags(0) && modified_flags(1)) {
850 std::string msg = "Kokkos::DualView::modify_device ERROR: ";
851 msg += "Concurrent modification of host and device views ";
852 msg += "in DualView \"";
853 msg += d_view.label();
854 msg += "\"\n";
855 Kokkos::abort(msg.c_str());
856 }
857#endif
858 }
859 }
860
861 template <
862 class Dummy = DualView,
863 std::enable_if_t<Dummy::impl_dualview_is_single_device::value>* = nullptr>
864 inline void modify_device() {
865 return;
866 }
867
868 inline void clear_sync_state() {
869 if (modified_flags.data() != nullptr)
870 modified_flags(1) = modified_flags(0) = 0;
871 }
872
874
876
882 template <class... ViewCtorArgs>
883 void impl_realloc(const size_t n0, const size_t n1, const size_t n2,
884 const size_t n3, const size_t n4, const size_t n5,
885 const size_t n6, const size_t n7,
886 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop) {
887 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
888
889 static_assert(!alloc_prop_input::has_label,
890 "The view constructor arguments passed to Kokkos::realloc "
891 "must not include a label!");
892 static_assert(
893 !alloc_prop_input::has_pointer,
894 "The view constructor arguments passed to Kokkos::realloc must "
895 "not include a pointer!");
896 static_assert(
897 !alloc_prop_input::has_memory_space,
898 "The view constructor arguments passed to Kokkos::realloc must "
899 "not include a memory space instance!");
900
901 const size_t new_extents[8] = {n0, n1, n2, n3, n4, n5, n6, n7};
902 const bool sizeMismatch =
903 Impl::size_mismatch(h_view, h_view.rank_dynamic, new_extents);
904
905 if (sizeMismatch) {
906 ::Kokkos::realloc(arg_prop, d_view, n0, n1, n2, n3, n4, n5, n6, n7);
907 if (alloc_prop_input::initialize) {
908 h_view = create_mirror_view(typename t_host::memory_space(), d_view);
909 } else {
910 h_view = create_mirror_view(Kokkos::WithoutInitializing,
911 typename t_host::memory_space(), d_view);
912 }
913 } else if (alloc_prop_input::initialize) {
914 if constexpr (alloc_prop_input::has_execution_space) {
915 const auto& exec_space =
916 Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop);
917 ::Kokkos::deep_copy(exec_space, d_view, typename t_dev::value_type{});
918 } else
919 ::Kokkos::deep_copy(d_view, typename t_dev::value_type{});
920 }
921
922 /* Reset dirty flags */
923 if (modified_flags.data() == nullptr) {
924 modified_flags = t_modified_flags("DualView::modified_flags");
925 } else
926 modified_flags(1) = modified_flags(0) = 0;
927 }
928
929 template <class... ViewCtorArgs>
930 void realloc(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
931 const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
932 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
933 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
934 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
935 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
936 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
937 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
938 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
939 impl_realloc(n0, n1, n2, n3, n4, n5, n6, n7, arg_prop);
940 }
941
942 void realloc(const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
943 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
944 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
945 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
946 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
947 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
948 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
949 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
950 impl_realloc(n0, n1, n2, n3, n4, n5, n6, n7, Impl::ViewCtorProp<>{});
951 }
952
953 template <typename I>
954 std::enable_if_t<Impl::is_view_ctor_property<I>::value> realloc(
955 const I& arg_prop, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
956 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
957 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
958 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
959 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
960 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
961 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
962 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
963 impl_realloc(n0, n1, n2, n3, n4, n5, n6, n7, Kokkos::view_alloc(arg_prop));
964 }
965
970 template <class... ViewCtorArgs>
971 void impl_resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
972 const size_t n0, const size_t n1, const size_t n2,
973 const size_t n3, const size_t n4, const size_t n5,
974 const size_t n6, const size_t n7) {
975 using alloc_prop_input = Impl::ViewCtorProp<ViewCtorArgs...>;
976
977 static_assert(!alloc_prop_input::has_label,
978 "The view constructor arguments passed to Kokkos::resize "
979 "must not include a label!");
980 static_assert(
981 !alloc_prop_input::has_pointer,
982 "The view constructor arguments passed to Kokkos::resize must "
983 "not include a pointer!");
984 static_assert(
985 !alloc_prop_input::has_memory_space,
986 "The view constructor arguments passed to Kokkos::resize must "
987 "not include a memory space instance!");
988
989 const size_t new_extents[8] = {n0, n1, n2, n3, n4, n5, n6, n7};
990 const bool sizeMismatch =
991 Impl::size_mismatch(h_view, h_view.rank_dynamic, new_extents);
992
993 if (modified_flags.data() == nullptr) {
994 modified_flags = t_modified_flags("DualView::modified_flags");
995 }
996
997 [[maybe_unused]] auto resize_on_device = [&](const auto& properties) {
998 /* Resize on Device */
999 if (sizeMismatch) {
1000 ::Kokkos::resize(properties, d_view, n0, n1, n2, n3, n4, n5, n6, n7);
1001 if (alloc_prop_input::initialize) {
1002 h_view = create_mirror_view(typename t_host::memory_space(), d_view);
1003 } else {
1004 h_view = create_mirror_view(Kokkos::WithoutInitializing,
1005 typename t_host::memory_space(), d_view);
1006 }
1007
1008 /* Mark Device copy as modified */
1009 ++modified_flags(1);
1010 }
1011 };
1012
1013 [[maybe_unused]] auto resize_on_host = [&](const auto& properties) {
1014 /* Resize on Host */
1015 if (sizeMismatch) {
1016 ::Kokkos::resize(properties, h_view, n0, n1, n2, n3, n4, n5, n6, n7);
1017 if (alloc_prop_input::initialize) {
1018 d_view = create_mirror_view(typename t_dev::memory_space(), h_view);
1019
1020 } else {
1021 d_view = create_mirror_view(Kokkos::WithoutInitializing,
1022 typename t_dev::memory_space(), h_view);
1023 }
1024
1025 /* Mark Host copy as modified */
1026 ++modified_flags(0);
1027 }
1028 };
1029
1030 constexpr bool has_execution_space = alloc_prop_input::has_execution_space;
1031
1032 if constexpr (has_execution_space) {
1033 using ExecSpace = typename alloc_prop_input::execution_space;
1034 const auto& exec_space =
1035 Impl::get_property<Impl::ExecutionSpaceTag>(arg_prop);
1036 constexpr bool exec_space_can_access_device =
1037 SpaceAccessibility<ExecSpace,
1038 typename t_dev::memory_space>::accessible;
1039 constexpr bool exec_space_can_access_host =
1040 SpaceAccessibility<ExecSpace,
1041 typename t_host::memory_space>::accessible;
1042 static_assert(exec_space_can_access_device || exec_space_can_access_host);
1043 if constexpr (exec_space_can_access_device) {
1044 sync<typename t_dev::memory_space>(exec_space);
1045 resize_on_device(arg_prop);
1046 return;
1047 }
1048 if constexpr (exec_space_can_access_host) {
1049 sync<typename t_host::memory_space>(exec_space);
1050 resize_on_host(arg_prop);
1051 return;
1052 }
1053 } else {
1054 if (modified_flags(1) >= modified_flags(0)) {
1055 resize_on_device(arg_prop);
1056 } else {
1057 resize_on_host(arg_prop);
1058 }
1059 }
1060 }
1061
1062 void resize(const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1063 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1064 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1065 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1066 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1067 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1068 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1069 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
1070 impl_resize(Impl::ViewCtorProp<>{}, n0, n1, n2, n3, n4, n5, n6, n7);
1071 }
1072
1073 template <class... ViewCtorArgs>
1074 void resize(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
1075 const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1076 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1077 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1078 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1079 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1080 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1081 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1082 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
1083 impl_resize(arg_prop, n0, n1, n2, n3, n4, n5, n6, n7);
1084 }
1085
1086 template <class I>
1087 std::enable_if_t<Impl::is_view_ctor_property<I>::value> resize(
1088 const I& arg_prop, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1089 const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1090 const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1091 const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1092 const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1093 const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1094 const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG,
1095 const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) {
1096 impl_resize(Kokkos::view_alloc(arg_prop), n0, n1, n2, n3, n4, n5, n6, n7);
1097 }
1098
1100
1102
1104 KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return d_view.span(); }
1105
1106 KOKKOS_INLINE_FUNCTION bool span_is_contiguous() const {
1107 return d_view.span_is_contiguous();
1108 }
1109
1111 template <typename iType>
1112 void stride(iType* stride_) const {
1113 d_view.stride(stride_);
1114 }
1115
1116 template <typename iType>
1117 KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t<
1118 std::is_integral<iType>::value, size_t>
1119 extent(const iType& r) const {
1120 return d_view.extent(r);
1121 }
1122
1123 template <typename iType>
1124 KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t<
1125 std::is_integral<iType>::value, int>
1126 extent_int(const iType& r) const {
1127 return static_cast<int>(d_view.extent(r));
1128 }
1129
1131};
1132
1133} // namespace Kokkos
1134
1135//----------------------------------------------------------------------------
1136//----------------------------------------------------------------------------
1137//
1138// Partial specializations of Kokkos::subview() for DualView objects.
1139//
1140
1141namespace Kokkos {
1142namespace Impl {
1143
1144template <class D, class A1, class A2, class A3, class... Args>
1145struct DualViewSubview {
1146 using dst_traits = typename Kokkos::Impl::ViewMapping<
1147 void, Kokkos::ViewTraits<D, A1, A2, A3>, Args...>::traits_type;
1148
1149 using type = Kokkos::DualView<
1150 typename dst_traits::data_type, typename dst_traits::array_layout,
1151 typename dst_traits::device_type, typename dst_traits::memory_traits>;
1152};
1153
1154} /* namespace Impl */
1155
1156template <class D, class A1, class A2, class A3, class... Args>
1157typename Impl::DualViewSubview<D, A1, A2, A3, Args...>::type subview(
1158 const DualView<D, A1, A2, A3>& src, Args... args) {
1159 return typename Impl::DualViewSubview<D, A1, A2, A3, Args...>::type(src,
1160 args...);
1161}
1162
1163} /* namespace Kokkos */
1164
1165//----------------------------------------------------------------------------
1166//----------------------------------------------------------------------------
1167
1168namespace Kokkos {
1169
1170//
1171// Partial specialization of Kokkos::deep_copy() for DualView objects.
1172//
1173
1174template <class DT, class DL, class DD, class DM, class ST, class SL, class SD,
1175 class SM>
1176void deep_copy(
1177 DualView<DT, DL, DD, DM> dst, // trust me, this must not be a reference
1178 const DualView<ST, SL, SD, SM>& src) {
1179 if (src.need_sync_device()) {
1180 deep_copy(dst.h_view, src.h_view);
1181 dst.modify_host();
1182 } else {
1183 deep_copy(dst.d_view, src.d_view);
1184 dst.modify_device();
1185 }
1186}
1187
1188template <class ExecutionSpace, class DT, class DL, class DD, class DM,
1189 class ST, class SL, class SD, class SM>
1190void deep_copy(
1191 const ExecutionSpace& exec,
1192 DualView<DT, DL, DD, DM> dst, // trust me, this must not be a reference
1193 const DualView<ST, SL, SD, SM>& src) {
1194 if (src.need_sync_device()) {
1195 deep_copy(exec, dst.h_view, src.h_view);
1196 dst.modify_host();
1197 } else {
1198 deep_copy(exec, dst.d_view, src.d_view);
1199 dst.modify_device();
1200 }
1201}
1202
1203} // namespace Kokkos
1204
1205//----------------------------------------------------------------------------
1206//----------------------------------------------------------------------------
1207
1208namespace Kokkos {
1209
1210//
1211// Non-member resize and realloc
1212//
1213
1214template <class... Properties, class... Args>
1215void resize(DualView<Properties...>& dv, Args&&... args) noexcept(
1216 noexcept(dv.resize(std::forward<Args>(args)...))) {
1217 dv.resize(std::forward<Args>(args)...);
1218}
1219
1220template <class... ViewCtorArgs, class... Properties, class... Args>
1221void resize(
1222 const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
1223 DualView<Properties...>& dv,
1224 Args&&... args) noexcept(noexcept(dv.resize(arg_prop,
1225 std::forward<Args>(args)...))) {
1226 dv.resize(arg_prop, std::forward<Args>(args)...);
1227}
1228
1229template <class I, class... Properties, class... Args>
1230std::enable_if_t<Impl::is_view_ctor_property<I>::value> resize(
1231 const I& arg_prop, DualView<Properties...>& dv,
1232 Args&&... args) noexcept(noexcept(dv.resize(arg_prop,
1233 std::forward<Args>(args)...))) {
1234 dv.resize(arg_prop, std::forward<Args>(args)...);
1235}
1236
1237template <class... ViewCtorArgs, class... Properties, class... Args>
1238void realloc(const Impl::ViewCtorProp<ViewCtorArgs...>& arg_prop,
1239 DualView<Properties...>& dv,
1240 Args&&... args) noexcept(noexcept(dv
1241 .realloc(std::forward<Args>(
1242 args)...))) {
1243 dv.realloc(arg_prop, std::forward<Args>(args)...);
1244}
1245
1246template <class... Properties, class... Args>
1247void realloc(DualView<Properties...>& dv, Args&&... args) noexcept(
1248 noexcept(dv.realloc(std::forward<Args>(args)...))) {
1249 dv.realloc(std::forward<Args>(args)...);
1250}
1251
1252template <class I, class... Properties, class... Args>
1253std::enable_if_t<Impl::is_view_ctor_property<I>::value> realloc(
1254 const I& arg_prop, DualView<Properties...>& dv,
1255 Args&&... args) noexcept(noexcept(dv.realloc(arg_prop,
1256 std::forward<Args>(
1257 args)...))) {
1258 dv.realloc(arg_prop, std::forward<Args>(args)...);
1259}
1260
1261} // end namespace Kokkos
1262
1263#ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_DUALVIEW
1264#undef KOKKOS_IMPL_PUBLIC_INCLUDE
1265#undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_DUALVIEW
1266#endif
1267#endif
View
View< typename traits::non_const_data_type, typename traits::array_layout, Device< DefaultHostExecutionSpace, typename traits::host_mirror_space::memory_space >, typename traits::hooks_policy > HostMirror
Compatible HostMirror view.
Traits class for accessing attributes of a View.