Tpetra parallel linear algebra Version of the Day
Loading...
Searching...
No Matches
Tpetra_DistObject_decl.hpp
Go to the documentation of this file.
1// @HEADER
2// ***********************************************************************
3//
4// Tpetra: Templated Linear Algebra Services Package
5// Copyright (2008) Sandia Corporation
6//
7// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8// the U.S. Government retains certain rights in this software.
9//
10// Redistribution and use in source and binary forms, with or without
11// modification, are permitted provided that the following conditions are
12// met:
13//
14// 1. Redistributions of source code must retain the above copyright
15// notice, this list of conditions and the following disclaimer.
16//
17// 2. Redistributions in binary form must reproduce the above copyright
18// notice, this list of conditions and the following disclaimer in the
19// documentation and/or other materials provided with the distribution.
20//
21// 3. Neither the name of the Corporation nor the names of the
22// contributors may be used to endorse or promote products derived from
23// this software without specific prior written permission.
24//
25// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36//
37// ************************************************************************
38// @HEADER
39
40#ifndef TPETRA_DISTOBJECT_DECL_HPP
41#define TPETRA_DISTOBJECT_DECL_HPP
42
45
46#include "Tpetra_Details_DistributorActor.hpp"
47#include "Tpetra_Map.hpp"
48#include "Tpetra_Import.hpp"
49#include "Tpetra_Export.hpp"
52#include "Kokkos_ArithTraits.hpp"
53#include <memory>
54#include <type_traits>
55
56// #ifndef HAVE_TPETRA_TRANSFER_TIMERS
57// # define HAVE_TPETRA_TRANSFER_TIMERS 1
58// #endif // HAVE_TPETRA_TRANSFER_TIMERS
59
60#ifdef HAVE_TPETRA_TRANSFER_TIMERS
61# undef HAVE_TPETRA_TRANSFER_TIMERS
62#endif // HAVE_TPETRA_TRANSFER_TIMERS
63
64namespace KokkosClassic {
70 ReadWrite = 0,
71 OverwriteAll = 1
72 };
73} // namespace KokkosClassic
74
75namespace Tpetra {
76
159 template<class DistObjectType>
160 void
161 removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
162 const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
163 typename DistObjectType::global_ordinal_type,
164 typename DistObjectType::node_type> >& newMap);
165
202 template<class DistObjectType>
203 void
204 removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input);
205
317 template <class Packet,
318 class LocalOrdinal,
319 class GlobalOrdinal,
320 class Node>
322 virtual public SrcDistObject,
323 virtual public Teuchos::Describable
324 {
325 public:
327
328
333 using packet_type = typename ::Kokkos::Details::ArithTraits<Packet>::val_type;
335 using local_ordinal_type = LocalOrdinal;
337 using global_ordinal_type = GlobalOrdinal;
339 using node_type = Node;
340
342 using device_type = typename Node::device_type;
344 using execution_space = typename device_type::execution_space;
345
348
350
352
356 explicit DistObject (const Teuchos::RCP<const map_type>& map);
357
360
363
366
369
379 virtual ~DistObject () = default;
380
382
384
412 void
413 doImport (const SrcDistObject& source,
415 const CombineMode CM,
416 const bool restrictedMode = false);
417
445 void
446 doExport (const SrcDistObject& source,
448 const CombineMode CM,
449 const bool restrictedMode = false);
450
479 void
480 doImport (const SrcDistObject& source,
482 const CombineMode CM,
483 const bool restrictedMode = false);
484
513 void
514 doExport (const SrcDistObject& source,
516 const CombineMode CM,
517 const bool restrictedMode = false);
518
519 void
520 beginImport(const SrcDistObject& source,
522 const CombineMode CM,
523 const bool restrictedMode = false);
524
525 void
526 beginExport(const SrcDistObject& source,
528 const CombineMode CM,
529 const bool restrictedMode = false);
530
531 void
532 beginImport(const SrcDistObject& source,
534 const CombineMode CM,
535 const bool restrictedMode = false);
536
537 void
538 beginExport(const SrcDistObject& source,
540 const CombineMode CM,
541 const bool restrictedMode = false);
542
543 void
544 endImport(const SrcDistObject& source,
546 const CombineMode CM,
547 const bool restrictedMode = false);
548
549 void
550 endExport(const SrcDistObject& source,
552 const CombineMode CM,
553 const bool restrictedMode = false);
554
555 void
556 endImport(const SrcDistObject& source,
558 const CombineMode CM,
559 const bool restrictedMode = false);
560
561 void
562 endExport(const SrcDistObject& source,
564 const CombineMode CM,
565 const bool restrictedMode = false);
566
569 bool transferArrived() const;
570
572
574
580 bool isDistributed () const;
581
588 virtual Teuchos::RCP<const map_type> getMap () const { return map_; }
589
591
593
598 void print (std::ostream& os) const;
599
601
603
608 virtual std::string description () const;
609
614 virtual void
615 describe (Teuchos::FancyOStream &out,
616 const Teuchos::EVerbosityLevel verbLevel =
617 Teuchos::Describable::verbLevel_default) const;
618
620
622
669 virtual void
670 removeEmptyProcessesInPlace (const Teuchos::RCP<const map_type>& newMap);
671
673
674 protected:
684 DoForward, //*!< Perform the transfer in forward mode.
685 DoReverse //*!< Perform the transfer in reverse mode.
686 };
687
704 virtual size_t constantNumberOfPackets () const;
705
725 virtual void
726 doTransfer (const SrcDistObject& src,
727 const ::Tpetra::Details::Transfer<local_ordinal_type, global_ordinal_type, node_type>& transfer,
728 const char modeString[],
729 const ReverseOption revOp,
730 const CombineMode CM,
731 const bool restrictedMode);
732
747 virtual bool
748 reallocArraysForNumPacketsPerLid (const size_t numExportLIDs,
749 const size_t numImportLIDs);
750
751
755 ::Tpetra::Details::DefaultTypes::comm_buffer_memory_space<device_type>;
756
757 public:
769 Kokkos::Device<typename device_type::execution_space,
771 protected:
777 void beginTransfer(const SrcDistObject& src,
778 const ::Tpetra::Details::Transfer<local_ordinal_type, global_ordinal_type, node_type>& transfer,
779 const char modeString[],
780 const ReverseOption revOp,
781 const CombineMode CM,
782 const bool restrictedMode);
783
784 void endTransfer(const SrcDistObject& src,
785 const ::Tpetra::Details::Transfer<local_ordinal_type, global_ordinal_type, node_type>& transfer,
786 const char modeString[],
787 const ReverseOption revOp,
788 const CombineMode CM,
789 const bool restrictedMode);
790
791 void doPosts(const Details::DistributorPlan& distributorPlan,
792 size_t constantNumPackets,
793 bool commOnHost,
794 std::shared_ptr<std::string> prefix,
795 const bool canTryAliasing,
796 const CombineMode CM);
797
798 void doPackAndPrepare(const SrcDistObject& src,
799 const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& exportLIDs,
800 size_t& constantNumPackets);
801
802 void doUnpackAndCombine(const Kokkos::DualView<const local_ordinal_type*, buffer_device_type>& remoteLIDs,
803 size_t constantNumPackets,
804 CombineMode CM);
805
816
817
821 virtual bool
822 checkSizes (const SrcDistObject& source) = 0;
823
853 virtual void
854 copyAndPermute (const SrcDistObject& source,
855 const size_t numSameIDs,
856 const Kokkos::DualView<const local_ordinal_type*,
857 buffer_device_type>& permuteToLIDs,
858 const Kokkos::DualView<const local_ordinal_type*,
859 buffer_device_type>& permuteFromLIDs,
860 const CombineMode CM);
861
899 virtual void
900 packAndPrepare (const SrcDistObject& source,
901 const Kokkos::DualView<const local_ordinal_type*,
902 buffer_device_type>& exportLIDs,
903 Kokkos::DualView<packet_type*,
904 buffer_device_type>& exports,
905 Kokkos::DualView<size_t*,
906 buffer_device_type> numPacketsPerLID,
907 size_t& constantNumPackets);
908
948 virtual void
949 unpackAndCombine (const Kokkos::DualView<const local_ordinal_type*,
950 buffer_device_type>& importLIDs,
951 Kokkos::DualView<packet_type*,
952 buffer_device_type> imports,
953 Kokkos::DualView<size_t*,
954 buffer_device_type> numPacketsPerLID,
955 const size_t constantNumPackets,
956 const CombineMode combineMode);
957
958
960 Teuchos::RCP<const map_type> map_;
961
962 protected:
963 std::unique_ptr<std::string>
964 createPrefix(const char className[],
965 const char methodName[]) const;
966
973 Kokkos::DualView<packet_type*, buffer_device_type> imports_;
974
993 virtual bool
994 reallocImportsIfNeeded (const size_t newSize,
995 const bool verbose,
996 const std::string* prefix,
997 const bool remoteLIDsContiguous=false,
998 const CombineMode CM=INSERT);
999
1013 Kokkos::DualView<size_t*, buffer_device_type> numImportPacketsPerLID_;
1014
1020 Kokkos::DualView<packet_type*, buffer_device_type> exports_;
1021
1035 Kokkos::DualView<size_t*, buffer_device_type> numExportPacketsPerLID_;
1036
1037 private:
1039
1040 Details::DistributorActor distributorActor_;
1041
1042#ifdef HAVE_TPETRA_TRANSFER_TIMERS
1043 Teuchos::RCP<Teuchos::Time> doXferTimer_;
1044 Teuchos::RCP<Teuchos::Time> copyAndPermuteTimer_;
1045 Teuchos::RCP<Teuchos::Time> packAndPrepareTimer_;
1046 Teuchos::RCP<Teuchos::Time> doPostsAndWaitsTimer_;
1047 Teuchos::RCP<Teuchos::Time> unpackAndCombineTimer_;
1048#endif // HAVE_TPETRA_TRANSFER_TIMERS
1049 }; // class DistObject
1050} // namespace Tpetra
1051
1052#endif // TPETRA_DISTOBJECT_DECL_HPP
ReadWriteOption
Read/write options for non-const views.
Forward declaration of Tpetra::DistObject.
Abstract base class for sources of an Import or Export.
Base class for distributed Tpetra objects that support data redistribution.
DistObject(const DistObject< Packet, LocalOrdinal, GlobalOrdinal, Node > &)=default
Copy constructor (default).
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Print a descriptiion of this object to the given output stream.
virtual bool reallocImportsIfNeeded(const size_t newSize, const bool verbose, const std::string *prefix, const bool remoteLIDsContiguous=false, const CombineMode CM=INSERT)
Reallocate imports_ if needed.
::Tpetra::Details::DefaultTypes::comm_buffer_memory_space< device_type > buffer_memory_space
Kokkos memory space for communication buffers.
DistObject(DistObject< Packet, LocalOrdinal, GlobalOrdinal, Node > &&)=default
Move constructor (default).
Kokkos::DualView< size_t *, buffer_device_type > numImportPacketsPerLID_
Number of packets to receive for each receive operation.
Kokkos::DualView< packet_type *, buffer_device_type > exports_
Buffer from which packed data are exported (sent to other processes).
Kokkos::DualView< packet_type *, buffer_device_type > imports_
Buffer into which packed data are imported (received from other processes).
virtual bool reallocArraysForNumPacketsPerLid(const size_t numExportLIDs, const size_t numImportLIDs)
Reallocate numExportPacketsPerLID_ and/or numImportPacketsPerLID_, if necessary.
void doImport(const SrcDistObject &source, const Import< LocalOrdinal, GlobalOrdinal, Node > &importer, const CombineMode CM, const bool restrictedMode=false)
Import data into this object using an Import object ("forward mode").
void beginTransfer(const SrcDistObject &src, const ::Tpetra::Details::Transfer< local_ordinal_type, global_ordinal_type, node_type > &transfer, const char modeString[], const ReverseOption revOp, const CombineMode CM, const bool restrictedMode)
Implementation detail of doTransfer.
bool transferArrived() const
Whether the data from an import/export operation has arrived, and is ready for the unpack and combine...
virtual void packAndPrepare(const SrcDistObject &source, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &exportLIDs, Kokkos::DualView< packet_type *, buffer_device_type > &exports, Kokkos::DualView< size_t *, buffer_device_type > numPacketsPerLID, size_t &constantNumPackets)
Pack data and metadata for communication (sends).
DistObject & operator=(const DistObject< Packet, LocalOrdinal, GlobalOrdinal, Node > &)=default
Assignment operator (default).
Kokkos::Device< typename device_type::execution_space, buffer_memory_space > buffer_device_type
Kokkos::Device specialization for communication buffers.
LocalOrdinal local_ordinal_type
The type of local indices.
typename Node::device_type device_type
The Kokkos Device type.
virtual void doTransfer(const SrcDistObject &src, const ::Tpetra::Details::Transfer< local_ordinal_type, global_ordinal_type, node_type > &transfer, const char modeString[], const ReverseOption revOp, const CombineMode CM, const bool restrictedMode)
Redistribute data across (MPI) processes.
virtual bool checkSizes(const SrcDistObject &source)=0
Compare the source and target (this) objects for compatibility.
typename device_type::execution_space execution_space
The Kokkos execution space.
void print(std::ostream &os) const
Print this object to the given output stream.
GlobalOrdinal global_ordinal_type
The type of global indices.
Kokkos::DualView< size_t *, buffer_device_type > numExportPacketsPerLID_
Number of packets to send for each send operation.
virtual void unpackAndCombine(const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &importLIDs, Kokkos::DualView< packet_type *, buffer_device_type > imports, Kokkos::DualView< size_t *, buffer_device_type > numPacketsPerLID, const size_t constantNumPackets, const CombineMode combineMode)
Perform any unpacking and combining after communication.
typename ::Kokkos::Details::ArithTraits< Packet >::val_type packet_type
The type of each datum being sent or received in an Import or Export.
virtual void copyAndPermute(const SrcDistObject &source, const size_t numSameIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteToLIDs, const Kokkos::DualView< const local_ordinal_type *, buffer_device_type > &permuteFromLIDs, const CombineMode CM)
Perform copies and permutations that are local to the calling (MPI) process.
Teuchos::RCP< const map_type > map_
The Map over which this object is distributed.
ReverseOption
Whether the data transfer should be performed in forward or reverse mode.
Node node_type
The Node type. If you don't know what this is, don't use it.
virtual size_t constantNumberOfPackets() const
Whether the implementation's instance promises always to have a constant number of packets per LID (l...
void doExport(const SrcDistObject &source, const Export< LocalOrdinal, GlobalOrdinal, Node > &exporter, const CombineMode CM, const bool restrictedMode=false)
Export data into this object using an Export object ("forward mode").
virtual std::string description() const
One-line descriptiion of this object.
virtual Teuchos::RCP< const map_type > getMap() const
The Map describing the parallel distribution of this object.
virtual ~DistObject()=default
Destructor (virtual for memory safety of derived classes).
virtual void removeEmptyProcessesInPlace(const Teuchos::RCP< const map_type > &newMap)
Remove processes which contain no entries in this object's Map.
bool isDistributed() const
Whether this is a globally distributed object.
Communication plan for data redistribution from a (possibly) multiply-owned to a uniquely-owned distr...
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distr...
A parallel distribution of indices over processes.
Abstract base class for objects that can be the source of an Import or Export operation.
Namespace Tpetra contains the class and methods constituting the Tpetra library.
void removeEmptyProcessesInPlace(Teuchos::RCP< DistObjectType > &input, const Teuchos::RCP< const Map< typename DistObjectType::local_ordinal_type, typename DistObjectType::global_ordinal_type, typename DistObjectType::node_type > > &newMap)
Remove processes which contain no elements in this object's Map.
CombineMode
Rule for combining data in an Import or Export.
@ INSERT
Insert new values that don't currently exist.