40#ifndef TPETRA_DISTRIBUTOR_HPP
41#define TPETRA_DISTRIBUTOR_HPP
43#include "Tpetra_Details_DistributorActor.hpp"
44#include "Tpetra_Details_DistributorPlan.hpp"
47#include "Teuchos_as.hpp"
48#include "Teuchos_Describable.hpp"
49#include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
50#include "Teuchos_VerboseObject.hpp"
53#include "KokkosCompat_View.hpp"
54#include "Kokkos_Core.hpp"
55#include "Kokkos_TeuchosCommAdapters.hpp"
133 public Teuchos::Describable,
134 public Teuchos::ParameterListAcceptorDefaultBase {
147 explicit Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm);
160 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
161 const Teuchos::RCP<Teuchos::FancyOStream>& out);
176 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
177 const Teuchos::RCP<Teuchos::ParameterList>& plist);
195 Distributor (
const Teuchos::RCP<
const Teuchos::Comm<int> >& comm,
196 const Teuchos::RCP<Teuchos::FancyOStream>& out,
197 const Teuchos::RCP<Teuchos::ParameterList>& plist);
254 size_t createFromSends (
const Teuchos::ArrayView<const int>& exportProcIDs);
289 template <
class Ordinal>
292 const Teuchos::ArrayView<const int>& remoteProcIDs,
293 Teuchos::Array<Ordinal>& exportIDs,
294 Teuchos::Array<int>& exportProcIDs);
305 const Teuchos::ArrayView<const int>& remoteProcIDs);
340 Teuchos::ArrayView<const int>
getProcsTo()
const;
365 return plan_.howInitialized();
382 Teuchos::RCP<Distributor>
getReverse(
bool create=
true)
const;
424 template <
class ExpView,
class ImpView>
425 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
427 const ExpView &exports,
429 const ImpView &imports);
452 template <
class ExpView,
class ImpView>
453 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
455 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
456 const ImpView &imports,
457 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
483 template <
class ExpView,
class ImpView>
484 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
485 doPosts (
const ExpView &exports,
487 const ImpView &imports);
507 template <
class ExpView,
class ImpView>
508 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
509 doPosts (
const ExpView &exports,
510 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
511 const ImpView &imports,
512 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
518 template <
class ExpView,
class ImpView>
519 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
522 const ImpView &imports);
528 template <
class ExpView,
class ImpView>
529 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
531 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
532 const ImpView &imports,
533 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
539 template <
class ExpView,
class ImpView>
540 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
543 const ImpView &imports);
549 template <
class ExpView,
class ImpView>
550 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
552 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
553 const ImpView &imports,
554 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
585 describe (Teuchos::FancyOStream& out,
586 const Teuchos::EVerbosityLevel verbLevel =
587 Teuchos::Describable::verbLevel_default)
const;
597 Details::DistributorActor actor_;
603 static bool getVerbose();
609 std::unique_ptr<std::string>
610 createPrefix(
const char methodName[])
const;
613 bool verbose_ = getVerbose();
620 mutable Teuchos::RCP<Distributor> reverseDistributor_;
634 template <
class Ordinal>
635 void computeSends (
const Teuchos::ArrayView<const Ordinal> &remoteGIDs,
636 const Teuchos::ArrayView<const int> &remoteProcIDs,
637 Teuchos::Array<Ordinal> &exportGIDs,
638 Teuchos::Array<int> &exportProcIDs);
641 void createReverseDistributor()
const;
649 localDescribeToString (
const Teuchos::EVerbosityLevel vl)
const;
652 template <
class ExpView,
class ImpView>
653 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
655 doPostsAndWaits (
const ExpView& exports,
657 const ImpView& imports)
662 template <
class ExpView,
class ImpView>
663 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
665 doPostsAndWaits(
const ExpView& exports,
666 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
667 const ImpView& imports,
668 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
670 actor_.
doPostsAndWaits(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
674 template <
class ExpView,
class ImpView>
675 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
677 doPosts (
const ExpView &exports,
679 const ImpView &imports)
681 actor_.
doPosts(plan_, exports, numPackets, imports);
684 template <
class ExpView,
class ImpView>
685 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
687 doPosts (
const ExpView &exports,
688 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
689 const ImpView &imports,
690 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
692 actor_.
doPosts(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
695 template <
class ExpView,
class ImpView>
696 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
698 doReversePostsAndWaits (
const ExpView& exports,
700 const ImpView& imports)
702 doReversePosts (exports, numPackets, imports);
706 template <
class ExpView,
class ImpView>
707 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
709 doReversePostsAndWaits (
const ExpView& exports,
710 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
711 const ImpView& imports,
712 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
714 doReversePosts (exports, numExportPacketsPerLID, imports,
715 numImportPacketsPerLID);
719 template <
class ExpView,
class ImpView>
720 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
722 doReversePosts (
const ExpView &exports,
724 const ImpView &imports)
727 TEUCHOS_TEST_FOR_EXCEPTION(
728 ! plan_.getIndicesTo().is_null(), std::runtime_error,
729 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
730 "reverse communication when original data are blocked by process.");
731 if (reverseDistributor_.is_null ()) {
732 createReverseDistributor ();
734 reverseDistributor_->doPosts (exports, numPackets, imports);
737 template <
class ExpView,
class ImpView>
738 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
740 doReversePosts (
const ExpView &exports,
741 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
742 const ImpView &imports,
743 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
746 TEUCHOS_TEST_FOR_EXCEPTION(
747 ! plan_.getIndicesTo().is_null(), std::runtime_error,
748 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
749 "reverse communication when original data are blocked by process.");
750 if (reverseDistributor_.is_null ()) {
751 createReverseDistributor ();
753 reverseDistributor_->doPosts (exports, numExportPacketsPerLID,
754 imports, numImportPacketsPerLID);
757 template <
class OrdinalType>
759 computeSends(
const Teuchos::ArrayView<const OrdinalType>& importGIDs,
760 const Teuchos::ArrayView<const int>& importProcIDs,
761 Teuchos::Array<OrdinalType>& exportGIDs,
762 Teuchos::Array<int>& exportProcIDs)
771 using Teuchos::ArrayView;
773 using size_type =
typename ArrayView<const OrdinalType>::size_type;
774 const char errPrefix[] =
"Tpetra::Distributor::computeSends: ";
775 const char suffix[] =
776 " Please report this bug to the Tpetra developers.";
778 const int myRank = plan_.getComm()->getRank ();
780 TEUCHOS_TEST_FOR_EXCEPTION
781 (importGIDs.size () != importProcIDs.size (),
782 std::invalid_argument, errPrefix <<
"On Process " << myRank
783 <<
": importProcIDs.size()=" << importProcIDs.size()
784 <<
" != importGIDs.size()=" << importGIDs.size() <<
".");
786 const size_type numImports = importProcIDs.size();
787 Kokkos::View<size_t*, Kokkos::HostSpace> importObjs(
"importObjs", 2*numImports);
789 for (size_type i = 0; i < numImports; ++i) {
790 importObjs[2*i] =
static_cast<size_t>(importGIDs[i]);
791 importObjs[2*i+1] =
static_cast<size_t>(myRank);
797 Distributor tempPlan(plan_.getComm());
800 const size_t numExportsAsSizeT =
801 tempPlan.createFromSends(importProcIDs);
802 const size_type numExports =
803 static_cast<size_type
>(numExportsAsSizeT);
804 TEUCHOS_TEST_FOR_EXCEPTION
805 (numExports < 0, std::logic_error, errPrefix <<
806 "tempPlan.createFromSends() returned numExports="
807 << numExportsAsSizeT <<
" as a size_t, which overflows to "
808 << numExports <<
" when cast to " <<
809 Teuchos::TypeNameTraits<size_type>::name () <<
"." << suffix);
810 TEUCHOS_TEST_FOR_EXCEPTION
811 (size_type(tempPlan.getTotalReceiveLength()) != numExports,
812 std::logic_error, errPrefix <<
"tempPlan.getTotalReceiveLength()="
813 << tempPlan.getTotalReceiveLength () <<
" != numExports="
814 << numExports <<
"." << suffix);
816 if (numExports > 0) {
817 exportGIDs.resize(numExports);
818 exportProcIDs.resize(numExports);
829 static_assert(
sizeof(size_t) >=
sizeof(OrdinalType),
830 "Tpetra::Distributor::computeSends: "
831 "sizeof(size_t) < sizeof(OrdinalType).");
833 TEUCHOS_TEST_FOR_EXCEPTION
834 (tempPlan.getTotalReceiveLength () <
size_t(numExports),
836 errPrefix <<
"tempPlan.getTotalReceiveLength()="
837 << tempPlan.getTotalReceiveLength() <<
" < numExports="
838 << numExports <<
"." << suffix);
840 Kokkos::View<size_t*, Kokkos::HostSpace> exportObjs(
"exportObjs", tempPlan.getTotalReceiveLength() * 2);
841 tempPlan.doPostsAndWaits(importObjs, 2, exportObjs);
844 for (size_type i = 0; i < numExports; ++i) {
845 exportGIDs[i] =
static_cast<OrdinalType
> (exportObjs[2*i]);
846 exportProcIDs[i] =
static_cast<int> (exportObjs[2*i+1]);
850 template <
class OrdinalType>
852 createFromRecvs (
const Teuchos::ArrayView<const OrdinalType> &remoteGIDs,
853 const Teuchos::ArrayView<const int> &remoteProcIDs,
854 Teuchos::Array<OrdinalType> &exportGIDs,
855 Teuchos::Array<int> &exportProcIDs)
858 const char errPrefix[] =
"Tpetra::Distributor::createFromRecvs: ";
859 const int myRank = plan_.getComm()->getRank();
861 std::unique_ptr<std::string> prefix;
863 prefix = createPrefix(
"createFromRecvs");
864 std::ostringstream os;
865 os << *prefix <<
"Start" << endl;
866 std::cerr << os.str();
871 using Teuchos::outArg;
872 using Teuchos::REDUCE_MAX;
873 using Teuchos::reduceAll;
877 (remoteGIDs.size () != remoteProcIDs.size ()) ? myRank : -1;
879 reduceAll(*plan_.getComm(), REDUCE_MAX, errProc, outArg(maxErrProc));
880 TEUCHOS_TEST_FOR_EXCEPTION
881 (maxErrProc != -1, std::runtime_error, errPrefix <<
"Lists "
882 "of remote IDs and remote process IDs must have the same "
883 "size on all participating processes. Maximum process ID "
884 "with error: " << maxErrProc <<
".");
889 TEUCHOS_TEST_FOR_EXCEPTION
890 (remoteGIDs.size() != remoteProcIDs.size(), std::runtime_error,
891 errPrefix <<
"On Process " << myRank <<
": "
892 "remoteGIDs.size()=" << remoteGIDs.size() <<
893 " != remoteProcIDs.size()=" << remoteProcIDs.size() <<
".");
896 computeSends(remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
898 plan_.createFromRecvs(remoteProcIDs);
901 std::ostringstream os;
902 os << *prefix <<
"Done" << endl;
903 std::cerr << os.str();
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.
Stand-alone utility functions and macros.
static bool debug()
Whether Tpetra is in debug mode.
Sets up and executes a communication plan for a Tpetra DistObject.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a forward plan, but do not execute the waits yet.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan.
const Details::DistributorPlan & getPlan() const
Get this Distributor's DistributorPlan.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
size_t getNumReceives() const
The number of processes from which we will receive data.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the reverse communication plan.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
virtual ~Distributor()=default
Destructor (virtual for memory safety).
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
std::string description() const
Return a one-line description of this object.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a reverse plan, but do not execute the waits yet.
Details::EDistributorHowInitialized howInitialized() const
Return an enum indicating whether and how a Distributor was initialized.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Teuchos::Array< std::string > distributorSendTypes()
Valid values for Distributor's "Send type" parameter.