19 #include <deal.II/base/config.h> 21 #include <deal.II/base/array_view.h> 26 #if !defined(DEAL_II_WITH_MPI) && !defined(DEAL_II_WITH_PETSC) 31 using MPI_Datatype = int;
33 # ifndef MPI_COMM_WORLD 34 # define MPI_COMM_WORLD 0 36 # ifndef MPI_COMM_SELF 37 # define MPI_COMM_SELF 0 50 DEAL_II_NAMESPACE_OPEN
54 template <
int rank,
int dim,
typename Number>
56 template <
int rank,
int dim,
typename Number>
58 template <
typename Number>
115 std::vector<unsigned int>
117 const MPI_Comm & mpi_comm,
118 const std::vector<unsigned int> &destinations);
163 #ifdef DEAL_II_WITH_MPI 166 const MPI_Group &group,
168 MPI_Comm * new_comm);
190 template <
typename T>
192 sum(
const T &t,
const MPI_Comm &mpi_communicator);
203 template <
typename T,
typename U>
205 sum(
const T &values,
const MPI_Comm &mpi_communicator, U &sums);
216 template <
typename T>
219 const MPI_Comm & mpi_communicator,
227 template <
int rank,
int dim,
typename Number>
230 const MPI_Comm & mpi_communicator);
237 template <
int rank,
int dim,
typename Number>
240 const MPI_Comm & mpi_communicator);
250 template <
typename Number>
253 const MPI_Comm & mpi_communicator,
275 template <
typename T>
277 max(
const T &t,
const MPI_Comm &mpi_communicator);
288 template <
typename T,
typename U>
290 max(
const T &values,
const MPI_Comm &mpi_communicator, U &maxima);
301 template <
typename T>
304 const MPI_Comm & mpi_communicator,
326 template <
typename T>
328 min(
const T &t,
const MPI_Comm &mpi_communicator);
339 template <
typename T,
typename U>
341 min(
const T &values,
const MPI_Comm &mpi_communicator, U &minima);
352 template <
typename T>
355 const MPI_Comm & mpi_communicator,
434 min_max_avg(
const double my_value,
const MPI_Comm &mpi_communicator);
569 template <
typename T>
570 std::map<unsigned int, T>
572 const std::map<unsigned int, T> &objects_to_send);
589 template <
typename T>
591 all_gather(
const MPI_Comm &comm,
const T &object_to_send);
610 template <
typename T>
612 gather(
const MPI_Comm & comm,
613 const T & object_to_send,
614 const unsigned int root_process = 0);
620 template <
typename T>
622 all_reduce(
const MPI_Op & mpi_op,
624 const MPI_Comm & mpi_communicator,
629 template <
typename T,
unsigned int N>
631 sum(
const T (&values)[N],
const MPI_Comm &mpi_communicator, T (&sums)[N])
633 internal::all_reduce(MPI_SUM,
639 template <
typename T,
unsigned int N>
641 max(
const T (&values)[N],
const MPI_Comm &mpi_communicator, T (&maxima)[N])
643 internal::all_reduce(MPI_MAX,
649 template <
typename T,
unsigned int N>
651 min(
const T (&values)[N],
const MPI_Comm &mpi_communicator, T (&minima)[N])
653 internal::all_reduce(MPI_MIN,
659 template <
typename T>
660 std::map<unsigned int, T>
662 const std::map<unsigned int, T> &objects_to_send)
664 # ifndef DEAL_II_WITH_MPI 666 Assert(objects_to_send.size() == 0,
667 ExcMessage(
"Cannot send to more than one processor."));
668 Assert(objects_to_send.find(0) != objects_to_send.end() ||
669 objects_to_send.size() == 0,
670 ExcMessage(
"Can only send to myself or to nobody."));
671 return objects_to_send;
674 std::vector<unsigned int> send_to(objects_to_send.size());
677 for (
const auto &m : objects_to_send)
678 send_to[i++] = m.first;
682 const auto receive_from =
687 std::vector<std::vector<char>> buffers_to_send(send_to.size());
688 std::vector<MPI_Request> buffer_send_requests(send_to.size());
691 for (
const auto &rank_obj : objects_to_send)
693 const auto &rank = rank_obj.first;
695 const int ierr = MPI_Isend(buffers_to_send[i].data(),
696 buffers_to_send[i].size(),
701 &buffer_send_requests[i]);
708 std::map<unsigned int, T> received_objects;
710 std::vector<char> buffer;
712 for (
unsigned int i = 0; i < receive_from.size(); ++i)
716 int ierr = MPI_Probe(MPI_ANY_SOURCE, 21, comm, &status);
721 ierr = MPI_Get_count(&status, MPI_CHAR, &len);
726 const unsigned int rank = status.MPI_SOURCE;
730 buffer.data(), len, MPI_CHAR, rank, 21, comm, MPI_STATUS_IGNORE);
732 Assert(received_objects.find(rank) == received_objects.end(),
734 "I should not receive again from this rank"));
735 received_objects[rank] = Utilities::unpack<T>(buffer);
740 MPI_Waitall(send_to.size(),
741 buffer_send_requests.data(),
742 MPI_STATUSES_IGNORE);
744 return received_objects;
745 # endif // deal.II with MPI 748 template <
typename T>
750 all_gather(
const MPI_Comm &comm,
const T &
object)
752 # ifndef DEAL_II_WITH_MPI 754 std::vector<T> v(1,
object);
757 const auto n_procs = ::Utilities::MPI::n_mpi_processes(comm);
761 int n_local_data = buffer.size();
764 std::vector<int> size_all_data(n_procs, 0);
768 &n_local_data, 1, MPI_INT, &(size_all_data[0]), 1, MPI_INT, comm);
772 std::vector<int> rdispls(n_procs);
774 for (
unsigned int i = 1; i < n_procs; ++i)
775 rdispls[i] = rdispls[i - 1] + size_all_data[i - 1];
778 std::vector<char> received_unrolled_buffer(rdispls.back() +
779 size_all_data.back());
781 MPI_Allgatherv(buffer.data(),
784 received_unrolled_buffer.data(),
785 size_all_data.data(),
790 std::vector<T> received_objects(n_procs);
791 for (
unsigned int i = 0; i < n_procs; ++i)
793 std::vector<char> local_buffer(received_unrolled_buffer.begin() +
795 received_unrolled_buffer.begin() +
796 rdispls[i] + size_all_data[i]);
797 received_objects[i] = Utilities::unpack<T>(local_buffer);
800 return received_objects;
804 template <
typename T>
806 gather(
const MPI_Comm & comm,
807 const T & object_to_send,
808 const unsigned int root_process)
810 # ifndef DEAL_II_WITH_MPI 813 std::vector<T> v(1, object_to_send);
816 const auto n_procs = ::Utilities::MPI::n_mpi_processes(comm);
817 const auto my_rank = ::Utilities::MPI::this_mpi_process(comm);
822 int n_local_data = buffer.size();
826 std::vector<int> size_all_data;
827 if (my_rank == root_process)
828 size_all_data.resize(n_procs, 0);
831 int ierr = MPI_Gather(&n_local_data,
834 size_all_data.data(),
843 std::vector<int> rdispls;
844 if (my_rank == root_process)
846 rdispls.resize(n_procs, 0);
847 for (
unsigned int i = 1; i < n_procs; ++i)
848 rdispls[i] = rdispls[i - 1] + size_all_data[i - 1];
851 std::vector<char> received_unrolled_buffer;
852 if (my_rank == root_process)
853 received_unrolled_buffer.resize(rdispls.back() + size_all_data.back());
855 ierr = MPI_Gatherv(buffer.data(),
858 received_unrolled_buffer.data(),
859 size_all_data.data(),
866 std::vector<T> received_objects;
868 if (my_rank == root_process)
870 received_objects.resize(n_procs);
872 for (
unsigned int i = 0; i < n_procs; ++i)
874 const std::vector<char> local_buffer(
875 received_unrolled_buffer.begin() + rdispls[i],
876 received_unrolled_buffer.begin() + rdispls[i] +
878 received_objects[i] = Utilities::unpack<T>(local_buffer);
881 return received_objects;
890 DEAL_II_NAMESPACE_CLOSE
static const unsigned int invalid_unsigned_int
#define AssertDimension(dim1, dim2)
static::ExceptionBase & ExcIndexRange(int arg1, int arg2, int arg3)
static::ExceptionBase & ExcMessage(std::string arg1)
T sum(const T &t, const MPI_Comm &mpi_communicator)
#define Assert(cond, exc)
std::vector< T > gather(const MPI_Comm &comm, const T &object_to_send, const unsigned int root_process=0)
int create_group(const MPI_Comm &comm, const MPI_Group &group, const int tag, MPI_Comm *new_comm)
size_t pack(const T &object, std::vector< char > &dest_buffer, const bool allow_compression=true)
unsigned int n_mpi_processes(const MPI_Comm &mpi_communicator)
#define AssertThrowMPI(error_code)
MPI_Comm duplicate_communicator(const MPI_Comm &mpi_communicator)
T min(const T &t, const MPI_Comm &mpi_communicator)
unsigned int this_mpi_process(const MPI_Comm &mpi_communicator)
std::vector< T > all_gather(const MPI_Comm &comm, const T &object_to_send)
MinMaxAvg min_max_avg(const double my_value, const MPI_Comm &mpi_communicator)
std::vector< unsigned int > compute_point_to_point_communication_pattern(const MPI_Comm &mpi_comm, const std::vector< unsigned int > &destinations)
std::map< unsigned int, T > some_to_some(const MPI_Comm &comm, const std::map< unsigned int, T > &objects_to_send)
T max(const T &t, const MPI_Comm &mpi_communicator)
static::ExceptionBase & ExcInternalError()