libMesh::Parallel::Communicator Class Reference

#include <communicator.h>

Public Types

enum  SendMode { DEFAULT =0, SYNCHRONOUS }
 

Public Member Functions

 Communicator ()
 
 Communicator (const communicator &comm)
 
 Communicator (const Communicator &)=delete
 
Communicatoroperator= (const Communicator &)=delete
 
 Communicator (Communicator &&)=default
 
Communicatoroperator= (Communicator &&)=default
 
 ~Communicator ()
 
void split (int color, int key, Communicator &target) const
 
void duplicate (const Communicator &comm)
 
void duplicate (const communicator &comm)
 
communicatorget ()
 
const communicatorget () const
 
MessageTag get_unique_tag (int tagvalue) const
 
void reference_unique_tag (int tagvalue) const
 
void dereference_unique_tag (int tagvalue) const
 
void clear ()
 
Communicatoroperator= (const communicator &comm)
 
processor_id_type rank () const
 
processor_id_type size () const
 
void send_mode (const SendMode sm)
 
SendMode send_mode () const
 
void barrier () const
 
template<typename T >
bool verify (const T &r) const
 
template<typename T >
bool semiverify (const T *r) const
 
template<typename T >
void min (T &r) const
 
template<typename T >
void minloc (T &r, unsigned int &min_id) const
 
template<typename T , typename A1 , typename A2 >
void minloc (std::vector< T, A1 > &r, std::vector< unsigned int, A2 > &min_id) const
 
template<typename T >
void max (T &r) const
 
template<typename T >
void maxloc (T &r, unsigned int &max_id) const
 
template<typename T , typename A1 , typename A2 >
void maxloc (std::vector< T, A1 > &r, std::vector< unsigned int, A2 > &max_id) const
 
template<typename T >
void sum (T &r) const
 
template<typename T >
void set_union (T &data, const unsigned int root_id) const
 
template<typename T >
void set_union (T &data) const
 
status probe (const unsigned int src_processor_id, const MessageTag &tag=any_tag) const
 
template<typename T >
Status packed_range_probe (const unsigned int src_processor_id, const MessageTag &tag, bool &flag) const
 
template<typename T >
void send (const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
 
template<typename T >
void send (const unsigned int dest_processor_id, const T &buf, Request &req, const MessageTag &tag=no_tag) const
 
template<typename T >
void send (const unsigned int dest_processor_id, const T &buf, const DataType &type, const MessageTag &tag=no_tag) const
 
template<typename T >
void send (const unsigned int dest_processor_id, const T &buf, const DataType &type, Request &req, const MessageTag &tag=no_tag) const
 
template<typename T >
Status receive (const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
 
template<typename T >
void receive (const unsigned int dest_processor_id, T &buf, Request &req, const MessageTag &tag=any_tag) const
 
template<typename T >
Status receive (const unsigned int dest_processor_id, T &buf, const DataType &type, const MessageTag &tag=any_tag) const
 
template<typename T >
void receive (const unsigned int dest_processor_id, T &buf, const DataType &type, Request &req, const MessageTag &tag=any_tag) const
 
template<typename Context , typename Iter >
void send_packed_range (const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, const MessageTag &tag=no_tag) const
 
template<typename Context , typename Iter >
void send_packed_range (const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, Request &req, const MessageTag &tag=no_tag) const
 
template<typename Context , typename Iter >
void nonblocking_send_packed_range (const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, Request &req, const MessageTag &tag=no_tag) const
 
template<typename Context , typename Iter >
void nonblocking_send_packed_range (const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, Request &req, std::shared_ptr< std::vector< typename Parallel::Packing< typename std::iterator_traits< Iter >::value_type >::buffer_type >> &buffer, const MessageTag &tag=no_tag) const
 
template<typename Context , typename OutputIter , typename T >
void receive_packed_range (const unsigned int dest_processor_id, Context *context, OutputIter out, const T *output_type, const MessageTag &tag=any_tag) const
 
template<typename Context , typename OutputIter , typename T >
void nonblocking_receive_packed_range (const unsigned int src_processor_id, Context *context, OutputIter out, const T *output_type, Request &req, Status &stat, const MessageTag &tag=any_tag) const
 
template<typename Context , typename OutputIter , typename T >
void nonblocking_receive_packed_range (const unsigned int src_processor_id, Context *context, OutputIter out, const T *output_type, Request &req, Status &stat, std::shared_ptr< std::vector< typename Parallel::Packing< T >::buffer_type >> &buffer, const MessageTag &tag=any_tag) const
 
template<typename T1 , typename T2 >
void send_receive (const unsigned int dest_processor_id, const T1 &send, const unsigned int source_processor_id, T2 &recv, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
 
template<typename Context1 , typename RangeIter , typename Context2 , typename OutputIter , typename T >
void send_receive_packed_range (const unsigned int dest_processor_id, const Context1 *context1, RangeIter send_begin, const RangeIter send_end, const unsigned int source_processor_id, Context2 *context2, OutputIter out, const T *output_type, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
 
template<typename T1 , typename T2 >
void send_receive (const unsigned int dest_processor_id, const T1 &send, const DataType &type1, const unsigned int source_processor_id, T2 &recv, const DataType &type2, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
 
template<typename T , typename A >
void gather (const unsigned int root_id, const T &send, std::vector< T, A > &recv) const
 
template<typename T , typename A >
void gather (const unsigned int root_id, const std::basic_string< T > &send, std::vector< std::basic_string< T >, A > &recv, const bool identical_buffer_sizes=false) const
 
template<typename T , typename A >
void gather (const unsigned int root_id, std::vector< T, A > &r) const
 
template<typename T , typename A >
void allgather (const T &send, std::vector< T, A > &recv) const
 
template<typename T , typename A >
void allgather (const std::basic_string< T > &send, std::vector< std::basic_string< T >, A > &recv, const bool identical_buffer_sizes=false) const
 
template<typename T , typename A >
void allgather (std::vector< T, A > &r, const bool identical_buffer_sizes=false) const
 
template<typename T , typename A >
void allgather (std::vector< std::basic_string< T >, A > &r, const bool identical_buffer_sizes=false) const
 
template<typename T , typename A >
void scatter (const std::vector< T, A > &data, T &recv, const unsigned int root_id=0) const
 
template<typename T , typename A >
void scatter (const std::vector< T, A > &data, std::vector< T, A > &recv, const unsigned int root_id=0) const
 
template<typename T , typename A1 , typename A2 >
void scatter (const std::vector< T, A1 > &data, const std::vector< int, A2 > counts, std::vector< T, A1 > &recv, const unsigned int root_id=0) const
 
template<typename T , typename A1 , typename A2 >
void scatter (const std::vector< std::vector< T, A1 >, A2 > &data, std::vector< T, A1 > &recv, const unsigned int root_id=0, const bool identical_buffer_sizes=false) const
 
template<typename Context , typename Iter , typename OutputIter >
void gather_packed_range (const unsigned int root_id, Context *context, Iter range_begin, const Iter range_end, OutputIter out) const
 
template<typename Context , typename Iter , typename OutputIter >
void allgather_packed_range (Context *context, Iter range_begin, const Iter range_end, OutputIter out) const
 
template<typename T , typename A >
void alltoall (std::vector< T, A > &r) const
 
template<typename T >
void broadcast (T &data, const unsigned int root_id=0) const
 
template<typename Context , typename OutputContext , typename Iter , typename OutputIter >
void broadcast_packed_range (const Context *context1, Iter range_begin, const Iter range_end, OutputContext *context2, OutputIter out, const unsigned int root_id=0) const
 
template<typename T >
void send (const unsigned int dest_processor_id, const std::basic_string< T > &buf, const MessageTag &tag) const
 
template<typename T >
void send (const unsigned int dest_processor_id, const std::basic_string< T > &buf, Request &req, const MessageTag &tag) const
 
template<typename T >
Status receive (const unsigned int src_processor_id, std::basic_string< T > &buf, const MessageTag &tag) const
 
template<typename T >
void receive (const unsigned int src_processor_id, std::basic_string< T > &buf, Request &req, const MessageTag &tag) const
 
template<typename T , typename A >
void send_receive (const unsigned int dest_processor_id, const std::vector< T, A > &sendvec, const unsigned int source_processor_id, std::vector< T, A > &recv, const MessageTag &send_tag, const MessageTag &recv_tag) const
 
template<>
void broadcast (bool &data, const unsigned int root_id) const
 
template<typename T >
void broadcast (std::basic_string< T > &data, const unsigned int root_id) const
 
template<typename Context1 , typename RangeIter , typename Context2 , typename OutputIter , typename T >
void send_receive_packed_range (const unsigned int libmesh_dbg_var(dest_processor_id), const Context1 *context1, RangeIter send_begin, const RangeIter send_end, const unsigned int libmesh_dbg_var(source_processor_id), Context2 *context2, OutputIter out_iter, const T *output_type, const MessageTag &, const MessageTag &) const
 
template<typename T >
void min (T &libmesh_mpi_var(r)) const
 
template<typename A >
void min (std::vector< bool, A > &r) const
 
template<typename A1 , typename A2 >
void minloc (std::vector< bool, A1 > &r, std::vector< unsigned int, A2 > &min_id) const
 
template<typename T >
void max (T &libmesh_mpi_var(r)) const
 
template<typename A >
void max (std::vector< bool, A > &r) const
 
template<typename A1 , typename A2 >
void maxloc (std::vector< bool, A1 > &r, std::vector< unsigned int, A2 > &max_id) const
 
template<typename T >
void sum (T &libmesh_mpi_var(r)) const
 
template<typename T >
void sum (std::complex< T > &libmesh_mpi_var(r)) const
 
template<typename T >
void broadcast (T &libmesh_mpi_var(data), const unsigned int root_id) const
 

Private Member Functions

void assign (const communicator &comm)
 

Private Attributes

communicator _communicator
 
processor_id_type _rank
 
processor_id_type _size
 
SendMode _send_mode
 
std::map< int, unsigned int > used_tag_values
 
bool _I_duped_it
 

Detailed Description

Encapsulates the MPI_Comm object. Allows the size of the group and this process's position in the group to be determined.

Methods of this object are the preferred way to perform distributed-memory parallel operations.

Definition at line 92 of file communicator.h.

Member Enumeration Documentation

◆ SendMode

Whether to use default or synchronous sends?

Enumerator
DEFAULT 
SYNCHRONOUS 

Definition at line 180 of file communicator.h.

Constructor & Destructor Documentation

◆ Communicator() [1/4]

libMesh::Parallel::Communicator::Communicator ( )

Default Constructor.

Definition at line 57 of file communicator.C.

57  :
58 #ifdef LIBMESH_HAVE_MPI
59  _communicator(MPI_COMM_SELF),
60 #endif
61  _rank(0),
62  _size(1),
65  _I_duped_it(false) {}
std::map< int, unsigned int > used_tag_values
Definition: communicator.h:196

◆ Communicator() [2/4]

libMesh::Parallel::Communicator::Communicator ( const communicator comm)
explicit

Definition at line 68 of file communicator.C.

References assign().

68  :
69 #ifdef LIBMESH_HAVE_MPI
70  _communicator(MPI_COMM_SELF),
71 #endif
72  _rank(0),
73  _size(1),
76  _I_duped_it(false)
77 {
78  this->assign(comm);
79 }
void assign(const communicator &comm)
Definition: communicator.C:155
std::map< int, unsigned int > used_tag_values
Definition: communicator.h:196

◆ Communicator() [3/4]

libMesh::Parallel::Communicator::Communicator ( const Communicator )
delete

◆ Communicator() [4/4]

libMesh::Parallel::Communicator::Communicator ( Communicator &&  )
default

◆ ~Communicator()

libMesh::Parallel::Communicator::~Communicator ( )

Definition at line 82 of file communicator.C.

References clear().

83 {
84  this->clear();
85 }

Member Function Documentation

◆ allgather() [1/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::allgather ( const T &  send,
std::vector< T, A > &  recv 
) const
inline

Take a vector of length this->size(), and fill in recv[processor_id] = the value of send on that processor

Definition at line 2746 of file parallel_implementation.h.

References size().

Referenced by allgather(), libMesh::LaplaceMeshSmoother::allgather_graph(), allgather_packed_range(), libMesh::Nemesis_IO_Helper::compute_num_global_elem_blocks(), libMesh::DofMap::distribute_dofs(), libMesh::MeshRefinement::flag_elements_by_elem_fraction(), libMesh::MeshRefinement::flag_elements_by_nelem_target(), gather(), libMesh::Nemesis_IO::read(), and libMesh::DistributedMesh::renumber_dof_objects().

2748 {
2749  LOG_SCOPE ("allgather()","Parallel");
2750 
2751  libmesh_assert(this->size());
2752  recv.resize(this->size());
2753 
2754  unsigned int comm_size = this->size();
2755  if (comm_size > 1)
2756  {
2757  StandardType<T> send_type(&sendval);
2758 
2759  libmesh_call_mpi
2760  (MPI_Allgather (const_cast<T*>(&sendval), 1, send_type, recv.data(), 1,
2761  send_type, this->get()));
2762  }
2763  else if (comm_size > 0)
2764  recv[0] = sendval;
2765 }
processor_id_type size() const
Definition: communicator.h:175

◆ allgather() [2/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::allgather ( const std::basic_string< T > &  send,
std::vector< std::basic_string< T >, A > &  recv,
const bool  identical_buffer_sizes = false 
) const
inline

The allgather overload for string types has an optional identical_buffer_sizes optimization for when all strings are the same length.

Definition at line 1374 of file parallel_implementation.h.

References allgather(), and size().

1377 {
1378  LOG_SCOPE ("allgather()","Parallel");
1379 
1380  libmesh_assert(this->size());
1381  recv.assign(this->size(), "");
1382 
1383  // serial case
1384  if (this->size() < 2)
1385  {
1386  recv.resize(1);
1387  recv[0] = sendval;
1388  return;
1389  }
1390 
1391  std::vector<int>
1392  sendlengths (this->size(), 0),
1393  displacements(this->size(), 0);
1394 
1395  const int mysize = static_cast<int>(sendval.size());
1396 
1397  if (identical_buffer_sizes)
1398  sendlengths.assign(this->size(), mysize);
1399  else
1400  // first comm step to determine buffer sizes from all processors
1401  this->allgather(mysize, sendlengths);
1402 
1403  // Find the total size of the final array and
1404  // set up the displacement offsets for each processor
1405  unsigned int globalsize = 0;
1406  for (unsigned int i=0; i != this->size(); ++i)
1407  {
1408  displacements[i] = globalsize;
1409  globalsize += sendlengths[i];
1410  }
1411 
1412  // Check for quick return
1413  if (globalsize == 0)
1414  return;
1415 
1416  // monolithic receive buffer
1417  std::string r(globalsize, 0);
1418 
1419  // and get the data from the remote processors.
1420  libmesh_call_mpi
1421  (MPI_Allgatherv (const_cast<T*>(mysize ? sendval.data() : nullptr),
1422  mysize, StandardType<T>(),
1423  &r[0], sendlengths.data(), displacements.data(),
1424  StandardType<T>(), this->get()));
1425 
1426  // slice receive buffer up
1427  for (unsigned int i=0; i != this->size(); ++i)
1428  recv[i] = r.substr(displacements[i], sendlengths[i]);
1429 }
processor_id_type size() const
Definition: communicator.h:175
void allgather(const T &send, std::vector< T, A > &recv) const

◆ allgather() [3/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::allgather ( std::vector< T, A > &  r,
const bool  identical_buffer_sizes = false 
) const
inline

Take a vector of local variables and expand it to include values from all processors. By default, each processor is allowed to have its own unique input buffer length. If it is known that all processors have the same input sizes additional communication can be avoided.

Specifically, this function transforms this:

* Processor 0: [ ... N_0 ]
* Processor 1: [ ....... N_1 ]
* ...
* Processor M: [ .. N_M]
* 

into this:

* [ [ ... N_0 ] [ ....... N_1 ] ... [ .. N_M] ]
* 

on each processor. This function is collective and therefore must be called by all processors in the Communicator.

Definition at line 2770 of file parallel_implementation.h.

References allgather(), size(), and verify().

2772 {
2773  if (this->size() < 2)
2774  return;
2775 
2776  LOG_SCOPE("allgather()", "Parallel");
2777 
2778  if (identical_buffer_sizes)
2779  {
2780  if (r.empty())
2781  return;
2782 
2783  libmesh_assert(this->verify(r.size()));
2784 
2785  std::vector<T,A> r_src(r.size()*this->size());
2786  r_src.swap(r);
2787  StandardType<T> send_type(r_src.data());
2788 
2789  libmesh_call_mpi
2790  (MPI_Allgather (r_src.data(), cast_int<int>(r_src.size()),
2791  send_type, r.data(), cast_int<int>(r_src.size()),
2792  send_type, this->get()));
2793  // libmesh_assert(this->verify(r));
2794  return;
2795  }
2796 
2797  std::vector<int>
2798  sendlengths (this->size(), 0),
2799  displacements(this->size(), 0);
2800 
2801  const int mysize = static_cast<int>(r.size());
2802  this->allgather(mysize, sendlengths);
2803 
2804  // Find the total size of the final array and
2805  // set up the displacement offsets for each processor.
2806  unsigned int globalsize = 0;
2807  for (unsigned int i=0; i != this->size(); ++i)
2808  {
2809  displacements[i] = globalsize;
2810  globalsize += sendlengths[i];
2811  }
2812 
2813  // Check for quick return
2814  if (globalsize == 0)
2815  return;
2816 
2817  // copy the input buffer
2818  std::vector<T,A> r_src(globalsize);
2819  r_src.swap(r);
2820 
2821  StandardType<T> send_type(r.data());
2822 
2823  // and get the data from the remote processors.
2824  // Pass nullptr if our vector is empty.
2825  libmesh_call_mpi
2826  (MPI_Allgatherv (r_src.empty() ? nullptr : r_src.data(), mysize,
2827  send_type, r.data(), sendlengths.data(),
2828  displacements.data(), send_type, this->get()));
2829 }
processor_id_type size() const
Definition: communicator.h:175
void allgather(const T &send, std::vector< T, A > &recv) const

◆ allgather() [4/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::allgather ( std::vector< std::basic_string< T >, A > &  r,
const bool  identical_buffer_sizes = false 
) const
inline

AllGather overload for vectors of string types

Definition at line 2834 of file parallel_implementation.h.

References allgather(), end, size(), and verify().

2836 {
2837  if (this->size() < 2)
2838  return;
2839 
2840  LOG_SCOPE("allgather()", "Parallel");
2841 
2842  if (identical_buffer_sizes)
2843  {
2844  libmesh_assert(this->verify(r.size()));
2845 
2846  // identical_buffer_sizes doesn't buy us much since we have to
2847  // communicate the lengths of strings within each buffer anyway
2848  if (r.empty())
2849  return;
2850  }
2851 
2852  // Concatenate the input buffer into a send buffer, and keep track
2853  // of input string lengths
2854  std::vector<int> mystrlengths (r.size());
2855  std::vector<T> concat_src;
2856 
2857  int myconcatsize = 0;
2858  for (unsigned int i=0; i != r.size(); ++i)
2859  {
2860  int stringlen = cast_int<int>(r[i].size());
2861  mystrlengths[i] = stringlen;
2862  myconcatsize += stringlen;
2863  }
2864  concat_src.reserve(myconcatsize);
2865  for (unsigned int i=0; i != r.size(); ++i)
2866  concat_src.insert
2867  (concat_src.end(), r[i].begin(), r[i].end());
2868 
2869  // Get the string lengths from all other processors
2870  std::vector<int> strlengths = mystrlengths;
2871  this->allgather(strlengths, identical_buffer_sizes);
2872 
2873  // We now know how many strings we'll be receiving
2874  r.resize(strlengths.size());
2875 
2876  // Get the concatenated data sizes from all other processors
2877  std::vector<int> concat_sizes;
2878  this->allgather(myconcatsize, concat_sizes);
2879 
2880  // Find the total size of the final concatenated array and
2881  // set up the displacement offsets for each processor.
2882  std::vector<int> displacements(this->size(), 0);
2883  unsigned int globalsize = 0;
2884  for (unsigned int i=0; i != this->size(); ++i)
2885  {
2886  displacements[i] = globalsize;
2887  globalsize += concat_sizes[i];
2888  }
2889 
2890  // Check for quick return
2891  if (globalsize == 0)
2892  return;
2893 
2894  // Get the concatenated data from the remote processors.
2895  // Pass nullptr if our vector is empty.
2896  std::vector<T> concat(globalsize);
2897 
2898  // We may have concat_src.empty(), but we know concat has at least
2899  // one element we can use as an example for StandardType
2900  StandardType<T> send_type(concat.data());
2901 
2902  libmesh_call_mpi
2903  (MPI_Allgatherv (concat_src.empty() ?
2904  nullptr : concat_src.data(), myconcatsize,
2905  send_type, concat.data(), concat_sizes.data(),
2906  displacements.data(), send_type, this->get()));
2907 
2908  // Finally, split concatenated data into strings
2909  const T * begin = concat.data();
2910  for (unsigned int i=0; i != r.size(); ++i)
2911  {
2912  const T * end = begin + strlengths[i];
2913  r[i].assign(begin, end);
2914  begin = end;
2915  }
2916 }
processor_id_type size() const
Definition: communicator.h:175
void allgather(const T &send, std::vector< T, A > &recv) const
IterBase * end

◆ allgather_packed_range()

template<typename Context , typename Iter , typename OutputIter >
void libMesh::Parallel::Communicator::allgather_packed_range ( Context *  context,
Iter  range_begin,
const Iter  range_end,
OutputIter  out 
) const
inline

Take a range of local variables, combine it with ranges from all processors, and write the output to the output iterator.

Definition at line 3176 of file parallel_implementation.h.

References allgather(), max(), libMesh::Parallel::pack_range(), and libMesh::Parallel::unpack_range().

3180 {
3181  typedef typename std::iterator_traits<Iter>::value_type T;
3182  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
3183 
3184  bool nonempty_range = (range_begin != range_end);
3185  this->max(nonempty_range);
3186 
3187  while (nonempty_range)
3188  {
3189  // We will serialize variable size objects from *range_begin to
3190  // *range_end as a sequence of ints in this buffer
3191  std::vector<buffer_t> buffer;
3192 
3193  range_begin = Parallel::pack_range
3194  (context, range_begin, range_end, buffer);
3195 
3196  this->allgather(buffer, false);
3197 
3198  libmesh_assert(buffer.size());
3199 
3201  (buffer, context, out_iter, (T*)nullptr);
3202 
3203  nonempty_range = (range_begin != range_end);
3204  this->max(nonempty_range);
3205  }
3206 }
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Definition: packing.h:139
void allgather(const T &send, std::vector< T, A > &recv) const
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)

◆ alltoall()

template<typename T , typename A >
void libMesh::Parallel::Communicator::alltoall ( std::vector< T, A > &  r) const
inline

Effectively transposes the input vector across all processors. The jth entry on processor i is replaced with the ith entry from processor j.

Definition at line 3092 of file parallel_implementation.h.

References libMesh::libmesh_ignore(), size(), and verify().

Referenced by libMesh::SparsityPattern::Build::parallel_sync(), libMesh::Parallel::push_parallel_vector_data(), and libMesh::Nemesis_IO::read().

3093 {
3094  if (this->size() < 2 || buf.empty())
3095  return;
3096 
3097  LOG_SCOPE("alltoall()", "Parallel");
3098 
3099  // the per-processor size. this is the same for all
3100  // processors using MPI_Alltoall, could be variable
3101  // using MPI_Alltoallv
3102  const int size_per_proc =
3103  cast_int<int>(buf.size()/this->size());
3104  libmesh_ignore(size_per_proc);
3105 
3106  libmesh_assert_equal_to (buf.size()%this->size(), 0);
3107 
3108  libmesh_assert(this->verify(size_per_proc));
3109 
3110  StandardType<T> send_type(buf.data());
3111 
3112  libmesh_call_mpi
3113  (MPI_Alltoall (MPI_IN_PLACE, size_per_proc, send_type, buf.data(),
3114  size_per_proc, send_type, this->get()));
3115 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)

◆ assign()

void libMesh::Parallel::Communicator::assign ( const communicator comm)
private

Utility function for setting our member variables from an MPI communicator

Definition at line 155 of file communicator.C.

References _communicator, _rank, _send_mode, _size, and DEFAULT.

Referenced by Communicator(), duplicate(), operator=(), and split().

156 {
157  _communicator = comm;
158 #ifdef LIBMESH_HAVE_MPI
159  if (_communicator != MPI_COMM_NULL)
160  {
161  int i;
162  libmesh_call_mpi
163  (MPI_Comm_size(_communicator, &i));
164 
165  libmesh_assert_greater_equal (i, 0);
166  _size = cast_int<processor_id_type>(i);
167 
168  libmesh_call_mpi
169  (MPI_Comm_rank(_communicator, &i));
170 
171  libmesh_assert_greater_equal (i, 0);
172  _rank = cast_int<processor_id_type>(i);
173  }
174  else
175  {
176  _rank = 0;
177  _size = 1;
178  }
179 #endif
181 }

◆ barrier()

void libMesh::Parallel::Communicator::barrier ( ) const

Pause execution until all processors reach a certain point.

Definition at line 188 of file communicator.C.

References size().

Referenced by libMesh::NameBasedIO::write(), and libMesh::XdrIO::write().

189 {
190  if (this->size() > 1)
191  {
192  LOG_SCOPE("barrier()", "Parallel");
193  libmesh_call_mpi(MPI_Barrier (this->get()));
194  }
195 }
processor_id_type size() const
Definition: communicator.h:175

◆ broadcast() [1/4]

◆ broadcast() [2/4]

template<>
void libMesh::Parallel::Communicator::broadcast ( bool &  data,
const unsigned int  root_id 
) const
inline

Definition at line 1434 of file parallel_implementation.h.

References data, rank(), and size().

1435 {
1436  if (this->size() == 1)
1437  {
1438  libmesh_assert (!this->rank());
1439  libmesh_assert (!root_id);
1440  return;
1441  }
1442 
1443  libmesh_assert_less (root_id, this->size());
1444 
1445  LOG_SCOPE("broadcast()", "Parallel");
1446 
1447  // We don't want to depend on MPI-2 or C++ MPI, so we don't have
1448  // MPI::BOOL available
1449  char char_data = data;
1450 
1451  // Spread data to remote processors.
1452  libmesh_call_mpi
1453  (MPI_Bcast (&char_data, 1, StandardType<char>(&char_data),
1454  root_id, this->get()));
1455 
1456  data = char_data;
1457 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173
IterBase * data

◆ broadcast() [3/4]

template<typename T >
void libMesh::Parallel::Communicator::broadcast ( std::basic_string< T > &  data,
const unsigned int  root_id 
) const
inline

Definition at line 1461 of file parallel_implementation.h.

References broadcast(), data, rank(), and size().

1463 {
1464  if (this->size() == 1)
1465  {
1466  libmesh_assert (!this->rank());
1467  libmesh_assert (!root_id);
1468  return;
1469  }
1470 
1471  libmesh_assert_less (root_id, this->size());
1472 
1473  LOG_SCOPE("broadcast()", "Parallel");
1474 
1475  std::size_t data_size = data.size();
1476  this->broadcast(data_size, root_id);
1477 
1478  std::vector<T> data_c(data_size);
1479 #ifndef NDEBUG
1480  std::string orig(data);
1481 #endif
1482 
1483  if (this->rank() == root_id)
1484  for (std::size_t i=0; i<data.size(); i++)
1485  data_c[i] = data[i];
1486 
1487  this->broadcast (data_c, root_id);
1488 
1489  data.assign(data_c.begin(), data_c.end());
1490 
1491 #ifndef NDEBUG
1492  if (this->rank() == root_id)
1493  libmesh_assert_equal_to (data, orig);
1494 #endif
1495 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173
IterBase * data
void broadcast(T &data, const unsigned int root_id=0) const

◆ broadcast() [4/4]

template<typename T >
void libMesh::Parallel::Communicator::broadcast ( T &  libmesh_mpi_vardata,
const unsigned int  root_id 
) const
inline

Definition at line 3120 of file parallel_implementation.h.

References data, libMesh::libmesh_ignore(), rank(), and size().

3121 {
3122  libmesh_ignore(root_id); // Only needed for MPI and/or dbg/devel
3123  if (this->size() == 1)
3124  {
3125  libmesh_assert (!this->rank());
3126  libmesh_assert (!root_id);
3127  return;
3128  }
3129 
3130  libmesh_assert_less (root_id, this->size());
3131 
3132  LOG_SCOPE("broadcast()", "Parallel");
3133 
3134  // Spread data to remote processors.
3135  libmesh_call_mpi
3136  (MPI_Bcast (&data, 1, StandardType<T>(&data), root_id,
3137  this->get()));
3138 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173
IterBase * data

◆ broadcast_packed_range()

template<typename Context , typename OutputContext , typename Iter , typename OutputIter >
void libMesh::Parallel::Communicator::broadcast_packed_range ( const Context *  context1,
Iter  range_begin,
const Iter  range_end,
OutputContext *  context2,
OutputIter  out,
const unsigned int  root_id = 0 
) const
inline

Blocking-broadcast range-of-pointers to one processor. This function does not send the raw pointers, but rather constructs new objects at the other end whose contents match the objects pointed to by the sender.

void Parallel::pack(const T *, vector<int> & data, const Context *) is used to serialize type T onto the end of a data vector.

unsigned int Parallel::packable_size(const T *, const Context *) is used to allow data vectors to reserve memory, and for additional error checking

unsigned int Parallel::packed_size(const T *, vector<int>::const_iterator) is used to advance to the beginning of the next object's data.

Definition at line 1672 of file parallel_implementation.h.

References broadcast(), libMesh::Parallel::pack_range(), rank(), and libMesh::Parallel::unpack_range().

1678 {
1679  typedef typename std::iterator_traits<Iter>::value_type T;
1680  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
1681 
1682  do
1683  {
1684  // We will serialize variable size objects from *range_begin to
1685  // *range_end as a sequence of ints in this buffer
1686  std::vector<buffer_t> buffer;
1687 
1688  if (this->rank() == root_id)
1689  range_begin = Parallel::pack_range
1690  (context1, range_begin, range_end, buffer);
1691 
1692  // this->broadcast(vector) requires the receiving vectors to
1693  // already be the appropriate size
1694  std::size_t buffer_size = buffer.size();
1695  this->broadcast (buffer_size, root_id);
1696 
1697  // We continue until there's nothing left to broadcast
1698  if (!buffer_size)
1699  break;
1700 
1701  buffer.resize(buffer_size);
1702 
1703  // Broadcast the packed data
1704  this->broadcast (buffer, root_id);
1705 
1706  if (this->rank() != root_id)
1708  (buffer, context2, out_iter, (T*)nullptr);
1709  } while (true); // break above when we reach buffer_size==0
1710 }
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Definition: packing.h:139
processor_id_type rank() const
Definition: communicator.h:173
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)
void broadcast(T &data, const unsigned int root_id=0) const

◆ clear()

void libMesh::Parallel::Communicator::clear ( )

Free and reset this communicator

Definition at line 132 of file communicator.C.

References _communicator, and _I_duped_it.

Referenced by operator=(), split(), and ~Communicator().

132  {
133 #ifdef LIBMESH_HAVE_MPI
134  if (_I_duped_it)
135  {
136  libmesh_assert (_communicator != MPI_COMM_NULL);
137  libmesh_call_mpi
138  (MPI_Comm_free(&_communicator));
139 
140  _communicator = MPI_COMM_NULL;
141  }
142  _I_duped_it = false;
143 #endif
144 }

◆ dereference_unique_tag()

void libMesh::Parallel::Communicator::dereference_unique_tag ( int  tagvalue) const

Dereference an already-acquired tag, and see if we can re-release it.

Definition at line 44 of file communicator.C.

References used_tag_values.

Referenced by libMesh::Parallel::MessageTag::operator=(), and libMesh::Parallel::MessageTag::~MessageTag().

45 {
46  // This had better be an already-acquired tag.
47  libmesh_assert(used_tag_values.count(tagvalue));
48 
49  used_tag_values[tagvalue]--;
50  // If we don't have any more outstanding references, we
51  // don't even need to keep this tag in our "used" set.
52  if (!used_tag_values[tagvalue])
53  used_tag_values.erase(tagvalue);
54 }
std::map< int, unsigned int > used_tag_values
Definition: communicator.h:196

◆ duplicate() [1/2]

void libMesh::Parallel::Communicator::duplicate ( const Communicator comm)

Definition at line 108 of file communicator.C.

References _communicator, and send_mode().

109 {
110  this->duplicate(comm._communicator);
111  this->send_mode(comm.send_mode());
112 }
void duplicate(const Communicator &comm)
Definition: communicator.C:108

◆ duplicate() [2/2]

void libMesh::Parallel::Communicator::duplicate ( const communicator comm)

Definition at line 116 of file communicator.C.

References _communicator, _I_duped_it, and assign().

117 {
118  if (_communicator != MPI_COMM_NULL)
119  {
120  libmesh_call_mpi
121  (MPI_Comm_dup(comm, &_communicator));
122 
123  _I_duped_it = true;
124  }
125  this->assign(_communicator);
126 }
void assign(const communicator &comm)
Definition: communicator.C:155

◆ gather() [1/3]

template<typename T , typename A >
void libMesh::Parallel::Communicator::gather ( const unsigned int  root_id,
const T &  send,
std::vector< T, A > &  recv 
) const
inline

Take a vector of length comm.size(), and on processor root_id fill in recv[processor_id] = the value of send on processor processor_id

Definition at line 2606 of file parallel_implementation.h.

References rank(), and size().

Referenced by gather(), gather_packed_range(), libMesh::XdrIO::write_serialized_bcs_helper(), libMesh::XdrIO::write_serialized_connectivity(), libMesh::XdrIO::write_serialized_nodes(), and libMesh::XdrIO::write_serialized_nodesets().

2609 {
2610  libmesh_assert_less (root_id, this->size());
2611 
2612  if (this->rank() == root_id)
2613  recv.resize(this->size());
2614 
2615  if (this->size() > 1)
2616  {
2617  LOG_SCOPE("gather()", "Parallel");
2618 
2619  StandardType<T> send_type(&sendval);
2620 
2621  libmesh_call_mpi
2622  (MPI_Gather(const_cast<T*>(&sendval), 1, send_type,
2623  recv.empty() ? nullptr : recv.data(), 1, send_type,
2624  root_id, this->get()));
2625  }
2626  else
2627  recv[0] = sendval;
2628 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173

◆ gather() [2/3]

template<typename T , typename A >
void libMesh::Parallel::Communicator::gather ( const unsigned int  root_id,
const std::basic_string< T > &  send,
std::vector< std::basic_string< T >, A > &  recv,
const bool  identical_buffer_sizes = false 
) const
inline

The gather overload for string types has an optional identical_buffer_sizes optimization for when all strings are the same length.

Definition at line 2686 of file parallel_implementation.h.

References gather(), rank(), and size().

2690 {
2691  libmesh_assert_less (root_id, this->size());
2692 
2693  if (this->rank() == root_id)
2694  recv.resize(this->size());
2695 
2696  if (this->size() > 1)
2697  {
2698  LOG_SCOPE ("gather()","Parallel");
2699 
2700  std::vector<int>
2701  sendlengths (this->size(), 0),
2702  displacements(this->size(), 0);
2703 
2704  const int mysize = static_cast<int>(sendval.size());
2705 
2706  if (identical_buffer_sizes)
2707  sendlengths.assign(this->size(), mysize);
2708  else
2709  // first comm step to determine buffer sizes from all processors
2710  this->gather(root_id, mysize, sendlengths);
2711 
2712  // Find the total size of the final array and
2713  // set up the displacement offsets for each processor
2714  unsigned int globalsize = 0;
2715  for (unsigned int i=0; i < this->size(); ++i)
2716  {
2717  displacements[i] = globalsize;
2718  globalsize += sendlengths[i];
2719  }
2720 
2721  // monolithic receive buffer
2722  std::string r;
2723  if (this->rank() == root_id)
2724  r.resize(globalsize, 0);
2725 
2726  // and get the data from the remote processors.
2727  libmesh_call_mpi
2728  (MPI_Gatherv (const_cast<T*>(sendval.data()),
2729  mysize, StandardType<T>(),
2730  this->rank() == root_id ? &r[0] : nullptr,
2731  sendlengths.data(), displacements.data(),
2732  StandardType<T>(), root_id, this->get()));
2733 
2734  // slice receive buffer up
2735  if (this->rank() == root_id)
2736  for (unsigned int i=0; i != this->size(); ++i)
2737  recv[i] = r.substr(displacements[i], sendlengths[i]);
2738  }
2739  else
2740  recv[0] = sendval;
2741 }
processor_id_type size() const
Definition: communicator.h:175
void gather(const unsigned int root_id, const T &send, std::vector< T, A > &recv) const
processor_id_type rank() const
Definition: communicator.h:173

◆ gather() [3/3]

template<typename T , typename A >
void libMesh::Parallel::Communicator::gather ( const unsigned int  root_id,
std::vector< T, A > &  r 
) const
inline

Take a vector of local variables and expand it on processor root_id to include values from all processors

This handles the case where the lengths of the vectors may vary. Specifically, this function transforms this:

* Processor 0: [ ... N_0 ]
* Processor 1: [ ....... N_1 ]
* ...
* Processor M: [ .. N_M]
* 

into this:

* [ [ ... N_0 ] [ ....... N_1 ] ... [ .. N_M] ]
* 

on processor root_id. This function is collective and therefore must be called by all processors in the Communicator.

Definition at line 2633 of file parallel_implementation.h.

References allgather(), rank(), and size().

2635 {
2636  if (this->size() == 1)
2637  {
2638  libmesh_assert (!this->rank());
2639  libmesh_assert (!root_id);
2640  return;
2641  }
2642 
2643  libmesh_assert_less (root_id, this->size());
2644 
2645  std::vector<int>
2646  sendlengths (this->size(), 0),
2647  displacements(this->size(), 0);
2648 
2649  const int mysize = static_cast<int>(r.size());
2650  this->allgather(mysize, sendlengths);
2651 
2652  LOG_SCOPE("gather()", "Parallel");
2653 
2654  // Find the total size of the final array and
2655  // set up the displacement offsets for each processor.
2656  unsigned int globalsize = 0;
2657  for (unsigned int i=0; i != this->size(); ++i)
2658  {
2659  displacements[i] = globalsize;
2660  globalsize += sendlengths[i];
2661  }
2662 
2663  // Check for quick return
2664  if (globalsize == 0)
2665  return;
2666 
2667  // copy the input buffer
2668  std::vector<T,A> r_src(r);
2669 
2670  // now resize it to hold the global data
2671  // on the receiving processor
2672  if (root_id == this->rank())
2673  r.resize(globalsize);
2674 
2675  // and get the data from the remote processors
2676  libmesh_call_mpi
2677  (MPI_Gatherv (r_src.empty() ? nullptr : r_src.data(), mysize,
2678  StandardType<T>(), r.empty() ? nullptr : r.data(),
2679  sendlengths.data(), displacements.data(),
2680  StandardType<T>(), root_id, this->get()));
2681 }
processor_id_type size() const
Definition: communicator.h:175
void allgather(const T &send, std::vector< T, A > &recv) const
processor_id_type rank() const
Definition: communicator.h:173

◆ gather_packed_range()

template<typename Context , typename Iter , typename OutputIter >
void libMesh::Parallel::Communicator::gather_packed_range ( const unsigned int  root_id,
Context *  context,
Iter  range_begin,
const Iter  range_end,
OutputIter  out 
) const
inline

Take a range of local variables, combine it with ranges from all processors, and write the output to the output iterator on rank root.

Definition at line 3143 of file parallel_implementation.h.

References gather(), max(), libMesh::Parallel::pack_range(), and libMesh::Parallel::unpack_range().

3148 {
3149  typedef typename std::iterator_traits<Iter>::value_type T;
3150  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
3151 
3152  bool nonempty_range = (range_begin != range_end);
3153  this->max(nonempty_range);
3154 
3155  while (nonempty_range)
3156  {
3157  // We will serialize variable size objects from *range_begin to
3158  // *range_end as a sequence of ints in this buffer
3159  std::vector<buffer_t> buffer;
3160 
3161  range_begin = Parallel::pack_range
3162  (context, range_begin, range_end, buffer);
3163 
3164  this->gather(root_id, buffer);
3165 
3167  (buffer, context, out_iter, (T*)(nullptr));
3168 
3169  nonempty_range = (range_begin != range_end);
3170  this->max(nonempty_range);
3171  }
3172 }
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Definition: packing.h:139
void gather(const unsigned int root_id, const T &send, std::vector< T, A > &recv) const
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)

◆ get() [1/2]

◆ get() [2/2]

const communicator& libMesh::Parallel::Communicator::get ( ) const
inline

Definition at line 143 of file communicator.h.

References _communicator.

143 { return _communicator; }

◆ get_unique_tag()

MessageTag libMesh::Parallel::Communicator::get_unique_tag ( int  tagvalue) const

Get a tag that is unique to this Communicator.

Note
If people are also using magic numbers or copying communicators around then we can't guarantee the tag is unique to this MPI_Comm.

Definition at line 201 of file communicator.C.

References used_tag_values.

Referenced by libMesh::BoundaryInfo::build_node_list_from_side_list(), libMesh::MeshRefinement::make_coarsening_compatible(), libMesh::SparsityPattern::Build::parallel_sync(), libMesh::Parallel::pull_parallel_vector_data(), libMesh::Parallel::push_parallel_vector_data(), libMesh::Nemesis_IO::read(), libMesh::System::read_SCALAR_dofs(), libMesh::System::read_serialized_blocked_dof_objects(), libMesh::MeshRefinement::uniformly_coarsen(), libMesh::System::write_serialized_blocked_dof_objects(), and libMesh::XdrIO::write_serialized_nodes().

202 {
203  if (used_tag_values.count(tagvalue))
204  {
205  // Get the largest value in the used values, and pick one
206  // larger
207  tagvalue = used_tag_values.rbegin()->first+1;
208  libmesh_assert(!used_tag_values.count(tagvalue));
209  }
210  used_tag_values[tagvalue] = 1;
211 
212  // #ifndef NDEBUG
213  // // Make sure everyone called get_unique_tag and make sure
214  // // everyone got the same value
215  // int maxval = tagvalue;
216  // this->max(maxval);
217  // libmesh_assert_equal_to (tagvalue, maxval);
218  // #endif
219 
220  return MessageTag(tagvalue, this);
221 }
std::map< int, unsigned int > used_tag_values
Definition: communicator.h:196

◆ max() [1/3]

template<typename T >
void libMesh::Parallel::Communicator::max ( T &  r) const

Take a local variable and replace it with the maximum of it's values on all processors. Containers are replaced element-wise.

Referenced by libMesh::MeshRefinement::_coarsen_elements(), libMesh::MeshRefinement::_refine_elements(), libMesh::UnstructuredMesh::all_second_order(), libMesh::MeshTools::Modification::all_tri(), allgather_packed_range(), libMesh::DofMap::attach_matrix(), libMesh::MeshTools::Generation::build_extrusion(), libMesh::System::calculate_norm(), libMesh::DofMap::check_dirichlet_bcid_consistency(), libMesh::MeshTools::create_bounding_box(), libMesh::MeshTools::create_nodal_bounding_box(), libMesh::MeshTools::create_processor_bounding_box(), libMesh::MeshTools::create_subdomain_bounding_box(), libMesh::MeshRefinement::eliminate_unrefined_patches(), libMesh::MeshRefinement::flag_elements_by_error_fraction(), libMesh::MeshRefinement::flag_elements_by_nelem_target(), gather_packed_range(), libMesh::DofMap::get_info(), libMesh::LocationMap< T >::init(), libMesh::MeshTools::libmesh_assert_consistent_distributed(), libMesh::MeshTools::libmesh_assert_consistent_distributed_nodes(), libMesh::MeshTools::libmesh_assert_parallel_consistent_new_node_procids(), libMesh::MeshTools::libmesh_assert_parallel_consistent_procids< Elem >(), libMesh::MeshTools::libmesh_assert_parallel_consistent_procids< Node >(), libMesh::MeshTools::libmesh_assert_topology_consistent_procids< Node >(), libMesh::MeshTools::libmesh_assert_valid_boundary_ids(), libMesh::MeshTools::libmesh_assert_valid_dof_ids(), libMesh::MeshTools::libmesh_assert_valid_neighbors(), libMesh::MeshTools::libmesh_assert_valid_refinement_flags(), libMesh::MeshTools::libmesh_assert_valid_unique_ids(), libMesh::MeshRefinement::limit_level_mismatch_at_edge(), libMesh::MeshRefinement::limit_level_mismatch_at_node(), libMesh::MeshRefinement::limit_overrefined_boundary(), libMesh::MeshRefinement::limit_underrefined_boundary(), libMesh::MeshTools::n_active_levels(), libMesh::MeshTools::n_levels(), libMesh::MeshTools::n_p_levels(), libMesh::DistributedMesh::parallel_max_elem_id(), libMesh::DistributedMesh::parallel_max_node_id(), libMesh::ReplicatedMesh::parallel_max_unique_id(), libMesh::DistributedMesh::parallel_max_unique_id(), libMesh::MeshTools::paranoid_n_levels(), libMesh::Nemesis_IO::read(), libMesh::MeshBase::recalculate_n_partitions(), libMesh::MeshRefinement::refine_and_coarsen_elements(), semiverify(), libMesh::Parallel::sync_dofobject_data_by_xyz(), libMesh::MeshRefinement::test_level_one(), libMesh::MeshRefinement::test_unflagged(), and verify().

◆ max() [2/3]

template<typename T >
void libMesh::Parallel::Communicator::max ( T &  libmesh_mpi_varr) const
inline

Definition at line 2280 of file parallel_implementation.h.

References size().

2281 {
2282  if (this->size() > 1)
2283  {
2284  LOG_SCOPE("max(scalar)", "Parallel");
2285 
2286  libmesh_call_mpi
2287  (MPI_Allreduce (MPI_IN_PLACE, &r, 1,
2288  StandardType<T>(&r),
2290  this->get()));
2291  }
2292 }
processor_id_type size() const
Definition: communicator.h:175
long double max(long double a, double b)

◆ max() [3/3]

template<typename A >
void libMesh::Parallel::Communicator::max ( std::vector< bool, A > &  r) const
inline

Definition at line 2332 of file parallel_implementation.h.

References size(), and verify().

2333 {
2334  if (this->size() > 1 && !r.empty())
2335  {
2336  LOG_SCOPE("max(vector<bool>)", "Parallel");
2337 
2338  libmesh_assert(this->verify(r.size()));
2339 
2340  std::vector<unsigned int> ruint;
2341  pack_vector_bool(r, ruint);
2342  std::vector<unsigned int> temp(ruint.size());
2343  libmesh_call_mpi
2344  (MPI_Allreduce (ruint.data(), temp.data(),
2345  cast_int<int>(ruint.size()),
2346  StandardType<unsigned int>(), MPI_BOR,
2347  this->get()));
2348  unpack_vector_bool(temp, r);
2349  }
2350 }
processor_id_type size() const
Definition: communicator.h:175

◆ maxloc() [1/3]

template<typename T >
void libMesh::Parallel::Communicator::maxloc ( T &  r,
unsigned int &  max_id 
) const
inline

Take a local variable and replace it with the maximum of it's values on all processors, returning the minimum rank of a processor which originally held the maximum value.

Definition at line 2354 of file parallel_implementation.h.

References libMesh::libmesh_ignore(), libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), and libMesh::Parallel::DataPlusInt< T >::val.

2356 {
2357  if (this->size() > 1)
2358  {
2359  LOG_SCOPE("maxloc(scalar)", "Parallel");
2360 
2361  DataPlusInt<T> data_in;
2362  libmesh_ignore(data_in); // unused ifndef LIBMESH_HAVE_MPI
2363  data_in.val = r;
2364  data_in.rank = this->rank();
2365  libmesh_call_mpi
2366  (MPI_Allreduce (MPI_IN_PLACE, &data_in, 1,
2367  dataplusint_type<T>(),
2368  OpFunction<T>::max_location(),
2369  this->get()));
2370  r = data_in.val;
2371  max_id = data_in.rank;
2372  }
2373  else
2374  max_id = this->rank();
2375 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173

◆ maxloc() [2/3]

template<typename T , typename A1 , typename A2 >
void libMesh::Parallel::Communicator::maxloc ( std::vector< T, A1 > &  r,
std::vector< unsigned int, A2 > &  max_id 
) const
inline

Take a vector of local variables and replace each entry with the maximum of it's values on all processors. Set each min_id entry to the minimum rank where a corresponding maximum was found.

Definition at line 2404 of file parallel_implementation.h.

References libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), libMesh::Parallel::DataPlusInt< T >::val, and verify().

2406 {
2407  if (this->size() > 1 && !r.empty())
2408  {
2409  LOG_SCOPE("maxloc(vector)", "Parallel");
2410 
2411  libmesh_assert(this->verify(r.size()));
2412 
2413  std::vector<DataPlusInt<T>> data_in(r.size());
2414  for (std::size_t i=0; i != r.size(); ++i)
2415  {
2416  data_in[i].val = r[i];
2417  data_in[i].rank = this->rank();
2418  }
2419  std::vector<DataPlusInt<T>> data_out(r.size());
2420  libmesh_call_mpi
2421  (MPI_Allreduce (data_in.data(), data_out.data(),
2422  cast_int<int>(r.size()),
2423  dataplusint_type<T>(),
2424  OpFunction<T>::max_location(),
2425  this->get()));
2426  for (std::size_t i=0; i != r.size(); ++i)
2427  {
2428  r[i] = data_out[i].val;
2429  max_id[i] = data_out[i].rank;
2430  }
2431  }
2432  else if (!r.empty())
2433  {
2434  for (std::size_t i=0; i != r.size(); ++i)
2435  max_id[i] = this->rank();
2436  }
2437 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173

◆ maxloc() [3/3]

template<typename A1 , typename A2 >
void libMesh::Parallel::Communicator::maxloc ( std::vector< bool, A1 > &  r,
std::vector< unsigned int, A2 > &  max_id 
) const
inline

Definition at line 2441 of file parallel_implementation.h.

References libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), libMesh::Parallel::DataPlusInt< T >::val, and verify().

2443 {
2444  if (this->size() > 1 && !r.empty())
2445  {
2446  LOG_SCOPE("maxloc(vector<bool>)", "Parallel");
2447 
2448  libmesh_assert(this->verify(r.size()));
2449 
2450  std::vector<DataPlusInt<int>> data_in(r.size());
2451  for (std::size_t i=0; i != r.size(); ++i)
2452  {
2453  data_in[i].val = r[i];
2454  data_in[i].rank = this->rank();
2455  }
2456  std::vector<DataPlusInt<int>> data_out(r.size());
2457  libmesh_call_mpi
2458  (MPI_Allreduce (data_in.data(), data_out.data(),
2459  cast_int<int>(r.size()),
2460  StandardType<int>(),
2461  OpFunction<int>::max_location(),
2462  this->get()));
2463  for (std::size_t i=0; i != r.size(); ++i)
2464  {
2465  r[i] = data_out[i].val;
2466  max_id[i] = data_out[i].rank;
2467  }
2468  }
2469  else if (!r.empty())
2470  {
2471  for (std::size_t i=0; i != r.size(); ++i)
2472  max_id[i] = this->rank();
2473  }
2474 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173

◆ min() [1/3]

◆ min() [2/3]

template<typename T >
void libMesh::Parallel::Communicator::min ( T &  libmesh_mpi_varr) const
inline

Definition at line 2087 of file parallel_implementation.h.

References size().

2088 {
2089  if (this->size() > 1)
2090  {
2091  LOG_SCOPE("min(scalar)", "Parallel");
2092 
2093  libmesh_call_mpi
2094  (MPI_Allreduce (MPI_IN_PLACE, &r, 1,
2095  StandardType<T>(&r), OpFunction<T>::min(),
2096  this->get()));
2097  }
2098 }
processor_id_type size() const
Definition: communicator.h:175
long double min(long double a, double b)

◆ min() [3/3]

template<typename A >
void libMesh::Parallel::Communicator::min ( std::vector< bool, A > &  r) const
inline

Definition at line 2139 of file parallel_implementation.h.

References size(), and verify().

2140 {
2141  if (this->size() > 1 && !r.empty())
2142  {
2143  LOG_SCOPE("min(vector<bool>)", "Parallel");
2144 
2145  libmesh_assert(this->verify(r.size()));
2146 
2147  std::vector<unsigned int> ruint;
2148  pack_vector_bool(r, ruint);
2149  std::vector<unsigned int> temp(ruint.size());
2150  libmesh_call_mpi
2151  (MPI_Allreduce (ruint.data(), temp.data(),
2152  cast_int<int>(ruint.size()),
2153  StandardType<unsigned int>(), MPI_BAND,
2154  this->get()));
2155  unpack_vector_bool(temp, r);
2156  }
2157 }
processor_id_type size() const
Definition: communicator.h:175

◆ minloc() [1/3]

template<typename T >
void libMesh::Parallel::Communicator::minloc ( T &  r,
unsigned int &  min_id 
) const
inline

Take a local variable and replace it with the minimum of it's values on all processors, returning the minimum rank of a processor which originally held the minimum value.

Definition at line 2161 of file parallel_implementation.h.

References libMesh::libmesh_ignore(), libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), and libMesh::Parallel::DataPlusInt< T >::val.

2163 {
2164  if (this->size() > 1)
2165  {
2166  LOG_SCOPE("minloc(scalar)", "Parallel");
2167 
2168  DataPlusInt<T> data_in;
2169  libmesh_ignore(data_in); // unused ifndef LIBMESH_HAVE_MPI
2170  data_in.val = r;
2171  data_in.rank = this->rank();
2172  libmesh_call_mpi
2173  (MPI_Allreduce (MPI_IN_PLACE, &data_in, 1, dataplusint_type<T>(),
2174  OpFunction<T>::min_location(), this->get()));
2175  r = data_in.val;
2176  min_id = data_in.rank;
2177  }
2178  else
2179  min_id = this->rank();
2180 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173

◆ minloc() [2/3]

template<typename T , typename A1 , typename A2 >
void libMesh::Parallel::Communicator::minloc ( std::vector< T, A1 > &  r,
std::vector< unsigned int, A2 > &  min_id 
) const
inline

Take a vector of local variables and replace each entry with the minimum of it's values on all processors. Set each min_id entry to the minimum rank where a corresponding minimum was found.

Definition at line 2208 of file parallel_implementation.h.

References libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), libMesh::Parallel::DataPlusInt< T >::val, and verify().

2210 {
2211  if (this->size() > 1 && !r.empty())
2212  {
2213  LOG_SCOPE("minloc(vector)", "Parallel");
2214 
2215  libmesh_assert(this->verify(r.size()));
2216 
2217  std::vector<DataPlusInt<T>> data_in(r.size());
2218  for (std::size_t i=0; i != r.size(); ++i)
2219  {
2220  data_in[i].val = r[i];
2221  data_in[i].rank = this->rank();
2222  }
2223  std::vector<DataPlusInt<T>> data_out(r.size());
2224  libmesh_call_mpi
2225  (MPI_Allreduce (data_in.data(), data_out.data(),
2226  cast_int<int>(r.size()),
2227  dataplusint_type<T>(),
2228  OpFunction<T>::min_location(), this->get()));
2229  for (std::size_t i=0; i != r.size(); ++i)
2230  {
2231  r[i] = data_out[i].val;
2232  min_id[i] = data_out[i].rank;
2233  }
2234  }
2235  else if (!r.empty())
2236  {
2237  for (std::size_t i=0; i != r.size(); ++i)
2238  min_id[i] = this->rank();
2239  }
2240 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173

◆ minloc() [3/3]

template<typename A1 , typename A2 >
void libMesh::Parallel::Communicator::minloc ( std::vector< bool, A1 > &  r,
std::vector< unsigned int, A2 > &  min_id 
) const
inline

Definition at line 2244 of file parallel_implementation.h.

References libMesh::Parallel::DataPlusInt< T >::rank, rank(), size(), libMesh::Parallel::DataPlusInt< T >::val, and verify().

2246 {
2247  if (this->size() > 1 && !r.empty())
2248  {
2249  LOG_SCOPE("minloc(vector<bool>)", "Parallel");
2250 
2251  libmesh_assert(this->verify(r.size()));
2252 
2253  std::vector<DataPlusInt<int>> data_in(r.size());
2254  for (std::size_t i=0; i != r.size(); ++i)
2255  {
2256  data_in[i].val = r[i];
2257  data_in[i].rank = this->rank();
2258  }
2259  std::vector<DataPlusInt<int>> data_out(r.size());
2260  libmesh_call_mpi
2261  (MPI_Allreduce (data_in.data(), data_out.data(),
2262  cast_int<int>(r.size()),
2263  StandardType<int>(),
2264  OpFunction<int>::min_location(), this->get()));
2265  for (std::size_t i=0; i != r.size(); ++i)
2266  {
2267  r[i] = data_out[i].val;
2268  min_id[i] = data_out[i].rank;
2269  }
2270  }
2271  else if (!r.empty())
2272  {
2273  for (std::size_t i=0; i != r.size(); ++i)
2274  min_id[i] = this->rank();
2275  }
2276 }
processor_id_type size() const
Definition: communicator.h:175
processor_id_type rank() const
Definition: communicator.h:173

◆ nonblocking_receive_packed_range() [1/2]

template<typename Context , typename OutputIter , typename T >
void libMesh::Parallel::Communicator::nonblocking_receive_packed_range ( const unsigned int  src_processor_id,
Context *  context,
OutputIter  out,
const T *  output_type,
Request req,
Status stat,
const MessageTag tag = any_tag 
) const
inline

Non-Blocking-receive range-of-pointers from one processor.

This is meant to receive messages from nonblocking_send_packed_range

Similar in design to the above receive_packed_range. However, this version requires a Request and a Status.

The Status must be a positively tested Status for a message of this type (i.e. a message does exist). It should most likely be generated by Communicator::packed_range_probe.

Definition at line 1117 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::out, receive(), and libMesh::Parallel::Status::size().

1124 {
1125  libmesh_experimental();
1126 
1127  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
1128 
1129  // Receive serialized variable size objects as a sequence of
1130  // buffer_t.
1131  // Allocate a buffer on the heap so we don't have to free it until
1132  // after the Request::wait()
1133  std::vector<buffer_t> * buffer = new std::vector<buffer_t>(stat.size());
1134  this->receive(src_processor_id, *buffer, req, tag);
1135 
1136  // Make the Request::wait() handle unpacking the buffer
1137  req.add_post_wait_work
1138  (new Parallel::PostWaitUnpackBuffer<std::vector<buffer_t>, Context, OutputIter, T>(*buffer, context, out));
1139 
1140  // Make the Request::wait() then handle deleting the buffer
1141  req.add_post_wait_work
1142  (new Parallel::PostWaitDeleteBuffer<std::vector<buffer_t>>(buffer));
1143 }
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
OStreamProxy out(std::cout)

◆ nonblocking_receive_packed_range() [2/2]

template<typename Context , typename OutputIter , typename T >
void libMesh::Parallel::Communicator::nonblocking_receive_packed_range ( const unsigned int  src_processor_id,
Context *  context,
OutputIter  out,
const T *  output_type,
Request req,
Status stat,
std::shared_ptr< std::vector< typename Parallel::Packing< T >::buffer_type >> &  buffer,
const MessageTag tag = any_tag 
) const
inline

Non-Blocking-receive range-of-pointers from one processor.

This is meant to receive messages from nonblocking_send_packed_range

Similar in design to the above receive_packed_range. However, this version requires a Request and a Status.

The Status must be a positively tested Status for a message of this type (i.e. a message does exist). It should most likely be generated by Communicator::packed_range_probe.

Definition at line 1715 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::out, receive(), and libMesh::Parallel::Status::size().

1723 {
1724  libmesh_experimental();
1725 
1726  // If they didn't pass in a buffer - let's make one
1727  if (buffer == nullptr)
1728  buffer = std::make_shared<std::vector<typename Parallel::Packing<T>::buffer_type>>();
1729  else
1730  buffer->clear();
1731 
1732  // Receive serialized variable size objects as a sequence of
1733  // buffer_t.
1734  // Allocate a buffer on the heap so we don't have to free it until
1735  // after the Request::wait()
1736  buffer->resize(stat.size());
1737  this->receive(src_processor_id, *buffer, req, tag);
1738 
1739  // Make the Request::wait() handle unpacking the buffer
1740  req.add_post_wait_work
1741  (new Parallel::PostWaitUnpackBuffer<std::vector<typename Parallel::Packing<T>::buffer_type>, Context, OutputIter, T>(*buffer, context, out));
1742 
1743  // Make it dereference the shared pointer (possibly freeing the buffer)
1744  req.add_post_wait_work
1745  (new Parallel::PostWaitDereferenceSharedPtr<std::vector<typename Parallel::Packing<T>::buffer_type>>(buffer));
1746 }
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
OStreamProxy out(std::cout)

◆ nonblocking_send_packed_range() [1/2]

template<typename Context , typename Iter >
void libMesh::Parallel::Communicator::nonblocking_send_packed_range ( const unsigned int  dest_processor_id,
const Context *  context,
Iter  range_begin,
const Iter  range_end,
Request req,
const MessageTag tag = no_tag 
) const
inline

Similar to the above Nonblocking send_packed_range with a few important differences:

  1. The total size of the packed buffer MUST be less than std::numeric_limits<int>::max()
  2. Only one message is generated
  3. On the receiving end the message should be tested for using Communicator::packed_range_probe()
  4. The message must be received by Communicator::nonblocking_receive_packed_range()

Definition at line 686 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), std::max(), libMesh::Parallel::pack_range(), and send().

692 {
693  libmesh_experimental();
694 
695  // Allocate a buffer on the heap so we don't have to free it until
696  // after the Request::wait()
697  typedef typename std::iterator_traits<Iter>::value_type T;
698  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
699 
700  if (range_begin != range_end)
701  {
702  std::vector<buffer_t> * buffer = new std::vector<buffer_t>();
703 
704  range_begin =
705  Parallel::pack_range(context,
706  range_begin,
707  range_end,
708  *buffer,
709  // MPI-2 can only use integers for size
711 
712  if (range_begin != range_end)
713  libmesh_error_msg("Non-blocking packed range sends cannot exceed " << std::numeric_limits<int>::max() << "in size");
714 
715  // Make the Request::wait() handle deleting the buffer
716  req.add_post_wait_work
717  (new Parallel::PostWaitDeleteBuffer<std::vector<buffer_t>>
718  (buffer));
719 
720  // Non-blocking send of the buffer
721  this->send(dest_processor_id, *buffer, req, tag);
722  }
723 }
void send(const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Definition: packing.h:139
long double max(long double a, double b)

◆ nonblocking_send_packed_range() [2/2]

template<typename Context , typename Iter >
void libMesh::Parallel::Communicator::nonblocking_send_packed_range ( const unsigned int  dest_processor_id,
const Context *  context,
Iter  range_begin,
const Iter  range_end,
Request req,
std::shared_ptr< std::vector< typename Parallel::Packing< typename std::iterator_traits< Iter >::value_type >::buffer_type >> &  buffer,
const MessageTag tag = no_tag 
) const
inline

Similar to the above Nonblocking send_packed_range with a few important differences:

  1. The total size of the packed buffer MUST be less than std::numeric_limits<int>::max()
  2. Only one message is generated
  3. On the receiving end the message should be tested for using Communicator::packed_range_probe()
  4. The message must be received by Communicator::nonblocking_receive_packed_range()

Definition at line 1329 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), std::max(), libMesh::Parallel::pack_range(), and send().

1336 {
1337  libmesh_experimental();
1338 
1339  // Allocate a buffer on the heap so we don't have to free it until
1340  // after the Request::wait()
1341  typedef typename std::iterator_traits<Iter>::value_type T;
1342  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
1343 
1344  if (range_begin != range_end)
1345  {
1346  if (buffer == nullptr)
1347  buffer = std::make_shared<std::vector<buffer_t>>();
1348  else
1349  buffer->clear();
1350 
1351  range_begin =
1352  Parallel::pack_range(context,
1353  range_begin,
1354  range_end,
1355  *buffer,
1356  // MPI-2 can only use integers for size
1358 
1359  if (range_begin != range_end)
1360  libmesh_error_msg("Non-blocking packed range sends cannot exceed " << std::numeric_limits<int>::max() << "in size");
1361 
1362  // Make it dereference the shared pointer (possibly freeing the buffer)
1363  req.add_post_wait_work
1364  (new Parallel::PostWaitDereferenceSharedPtr<std::vector<buffer_t>>(buffer));
1365 
1366  // Non-blocking send of the buffer
1367  this->send(dest_processor_id, *buffer, req, tag);
1368  }
1369 }
void send(const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Definition: packing.h:139
long double max(long double a, double b)

◆ operator=() [1/3]

Communicator& libMesh::Parallel::Communicator::operator= ( const Communicator )
delete

◆ operator=() [2/3]

Communicator& libMesh::Parallel::Communicator::operator= ( Communicator &&  )
default

◆ operator=() [3/3]

Communicator & libMesh::Parallel::Communicator::operator= ( const communicator comm)

Definition at line 147 of file communicator.C.

References assign(), and clear().

148 {
149  this->clear();
150  this->assign(comm);
151  return *this;
152 }
void assign(const communicator &comm)
Definition: communicator.C:155

◆ packed_range_probe()

template<typename T >
Status libMesh::Parallel::Communicator::packed_range_probe ( const unsigned int  src_processor_id,
const MessageTag tag,
bool &  flag 
) const
inline

Non-Blocking message probe for a packed range message. Allows information about a message to be examined before the message is actually received.

Template type must match the object type that will be in the packed range

Parameters
src_processor_idThe processor the message is expected from or Parallel::any_source
tagThe message tag or Parallel::any_tag
flagOutput. True if a message exists. False otherwise.

Definition at line 188 of file parallel_implementation.h.

References libMesh::Parallel::Status::get(), and libMesh::Parallel::MessageTag::value().

191 {
192  LOG_SCOPE("packed_range_probe()", "Parallel");
193 
194  libmesh_experimental();
195 
196  Status stat((StandardType<typename Packing<T>::buffer_type>()));
197 
198  int int_flag;
199 
200  libmesh_call_mpi(MPI_Iprobe(src_processor_id,
201  tag.value(),
202  this->get(),
203  &int_flag,
204  stat.get()));
205 
206  flag = int_flag;
207 
208  return stat;
209 }

◆ probe()

status libMesh::Parallel::Communicator::probe ( const unsigned int  src_processor_id,
const MessageTag tag = any_tag 
) const
inline

Blocking message probe. Allows information about a message to be examined before the message is actually received.

We do not currently support probes on one processor without MPI.

Definition at line 174 of file parallel_implementation.h.

References libMesh::Parallel::MessageTag::value().

Referenced by libMesh::Parallel::pull_parallel_vector_data(), libMesh::Parallel::push_parallel_vector_data(), and receive().

176 {
177  LOG_SCOPE("probe()", "Parallel");
178 
179  status stat;
180 
181  libmesh_call_mpi
182  (MPI_Probe (src_processor_id, tag.value(), this->get(), &stat));
183 
184  return stat;
185 }
MPI_Status status
Definition: status.h:41

◆ rank()

◆ receive() [1/6]

template<typename T >
Status libMesh::Parallel::Communicator::receive ( const unsigned int  dest_processor_id,
T &  buf,
const MessageTag tag = any_tag 
) const
inline

Blocking-receive from one processor with data-defined type.

We do not currently support receives on one processor without MPI.

Definition at line 771 of file parallel_implementation.h.

References libMesh::Parallel::Status::get(), probe(), and libMesh::Parallel::MessageTag::value().

Referenced by libMesh::BoundaryInfo::build_node_list_from_side_list(), libMesh::SystemSubsetBySubdomain::init(), libMesh::MeshRefinement::make_coarsening_compatible(), nonblocking_receive_packed_range(), libMesh::SparsityPattern::Build::parallel_sync(), libMesh::Parallel::pull_parallel_vector_data(), libMesh::Parallel::push_parallel_vector_data(), libMesh::Nemesis_IO::read(), libMesh::System::read_SCALAR_dofs(), libMesh::System::read_serialized_blocked_dof_objects(), receive(), receive_packed_range(), libMesh::MeshRefinement::uniformly_coarsen(), libMesh::System::write_SCALAR_dofs(), libMesh::XdrIO::write_serialized_bcs_helper(), libMesh::System::write_serialized_blocked_dof_objects(), libMesh::XdrIO::write_serialized_connectivity(), libMesh::XdrIO::write_serialized_nodes(), and libMesh::XdrIO::write_serialized_nodesets().

774 {
775  LOG_SCOPE("receive()", "Parallel");
776 
777  // Get the status of the message, explicitly provide the
778  // datatype so we can later query the size
779  Status stat(this->probe(src_processor_id, tag), StandardType<T>(&buf));
780 
781  libmesh_call_mpi
782  (MPI_Recv (&buf, 1, StandardType<T>(&buf), src_processor_id,
783  tag.value(), this->get(), stat.get()));
784 
785  return stat;
786 }
status probe(const unsigned int src_processor_id, const MessageTag &tag=any_tag) const

◆ receive() [2/6]

template<typename T >
void libMesh::Parallel::Communicator::receive ( const unsigned int  dest_processor_id,
T &  buf,
Request req,
const MessageTag tag = any_tag 
) const
inline

Nonblocking-receive from one processor with data-defined type.

Definition at line 791 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::Parallel::Request::get(), and libMesh::Parallel::MessageTag::value().

795 {
796  LOG_SCOPE("receive()", "Parallel");
797 
798  libmesh_call_mpi
799  (MPI_Irecv (&buf, 1, StandardType<T>(&buf), src_processor_id,
800  tag.value(), this->get(), req.get()));
801 
802  // The MessageTag should stay registered for the Request lifetime
803  req.add_post_wait_work
804  (new Parallel::PostWaitDereferenceTag(tag));
805 }

◆ receive() [3/6]

template<typename T >
Status libMesh::Parallel::Communicator::receive ( const unsigned int  dest_processor_id,
T &  buf,
const DataType type,
const MessageTag tag = any_tag 
) const
inline

Blocking-receive from one processor with user-defined type.

If T is a container, container-of-containers, etc., then type should be the DataType of the underlying fixed-size entries in the container(s).

Definition at line 1823 of file parallel_implementation.h.

1827 { libmesh_not_implemented(); return Status(); }

◆ receive() [4/6]

template<typename T >
void libMesh::Parallel::Communicator::receive ( const unsigned int  dest_processor_id,
T &  buf,
const DataType type,
Request req,
const MessageTag tag = any_tag 
) const
inline

Nonblocking-receive from one processor with user-defined type.

If T is a container, container-of-containers, etc., then type should be the DataType of the underlying fixed-size entries in the container(s).

Definition at line 1830 of file parallel_implementation.h.

1835 { libmesh_not_implemented(); }

◆ receive() [5/6]

template<typename T >
Status libMesh::Parallel::Communicator::receive ( const unsigned int  src_processor_id,
std::basic_string< T > &  buf,
const MessageTag tag 
) const
inline

Definition at line 727 of file parallel_implementation.h.

References receive().

730 {
731  std::vector<T> tempbuf; // Officially C++ won't let us get a
732  // modifiable array from a string
733 
734  Status stat = this->receive(src_processor_id, tempbuf, tag);
735  buf.assign(tempbuf.begin(), tempbuf.end());
736  return stat;
737 }
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const

◆ receive() [6/6]

template<typename T >
void libMesh::Parallel::Communicator::receive ( const unsigned int  src_processor_id,
std::basic_string< T > &  buf,
Request req,
const MessageTag tag 
) const
inline

Definition at line 742 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), and receive().

746 {
747  // Officially C++ won't let us get a modifiable array from a
748  // string, and we can't even put one on the stack for the
749  // non-blocking case.
750  std::vector<T> * tempbuf = new std::vector<T>();
751 
752  // We can clear the string, but the Request::wait() will need to
753  // handle copying our temporary buffer to it
754  buf.clear();
755 
756  req.add_post_wait_work
757  (new Parallel::PostWaitCopyBuffer<std::vector<T>,
758  std::back_insert_iterator<std::basic_string<T>>>
759  (tempbuf, std::back_inserter(buf)));
760 
761  // Make the Request::wait() then handle deleting the buffer
762  req.add_post_wait_work
763  (new Parallel::PostWaitDeleteBuffer<std::vector<T>>(tempbuf));
764 
765  this->receive(src_processor_id, tempbuf, req, tag);
766 }
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const

◆ receive_packed_range()

template<typename Context , typename OutputIter , typename T >
void libMesh::Parallel::Communicator::receive_packed_range ( const unsigned int  dest_processor_id,
Context *  context,
OutputIter  out,
const T *  output_type,
const MessageTag tag = any_tag 
) const
inline

Blocking-receive range-of-pointers from one processor. This function does not receive raw pointers, but rather constructs new objects whose contents match the objects pointed to by the sender.

The objects will be of type T = iterator_traits<OutputIter>::value_type.

Using std::back_inserter as the output iterator allows receive to fill any container type. Using libMesh::null_output_iterator allows the receive to be dealt with solely by Parallel::unpack(), for objects whose unpack() is written so as to not leak memory when used in this fashion.

A future version of this method should be created to preallocate memory when receiving vectors...

void Parallel::unpack(vector<int>::iterator in, T ** out, Context *) is used to unserialize type T, typically into a new heap-allocated object whose pointer is returned as *out.

unsigned int Parallel::packed_size(const T *, vector<int>::const_iterator) is used to advance to the beginning of the next object's data.

Definition at line 1060 of file parallel_implementation.h.

References receive(), libMesh::Parallel::Status::source(), libMesh::Parallel::Status::tag(), and libMesh::Parallel::unpack_range().

Referenced by send_receive_packed_range().

1065 {
1066  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
1067 
1068  // Receive serialized variable size objects as sequences of buffer_t
1069  std::size_t total_buffer_size = 0;
1070  Status stat = this->receive(src_processor_id, total_buffer_size, tag);
1071 
1072  // Use stat.source() and stat.tag() in subsequent receives - if
1073  // src_processor_id is or tag is "any" then we want to be sure we
1074  // try to receive messages all corresponding to the same send.
1075 
1076  std::size_t received_buffer_size = 0;
1077  while (received_buffer_size < total_buffer_size)
1078  {
1079  std::vector<buffer_t> buffer;
1080  this->receive(stat.source(), buffer, MessageTag(stat.tag()));
1081  received_buffer_size += buffer.size();
1083  (buffer, context, out_iter, output_type);
1084  }
1085 }
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)

◆ reference_unique_tag()

void libMesh::Parallel::Communicator::reference_unique_tag ( int  tagvalue) const

Reference an already-acquired tag, so that we know it will be dereferenced multiple times before we can re-release it.

Definition at line 35 of file communicator.C.

References used_tag_values.

Referenced by libMesh::Parallel::MessageTag::MessageTag(), and libMesh::Parallel::MessageTag::operator=().

36 {
37  // This had better be an already-acquired tag.
38  libmesh_assert(used_tag_values.count(tagvalue));
39 
40  used_tag_values[tagvalue]++;
41 }
std::map< int, unsigned int > used_tag_values
Definition: communicator.h:196

◆ scatter() [1/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::scatter ( const std::vector< T, A > &  data,
T &  recv,
const unsigned int  root_id = 0 
) const
inline

Take a vector of local variables and scatter the ith item to the ith processor in the communicator. The result is saved into recv.

Definition at line 2921 of file parallel_implementation.h.

References data, libMesh::libmesh_ignore(), rank(), and size().

Referenced by scatter().

2924 {
2925  libmesh_ignore(root_id); // Only needed for MPI and/or dbg/devel
2926  libmesh_assert_less (root_id, this->size());
2927 
2928  // Do not allow the root_id to scatter a nullptr vector.
2929  // That would leave recv in an indeterminate state.
2930  libmesh_assert (this->rank() != root_id || this->size() == data.size());
2931 
2932  if (this->size() == 1)
2933  {
2934  libmesh_assert (!this->rank());
2935  libmesh_assert (!root_id);
2936  recv = data[0];
2937  return;
2938  }
2939 
2940  LOG_SCOPE("scatter()", "Parallel");
2941 
2942  T * data_ptr = const_cast<T*>(data.empty() ? nullptr : data.data());
2943  libmesh_ignore(data_ptr); // unused ifndef LIBMESH_HAVE_MPI
2944 
2945  libmesh_call_mpi
2946  (MPI_Scatter (data_ptr, 1, StandardType<T>(data_ptr),
2947  &recv, 1, StandardType<T>(&recv), root_id, this->get()));
2948 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173
IterBase * data

◆ scatter() [2/4]

template<typename T , typename A >
void libMesh::Parallel::Communicator::scatter ( const std::vector< T, A > &  data,
std::vector< T, A > &  recv,
const unsigned int  root_id = 0 
) const
inline

Take a vector of local variables and scatter the ith equal-sized chunk to the ith processor in the communicator. The data size must be a multiple of the communicator size. The result is saved into recv buffer. The recv buffer does not have to be sized prior to this operation.

Definition at line 2953 of file parallel_implementation.h.

References broadcast(), data, libMesh::libmesh_ignore(), rank(), and size().

2956 {
2957  libmesh_assert_less (root_id, this->size());
2958 
2959  if (this->size() == 1)
2960  {
2961  libmesh_assert (!this->rank());
2962  libmesh_assert (!root_id);
2963  recv.assign(data.begin(), data.end());
2964  return;
2965  }
2966 
2967  LOG_SCOPE("scatter()", "Parallel");
2968 
2969  int recv_buffer_size = 0;
2970  if (this->rank() == root_id)
2971  {
2972  libmesh_assert(data.size() % this->size() == 0);
2973  recv_buffer_size = cast_int<int>(data.size() / this->size());
2974  }
2975 
2976  this->broadcast(recv_buffer_size);
2977  recv.resize(recv_buffer_size);
2978 
2979  T * data_ptr = const_cast<T*>(data.empty() ? nullptr : data.data());
2980  T * recv_ptr = recv.empty() ? nullptr : recv.data();
2981  libmesh_ignore(data_ptr, recv_ptr); // unused ifndef LIBMESH_HAVE_MPI
2982 
2983  libmesh_call_mpi
2984  (MPI_Scatter (data_ptr, recv_buffer_size, StandardType<T>(data_ptr),
2985  recv_ptr, recv_buffer_size, StandardType<T>(recv_ptr), root_id, this->get()));
2986 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173
IterBase * data
void broadcast(T &data, const unsigned int root_id=0) const

◆ scatter() [3/4]

template<typename T , typename A1 , typename A2 >
void libMesh::Parallel::Communicator::scatter ( const std::vector< T, A1 > &  data,
const std::vector< int, A2 >  counts,
std::vector< T, A1 > &  recv,
const unsigned int  root_id = 0 
) const
inline

Take a vector of local variables and scatter the ith variable-sized chunk to the ith processor in the communicator. The counts vector should contain the number of items for each processor. The result is saved into recv buffer. The recv buffer does not have to be sized prior to this operation.

Definition at line 2991 of file parallel_implementation.h.

References data, libMesh::libmesh_ignore(), rank(), scatter(), and size().

2995 {
2996  libmesh_assert_less (root_id, this->size());
2997 
2998  if (this->size() == 1)
2999  {
3000  libmesh_assert (!this->rank());
3001  libmesh_assert (!root_id);
3002  libmesh_assert (counts.size() == this->size());
3003  recv.assign(data.begin(), data.begin() + counts[0]);
3004  return;
3005  }
3006 
3007  std::vector<int,A2> displacements(this->size(), 0);
3008  if (root_id == this->rank())
3009  {
3010  libmesh_assert(counts.size() == this->size());
3011 
3012  // Create a displacements vector from the incoming counts vector
3013  unsigned int globalsize = 0;
3014  for (unsigned int i=0; i < this->size(); ++i)
3015  {
3016  displacements[i] = globalsize;
3017  globalsize += counts[i];
3018  }
3019 
3020  libmesh_assert(data.size() == globalsize);
3021  }
3022 
3023  LOG_SCOPE("scatter()", "Parallel");
3024 
3025  // Scatter the buffer sizes to size remote buffers
3026  int recv_buffer_size = 0;
3027  this->scatter(counts, recv_buffer_size, root_id);
3028  recv.resize(recv_buffer_size);
3029 
3030  T * data_ptr = const_cast<T*>(data.empty() ? nullptr : data.data());
3031  int * count_ptr = const_cast<int*>(counts.empty() ? nullptr : counts.data());
3032  T * recv_ptr = recv.empty() ? nullptr : recv.data();
3033  libmesh_ignore(data_ptr, count_ptr, recv_ptr); // unused ifndef LIBMESH_HAVE_MPI
3034 
3035  // Scatter the non-uniform chunks
3036  libmesh_call_mpi
3037  (MPI_Scatterv (data_ptr, count_ptr, displacements.data(), StandardType<T>(data_ptr),
3038  recv_ptr, recv_buffer_size, StandardType<T>(recv_ptr), root_id, this->get()));
3039 }
processor_id_type size() const
Definition: communicator.h:175
void libmesh_ignore(const Args &...)
processor_id_type rank() const
Definition: communicator.h:173
void scatter(const std::vector< T, A > &data, T &recv, const unsigned int root_id=0) const
IterBase * data

◆ scatter() [4/4]

template<typename T , typename A1 , typename A2 >
void libMesh::Parallel::Communicator::scatter ( const std::vector< std::vector< T, A1 >, A2 > &  data,
std::vector< T, A1 > &  recv,
const unsigned int  root_id = 0,
const bool  identical_buffer_sizes = false 
) const
inline

Take a vector of vectors and scatter the ith inner vector to the ith processor in the communicator. The result is saved into recv buffer. The recv buffer does not have to be sized prior to this operation.

Definition at line 3044 of file parallel_implementation.h.

References data, end, rank(), scatter(), and size().

3048 {
3049  libmesh_assert_less (root_id, this->size());
3050 
3051  if (this->size() == 1)
3052  {
3053  libmesh_assert (!this->rank());
3054  libmesh_assert (!root_id);
3055  libmesh_assert (data.size() == this->size());
3056  recv.assign(data[0].begin(), data[0].end());
3057  return;
3058  }
3059 
3060  std::vector<T,A1> stacked_data;
3061  std::vector<int> counts;
3062 
3063  if (root_id == this->rank())
3064  {
3065  libmesh_assert (data.size() == this->size());
3066 
3067  if (!identical_buffer_sizes)
3068  counts.resize(this->size());
3069 
3070  for (std::size_t i=0; i < data.size(); ++i)
3071  {
3072  if (!identical_buffer_sizes)
3073  counts[i] = cast_int<int>(data[i].size());
3074 #ifndef NDEBUG
3075  else
3076  // Check that buffer sizes are indeed equal
3077  libmesh_assert(!i || data[i-1].size() == data[i].size());
3078 #endif
3079  std::copy(data[i].begin(), data[i].end(), std::back_inserter(stacked_data));
3080  }
3081  }
3082 
3083  if (identical_buffer_sizes)
3084  this->scatter(stacked_data, recv, root_id);
3085  else
3086  this->scatter(stacked_data, counts, recv, root_id);
3087 }
processor_id_type size() const
Definition: communicator.h:175
IterBase * end
processor_id_type rank() const
Definition: communicator.h:173
void scatter(const std::vector< T, A > &data, T &recv, const unsigned int root_id=0) const
IterBase * data

◆ semiverify()

template<typename T >
bool libMesh::Parallel::Communicator::semiverify ( const T *  r) const
inline

Verify that a local pointer points to the same value on all processors where it is not nullptr. Containers must have the same value in every entry.

Definition at line 1948 of file parallel_implementation.h.

References max(), min(), libMesh::Parallel::Attributes< T >::set_highest(), libMesh::Parallel::Attributes< T >::set_lowest(), and size().

Referenced by libMesh::MeshTools::libmesh_assert_consistent_distributed_nodes(), libMesh::MeshTools::libmesh_assert_parallel_consistent_new_node_procids(), libMesh::MeshTools::libmesh_assert_parallel_consistent_procids< Node >(), libMesh::MeshTools::libmesh_assert_valid_boundary_ids(), libMesh::MeshTools::libmesh_assert_valid_neighbors(), and libMesh::MeshTools::libmesh_assert_valid_unique_ids().

1949 {
1950  if (this->size() > 1 && Attributes<T>::has_min_max == true)
1951  {
1952  T tempmin, tempmax;
1953  if (r)
1954  tempmin = tempmax = *r;
1955  else
1956  {
1957  Attributes<T>::set_highest(tempmin);
1958  Attributes<T>::set_lowest(tempmax);
1959  }
1960  this->min(tempmin);
1961  this->max(tempmax);
1962  bool invalid = r && ((*r != tempmin) ||
1963  (*r != tempmax));
1964  this->max(invalid);
1965  return !invalid;
1966  }
1967 
1968  static_assert(Attributes<T>::has_min_max,
1969  "Tried to semiverify an unverifiable type");
1970 
1971  return true;
1972 }
processor_id_type size() const
Definition: communicator.h:175
static void set_lowest(T &)
Definition: attributes.h:45
static void set_highest(T &)
Definition: attributes.h:46
static const bool has_min_max
Definition: attributes.h:44

◆ send() [1/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const std::basic_string< T > &  buf,
const MessageTag tag 
) const
inline

Definition at line 213 of file parallel_implementation.h.

References send_mode(), SYNCHRONOUS, and libMesh::Parallel::MessageTag::value().

216 {
217  LOG_SCOPE("send()", "Parallel");
218 
219  T * dataptr = buf.empty() ? nullptr : const_cast<T *>(buf.data());
220 
221  libmesh_call_mpi
222  (((this->send_mode() == SYNCHRONOUS) ?
223  MPI_Ssend : MPI_Send) (dataptr,
224  cast_int<int>(buf.size()),
225  StandardType<T>(dataptr),
226  dest_processor_id,
227  tag.value(),
228  this->get()));
229 }

◆ send() [2/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const std::basic_string< T > &  buf,
Request req,
const MessageTag tag 
) const
inline

Definition at line 234 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::Parallel::Request::get(), send_mode(), SYNCHRONOUS, and libMesh::Parallel::MessageTag::value().

238 {
239  LOG_SCOPE("send()", "Parallel");
240 
241  T * dataptr = buf.empty() ? nullptr : const_cast<T *>(buf.data());
242 
243  libmesh_call_mpi
244  (((this->send_mode() == SYNCHRONOUS) ?
245  MPI_Issend : MPI_Isend) (dataptr,
246  cast_int<int>(buf.size()),
247  StandardType<T>(dataptr),
248  dest_processor_id,
249  tag.value(),
250  this->get(),
251  req.get()));
252 
253  // The MessageTag should stay registered for the Request lifetime
254  req.add_post_wait_work
255  (new Parallel::PostWaitDereferenceTag(tag));
256 }

◆ send() [3/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const T &  buf,
const MessageTag tag = no_tag 
) const
inline

Blocking-send to one processor with data-defined type.

We do not currently support sends on one processor without MPI.

Definition at line 261 of file parallel_implementation.h.

References send_mode(), SYNCHRONOUS, and libMesh::Parallel::MessageTag::value().

Referenced by libMesh::BoundaryInfo::build_node_list_from_side_list(), libMesh::SystemSubsetBySubdomain::init(), libMesh::MeshRefinement::make_coarsening_compatible(), nonblocking_send_packed_range(), libMesh::SparsityPattern::Build::parallel_sync(), libMesh::Parallel::pull_parallel_vector_data(), libMesh::Parallel::push_parallel_vector_data(), libMesh::Nemesis_IO::read(), libMesh::System::read_SCALAR_dofs(), libMesh::System::read_serialized_blocked_dof_objects(), send_packed_range(), libMesh::MeshRefinement::uniformly_coarsen(), libMesh::System::write_SCALAR_dofs(), libMesh::XdrIO::write_serialized_bcs_helper(), libMesh::System::write_serialized_blocked_dof_objects(), libMesh::XdrIO::write_serialized_connectivity(), libMesh::XdrIO::write_serialized_nodes(), and libMesh::XdrIO::write_serialized_nodesets().

264 {
265  LOG_SCOPE("send()", "Parallel");
266 
267  T * dataptr = const_cast<T*> (&buf);
268 
269  libmesh_call_mpi
270  (((this->send_mode() == SYNCHRONOUS) ?
271  MPI_Ssend : MPI_Send) (dataptr,
272  1,
273  StandardType<T>(dataptr),
274  dest_processor_id,
275  tag.value(),
276  this->get()));
277 }

◆ send() [4/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const T &  buf,
Request req,
const MessageTag tag = no_tag 
) const
inline

Nonblocking-send to one processor with data-defined type.

Definition at line 282 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::Parallel::Request::get(), send_mode(), SYNCHRONOUS, and libMesh::Parallel::MessageTag::value().

286 {
287  LOG_SCOPE("send()", "Parallel");
288 
289  T * dataptr = const_cast<T*>(&buf);
290 
291  libmesh_call_mpi
292  (((this->send_mode() == SYNCHRONOUS) ?
293  MPI_Issend : MPI_Isend) (dataptr,
294  1,
295  StandardType<T>(dataptr),
296  dest_processor_id,
297  tag.value(),
298  this->get(),
299  req.get()));
300 
301  // The MessageTag should stay registered for the Request lifetime
302  req.add_post_wait_work
303  (new Parallel::PostWaitDereferenceTag(tag));
304 }

◆ send() [5/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const T &  buf,
const DataType type,
const MessageTag tag = no_tag 
) const
inline

Blocking-send to one processor with user-defined type.

If T is a container, container-of-containers, etc., then type should be the DataType of the underlying fixed-size entries in the container(s).

Definition at line 1775 of file parallel_implementation.h.

1779 { libmesh_not_implemented(); }

◆ send() [6/6]

template<typename T >
void libMesh::Parallel::Communicator::send ( const unsigned int  dest_processor_id,
const T &  buf,
const DataType type,
Request req,
const MessageTag tag = no_tag 
) const
inline

Nonblocking-send to one processor with user-defined type.

If T is a container, container-of-containers, etc., then type should be the DataType of the underlying fixed-size entries in the container(s).

Definition at line 1782 of file parallel_implementation.h.

1787 { libmesh_not_implemented(); }

◆ send_mode() [1/2]

void libMesh::Parallel::Communicator::send_mode ( const SendMode  sm)
inline

Explicitly sets the SendMode type used for send operations.

Definition at line 205 of file communicator.h.

References _send_mode.

Referenced by duplicate(), and split().

205 { _send_mode = sm; }

◆ send_mode() [2/2]

SendMode libMesh::Parallel::Communicator::send_mode ( ) const
inline

Gets the user-requested SendMode.

Definition at line 210 of file communicator.h.

References _send_mode.

Referenced by duplicate(), send(), and split().

210 { return _send_mode; }

◆ send_packed_range() [1/2]

template<typename Context , typename Iter >
void libMesh::Parallel::Communicator::send_packed_range ( const unsigned int  dest_processor_id,
const Context *  context,
Iter  range_begin,
const Iter  range_end,
const MessageTag tag = no_tag 
) const
inline

Blocking-send range-of-pointers to one processor. This function does not send the raw pointers, but rather constructs new objects at the other end whose contents match the objects pointed to by the sender.

void Parallel::pack(const T *, vector<int> & data, const Context *) is used to serialize type T onto the end of a data vector.

unsigned int Parallel::packable_size(const T *, const Context *) is used to allow data vectors to reserve memory, and for additional error checking

Definition at line 562 of file parallel_implementation.h.

References libMesh::Parallel::pack_range(), libMesh::Parallel::packed_range_size(), and send().

Referenced by send_receive_packed_range().

567 {
568  // We will serialize variable size objects from *range_begin to
569  // *range_end as a sequence of plain data (e.g. ints) in this buffer
570  typedef typename std::iterator_traits<Iter>::value_type T;
571 
572  std::size_t total_buffer_size =
573  Parallel::packed_range_size (context, range_begin, range_end);
574 
575  this->send(dest_processor_id, total_buffer_size, tag);
576 
577 #ifdef DEBUG
578  std::size_t used_buffer_size = 0;
579 #endif
580 
581  while (range_begin != range_end)
582  {
583  libmesh_assert_greater (std::distance(range_begin, range_end), 0);
584 
585  std::vector<typename Parallel::Packing<T>::buffer_type> buffer;
586 
587  const Iter next_range_begin = Parallel::pack_range
588  (context, range_begin, range_end, buffer);
589 
590  libmesh_assert_greater (std::distance(range_begin, next_range_begin), 0);
591 
592  range_begin = next_range_begin;
593 
594 #ifdef DEBUG
595  used_buffer_size += buffer.size();
596 #endif
597 
598  // Blocking send of the buffer
599  this->send(dest_processor_id, buffer, tag);
600  }
601 
602 #ifdef DEBUG
603  libmesh_assert_equal_to(used_buffer_size, total_buffer_size);
604 #endif
605 }
void send(const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Definition: packing.h:139
std::size_t packed_range_size(const Context *context, Iter range_begin, const Iter range_end)
Definition: packing.h:118

◆ send_packed_range() [2/2]

template<typename Context , typename Iter >
void libMesh::Parallel::Communicator::send_packed_range ( const unsigned int  dest_processor_id,
const Context *  context,
Iter  range_begin,
const Iter  range_end,
Request req,
const MessageTag tag = no_tag 
) const
inline

Nonblocking-send range-of-pointers to one processor. This function does not send the raw pointers, but rather constructs new objects at the other end whose contents match the objects pointed to by the sender.

void Parallel::pack(const T *, vector<int> & data, const Context *) is used to serialize type T onto the end of a data vector.

unsigned int Parallel::packable_size(const T *, const Context *) is used to allow data vectors to reserve memory, and for additional error checking

Definition at line 609 of file parallel_implementation.h.

References libMesh::Parallel::Request::add_post_wait_work(), libMesh::Parallel::Request::add_prior_request(), libMesh::Parallel::pack_range(), libMesh::Parallel::packed_range_size(), and send().

615 {
616  // Allocate a buffer on the heap so we don't have to free it until
617  // after the Request::wait()
618  typedef typename std::iterator_traits<Iter>::value_type T;
619  typedef typename Parallel::Packing<T>::buffer_type buffer_t;
620 
621  std::size_t total_buffer_size =
622  Parallel::packed_range_size (context, range_begin, range_end);
623 
624  // That local variable will be gone soon; we need a send buffer that
625  // will stick around. I heard you like buffering so I put a buffer
626  // for your buffer size so you can buffer the size of your buffer.
627  std::size_t * total_buffer_size_buffer = new std::size_t;
628  *total_buffer_size_buffer = total_buffer_size;
629 
630  // Delete the buffer size's buffer when we're done
631  Request intermediate_req = request();
632  intermediate_req.add_post_wait_work
633  (new Parallel::PostWaitDeleteBuffer<std::size_t>(total_buffer_size_buffer));
634  this->send(dest_processor_id, *total_buffer_size_buffer, intermediate_req, tag);
635 
636  // And don't finish up the full request until we're done with its
637  // dependencies
638  req.add_prior_request(intermediate_req);
639 
640 #ifdef DEBUG
641  std::size_t used_buffer_size = 0;
642 #endif
643 
644  while (range_begin != range_end)
645  {
646  libmesh_assert_greater (std::distance(range_begin, range_end), 0);
647 
648  std::vector<buffer_t> * buffer = new std::vector<buffer_t>();
649 
650  const Iter next_range_begin =
651  Parallel::pack_range(context, range_begin, range_end,
652  *buffer);
653 
654  libmesh_assert_greater (std::distance(range_begin, next_range_begin), 0);
655 
656  range_begin = next_range_begin;
657 
658 #ifdef DEBUG
659  used_buffer_size += buffer->size();
660 #endif
661 
662  Request next_intermediate_req;
663 
664  Request * my_req = (range_begin == range_end) ? &req : &next_intermediate_req;
665 
666  // Make the Request::wait() handle deleting the buffer
667  my_req->add_post_wait_work
668  (new Parallel::PostWaitDeleteBuffer<std::vector<buffer_t>>
669  (buffer));
670 
671  // Non-blocking send of the buffer
672  this->send(dest_processor_id, *buffer, *my_req, tag);
673 
674  if (range_begin != range_end)
675  req.add_prior_request(*my_req);
676  }
677 }
void send(const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Definition: packing.h:139
MPI_Request request
Definition: request.h:40
std::size_t packed_range_size(const Context *context, Iter range_begin, const Iter range_end)
Definition: packing.h:118

◆ send_receive() [1/3]

template<typename T1 , typename T2 >
void libMesh::Parallel::Communicator::send_receive ( const unsigned int  send_tgt,
const T1 &  send_val,
const unsigned int  recv_source,
T2 &  recv_val,
const MessageTag send_tag = no_tag,
const MessageTag recv_tag = any_tag 
) const
inline

Send data send to one processor while simultaneously receiving other data recv from a (potentially different) processor.

Send-receive data from one processor.

Definition at line 1178 of file parallel_implementation.h.

References rank(), and libMesh::Parallel::MessageTag::value().

Referenced by send_receive().

1184 {
1185  LOG_SCOPE("send_receive()", "Parallel");
1186 
1187  if (dest_processor_id == this->rank() &&
1188  source_processor_id == this->rank())
1189  {
1190  recv = sendvec;
1191  return;
1192  }
1193 
1194  // MPI_STATUS_IGNORE is from MPI-2; using it with some versions of
1195  // MPICH may cause a crash:
1196  // https://bugzilla.mcs.anl.gov/globus/show_bug.cgi?id=1798
1197  libmesh_call_mpi
1198  (MPI_Sendrecv(const_cast<T1*>(&sendvec), 1, StandardType<T1>(&sendvec),
1199  dest_processor_id, send_tag.value(), &recv, 1,
1200  StandardType<T2>(&recv), source_processor_id,
1201  recv_tag.value(), this->get(), MPI_STATUS_IGNORE));
1202 }
processor_id_type rank() const
Definition: communicator.h:173

◆ send_receive() [2/3]

template<typename T1 , typename T2 >
void libMesh::Parallel::Communicator::send_receive ( const unsigned int  dest_processor_id,
const T1 &  send,
const DataType type1,
const unsigned int  source_processor_id,
T2 &  recv,
const DataType type2,
const MessageTag send_tag = no_tag,
const MessageTag recv_tag = any_tag 
) const

Send data send to one processor while simultaneously receiving other data recv from a (potentially different) processor, using a user-specified MPI Dataype.

◆ send_receive() [3/3]

template<typename T , typename A >
void libMesh::Parallel::Communicator::send_receive ( const unsigned int  dest_processor_id,
const std::vector< T, A > &  sendvec,
const unsigned int  source_processor_id,
std::vector< T, A > &  recv,
const MessageTag send_tag,
const MessageTag recv_tag 
) const
inline

Definition at line 1214 of file parallel_implementation.h.

References rank(), and send_receive().

1220 {
1221  if (dest_processor_id == this->rank() &&
1222  source_processor_id == this->rank())
1223  {
1224  LOG_SCOPE("send_receive()", "Parallel");
1225  recv = sendvec;
1226  return;
1227  }
1228 
1229  const T* example = sendvec.empty() ?
1230  (recv.empty() ? nullptr : recv.data()) : sendvec.data();
1231 
1232  // Call the user-defined type version with automatic
1233  // type conversion based on template argument:
1234  this->send_receive (dest_processor_id, sendvec,
1235  StandardType<T>(example),
1236  source_processor_id, recv,
1237  StandardType<T>(example),
1238  send_tag, recv_tag);
1239 }
void send_receive(const unsigned int dest_processor_id, const T1 &send, const unsigned int source_processor_id, T2 &recv, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
processor_id_type rank() const
Definition: communicator.h:173

◆ send_receive_packed_range() [1/2]

template<typename Context1 , typename RangeIter , typename Context2 , typename OutputIter , typename T >
void libMesh::Parallel::Communicator::send_receive_packed_range ( const unsigned int  dest_processor_id,
const Context1 *  context1,
RangeIter  send_begin,
const RangeIter  send_end,
const unsigned int  source_processor_id,
Context2 *  context2,
OutputIter  out,
const T *  output_type,
const MessageTag send_tag = no_tag,
const MessageTag recv_tag = any_tag 
) const
inline

Send a range-of-pointers to one processor while simultaneously receiving another range from a (potentially different) processor. This function does not send or receive raw pointers, but rather constructs new objects at each receiver whose contents match the objects pointed to by the sender.

The objects being sent will be of type T1 = iterator_traits<RangeIter>::value_type, and the objects being received will be of type T2 = iterator_traits<OutputIter>::value_type

void Parallel::pack(const T1*, vector<int> & data, const Context1*) is used to serialize type T1 onto the end of a data vector.

Using std::back_inserter as the output iterator allows send_receive to fill any container type. Using libMesh::null_output_iterator allows the receive to be dealt with solely by Parallel::unpack(), for objects whose unpack() is written so as to not leak memory when used in this fashion.

A future version of this method should be created to preallocate memory when receiving vectors...

void Parallel::unpack(vector<int>::iterator in, T2** out, Context *) is used to unserialize type T2, typically into a new heap-allocated object whose pointer is returned as *out.

unsigned int Parallel::packable_size(const T1*, const Context1*) is used to allow data vectors to reserve memory, and for additional error checking.

unsigned int Parallel::packed_size(const T2*, vector<int>::const_iterator) is used to advance to the beginning of the next object's data.

Definition at line 1302 of file parallel_implementation.h.

References receive_packed_range(), send_packed_range(), and libMesh::Parallel::Request::wait().

1312 {
1313  LOG_SCOPE("send_receive()", "Parallel");
1314 
1315  Parallel::Request req;
1316 
1317  this->send_packed_range (dest_processor_id, context1, send_begin, send_end,
1318  req, send_tag);
1319 
1320  this->receive_packed_range (source_processor_id, context2, out_iter,
1321  output_type, recv_tag);
1322 
1323  req.wait();
1324 }
void send_packed_range(const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, const MessageTag &tag=no_tag) const
void receive_packed_range(const unsigned int dest_processor_id, Context *context, OutputIter out, const T *output_type, const MessageTag &tag=any_tag) const

◆ send_receive_packed_range() [2/2]

template<typename Context1 , typename RangeIter , typename Context2 , typename OutputIter , typename T >
void libMesh::Parallel::Communicator::send_receive_packed_range ( const unsigned int   libmesh_dbg_vardest_processor_id,
const Context1 *  context1,
RangeIter  send_begin,
const RangeIter  send_end,
const unsigned int   libmesh_dbg_varsource_processor_id,
Context2 *  context2,
OutputIter  out_iter,
const T *  output_type,
const MessageTag ,
const MessageTag  
) const
inline

Send-receive range-of-pointers from one processor.

If you call this without MPI you might be making a mistake, but we'll support it.

Definition at line 1876 of file parallel_implementation.h.

References libMesh::Parallel::pack_range(), and libMesh::Parallel::unpack_range().

1886 {
1887  // This makes no sense on one processor unless we're deliberately
1888  // sending to ourself.
1889  libmesh_assert_equal_to(dest_processor_id, 0);
1890  libmesh_assert_equal_to(source_processor_id, 0);
1891 
1892  // On one processor, we just need to pack the range and then unpack
1893  // it again.
1894  typedef typename std::iterator_traits<RangeIter>::value_type T1;
1895  typedef typename Parallel::Packing<T1>::buffer_type buffer_t;
1896 
1897  while (send_begin != send_end)
1898  {
1899  libmesh_assert_greater (std::distance(send_begin, send_end), 0);
1900 
1901  // We will serialize variable size objects from *range_begin to
1902  // *range_end as a sequence of ints in this buffer
1903  std::vector<buffer_t> buffer;
1904 
1905  const RangeIter next_send_begin = Parallel::pack_range
1906  (context1, send_begin, send_end, buffer);
1907 
1908  libmesh_assert_greater (std::distance(send_begin, next_send_begin), 0);
1909 
1910  send_begin = next_send_begin;
1911 
1913  (buffer, context2, out_iter, output_type);
1914  }
1915 }
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Definition: packing.h:139
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)

◆ set_union() [1/2]

template<typename T >
void libMesh::Parallel::Communicator::set_union ( T &  data,
const unsigned int  root_id 
) const

◆ set_union() [2/2]

template<typename T >
void libMesh::Parallel::Communicator::set_union ( T &  data) const

Take a container of local variables on each processor, and replace it with their union over all processors.

◆ size()

◆ split()

void libMesh::Parallel::Communicator::split ( int  color,
int  key,
Communicator target 
) const

Definition at line 89 of file communicator.C.

References _I_duped_it, assign(), clear(), and send_mode().

90 {
91  target.clear();
92  MPI_Comm newcomm;
93  libmesh_call_mpi
94  (MPI_Comm_split(this->get(), color, key, &newcomm));
95 
96  target.assign(newcomm);
97  target._I_duped_it = (color != MPI_UNDEFINED);
98  target.send_mode(this->send_mode());
99 }

◆ sum() [1/3]

◆ sum() [2/3]

template<typename T >
void libMesh::Parallel::Communicator::sum ( T &  libmesh_mpi_varr) const
inline

Definition at line 2478 of file parallel_implementation.h.

References size().

2479 {
2480  if (this->size() > 1)
2481  {
2482  LOG_SCOPE("sum()", "Parallel");
2483 
2484  libmesh_call_mpi
2485  (MPI_Allreduce (MPI_IN_PLACE, &r, 1,
2486  StandardType<T>(&r),
2487  OpFunction<T>::sum(),
2488  this->get()));
2489  }
2490 }
processor_id_type size() const
Definition: communicator.h:175

◆ sum() [3/3]

template<typename T >
void libMesh::Parallel::Communicator::sum ( std::complex< T > &  libmesh_mpi_varr) const
inline

Definition at line 2515 of file parallel_implementation.h.

References size().

2516 {
2517  if (this->size() > 1)
2518  {
2519  LOG_SCOPE("sum()", "Parallel");
2520 
2521  libmesh_call_mpi
2522  (MPI_Allreduce (MPI_IN_PLACE, &r, 2,
2523  StandardType<T>(),
2524  OpFunction<T>::sum(),
2525  this->get()));
2526  }
2527 }
processor_id_type size() const
Definition: communicator.h:175

◆ verify()

template<typename T >
bool libMesh::Parallel::Communicator::verify ( const T &  r) const
inline

Verify that a local variable has the same value on all processors. Containers must have the same value in every entry.

Definition at line 1928 of file parallel_implementation.h.

References max(), min(), and size().

Referenced by allgather(), alltoall(), libMesh::DofMap::check_dirichlet_bcid_consistency(), libMesh::MeshTools::create_processor_bounding_box(), libMesh::MeshTools::create_subdomain_bounding_box(), libMesh::MeshCommunication::delete_remote_elements(), max(), maxloc(), min(), and minloc().

1929 {
1930  if (this->size() > 1 && Attributes<T>::has_min_max == true)
1931  {
1932  T tempmin = r, tempmax = r;
1933  this->min(tempmin);
1934  this->max(tempmax);
1935  bool verified = (r == tempmin) &&
1936  (r == tempmax);
1937  this->min(verified);
1938  return verified;
1939  }
1940 
1941  static_assert(Attributes<T>::has_min_max,
1942  "Tried to verify an unverifiable type");
1943 
1944  return true;
1945 }
processor_id_type size() const
Definition: communicator.h:175
static const bool has_min_max
Definition: attributes.h:44

Member Data Documentation

◆ _communicator

communicator libMesh::Parallel::Communicator::_communicator
private

Definition at line 190 of file communicator.h.

Referenced by assign(), clear(), duplicate(), get(), and libMesh::ParallelObject::operator=().

◆ _I_duped_it

bool libMesh::Parallel::Communicator::_I_duped_it
private

Definition at line 197 of file communicator.h.

Referenced by clear(), duplicate(), and split().

◆ _rank

processor_id_type libMesh::Parallel::Communicator::_rank
private

Definition at line 191 of file communicator.h.

Referenced by assign(), and rank().

◆ _send_mode

SendMode libMesh::Parallel::Communicator::_send_mode
private

Definition at line 192 of file communicator.h.

Referenced by assign(), and send_mode().

◆ _size

processor_id_type libMesh::Parallel::Communicator::_size
private

Definition at line 191 of file communicator.h.

Referenced by assign(), and size().

◆ used_tag_values

std::map<int, unsigned int> libMesh::Parallel::Communicator::used_tag_values
mutableprivate

Definition at line 196 of file communicator.h.

Referenced by dereference_unique_tag(), get_unique_tag(), and reference_unique_tag().


The documentation for this class was generated from the following files: