parallel.h
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2018 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 #ifndef LIBMESH_PARALLEL_H
20 #define LIBMESH_PARALLEL_H
21 
22 // libMesh Includes
23 #include "libmesh/libmesh_common.h" // libmesh_assert, cast_int
25 #include "libmesh/auto_ptr.h" // deprecated
26 
27 // C++ includes
28 #include <cstddef>
29 #include <climits>
30 #include <iterator>
31 #include <limits>
32 #include <map>
33 #include <set>
34 #include <string>
35 #include <vector>
36 #include <memory>
37 
38 namespace libMesh
39 {
40 
41 
42 // Macro to identify and debug functions which should only be called in
43 // parallel on every processor at once
44 
45 #ifdef LIBMESH_ENABLE_DEPRECATED
46 #undef parallel_only
47 #ifndef NDEBUG
48 #define parallel_only() do { \
49  libmesh_deprecated(); \
50  libmesh_assert(CommWorld.verify(std::string(__FILE__).size())); \
51  libmesh_assert(CommWorld.verify(std::string(__FILE__))); \
52  libmesh_assert(CommWorld.verify(__LINE__)); } while (0)
53 #else
54 #define parallel_only() ((void) 0)
55 #endif
56 #endif
57 
58 #undef libmesh_parallel_only
59 #ifndef NDEBUG
60 #define libmesh_parallel_only(comm_obj) do { \
61  libmesh_assert((comm_obj).verify(std::string(__FILE__).size())); \
62  libmesh_assert((comm_obj).verify(std::string(__FILE__))); \
63  libmesh_assert((comm_obj).verify(__LINE__)); } while (0)
64 #else
65 #define libmesh_parallel_only(comm_obj) ((void) 0)
66 #endif
67 
68 // Macro to identify and debug functions which should only be called in
69 // parallel on every processor at once
70 
71 #ifdef LIBMESH_ENABLE_DEPRECATED
72 #undef parallel_only_on
73 #ifndef NDEBUG
74 #define parallel_only_on(comm_arg) do { \
75  libmesh_deprecated(); \
76  libmesh_assert(CommWorld.verify(std::string(__FILE__).size(), comm_arg)); \
77  libmesh_assert(CommWorld.verify(std::string(__FILE__), comm_arg)); \
78  libmesh_assert(CommWorld.verify(__LINE__), comm_arg); } while (0)
79 #else
80 #define parallel_only_on(comm_arg) ((void) 0)
81 #endif
82 #endif
83 
84 #undef libmesh_parallel_only_on
85 #ifndef NDEBUG
86 #define libmesh_parallel_only_on(comm_obj,comm_arg) do { \
87  libmesh_assert(comm_obj.verify(std::string(__FILE__).size(), comm_arg)); \
88  libmesh_assert(comm_obj.verify(std::string(__FILE__), comm_arg)); \
89  libmesh_assert(comm_obj.verify(__LINE__), comm_arg); } while (0)
90 #else
91 #define libmesh_parallel_only_on(comm_obj,comm_arg) ((void) 0)
92 #endif
93 
101 namespace Parallel
102 {
103 //-------------------------------------------------------------------
107 class Communicator;
108 class DataType;
109 class Request;
110 class Status;
111 
112 #ifdef LIBMESH_HAVE_MPI
113 
117 #ifndef NDEBUG
118 #define libmesh_assert_mpi_success(error_code) \
119  do \
120  { \
121  if (error_code != MPI_SUCCESS) \
122  { \
123  char libmesh_mpi_error_string[MPI_MAX_ERROR_STRING+1]; \
124  int libmesh_mpi_error_string_len; \
125  MPI_Error_string(error_code, libmesh_mpi_error_string, \
126  &libmesh_mpi_error_string_len); \
127  libmesh_assert_equal_to_msg(error_code, MPI_SUCCESS, \
128  libmesh_mpi_error_string); \
129  } \
130  } \
131  while (0)
132 
133 #else
134 
135 #define libmesh_assert_mpi_success(error_code) ((void) 0)
136 
137 #endif
138 
139 
140 
141 // Only catch MPI return values when asserts are active.
142 #ifndef NDEBUG
143 #define libmesh_call_mpi(mpi_call) \
144  do \
145  { \
146  unsigned int libmesh_mpi_error_code = mpi_call; \
147  libmesh_assert_mpi_success (libmesh_mpi_error_code); \
148  } \
149  while (0)
150 
151 #else
152 
153 #define libmesh_call_mpi(mpi_call) \
154  do \
155  { \
156  mpi_call; \
157  } \
158  while (0)
159 #endif
160 
161 
162 
163 //-------------------------------------------------------------------
167 typedef MPI_Datatype data_type;
168 
172 typedef MPI_Request request;
173 
177 typedef MPI_Status status;
178 
182 typedef MPI_Comm communicator;
183 
188 template <typename T>
189 inline data_type dataplusint_type();
190 
194 template <typename T>
196 {
197 public:
198  T val;
199  int rank;
200 };
201 
205 const unsigned int any_source =
206  static_cast<unsigned int>(MPI_ANY_SOURCE);
207 
208 
209 #else
210 
211 // These shouldn't actually be needed, but must be
212 // unique types for function overloading to work
213 // properly.
214 struct data_type { /* unsigned int t; */ };
215 struct request { /* unsigned int r; */ };
216 struct status { /* unsigned int s; */ };
217 typedef int communicator; // Must match petsc-nompi definition
218 
219 const unsigned int any_source=0;
220 #endif // LIBMESH_HAVE_MPI
221 
222 
223 
224 //-------------------------------------------------------------------
229 {
230 public:
231 
235  static const int invalid_tag = INT_MIN;
236 
241  explicit MessageTag(int tagvalue = invalid_tag)
242  : _tagvalue(tagvalue), _comm(libmesh_nullptr) {}
243 
248  MessageTag(const MessageTag & other);
249 
254  ~MessageTag();
255 
256  int value() const {
257  return _tagvalue;
258  }
259 
260 private:
263 
264  // Constructor for reference-counted unique tags
265  MessageTag(int tagvalue, const Communicator * comm)
266  : _tagvalue(tagvalue), _comm(comm) {}
267 
268  // Let Communicator handle the reference counting
269  friend class Communicator;
270 };
271 
272 
273 //-------------------------------------------------------------------
277 #ifdef LIBMESH_HAVE_MPI
278 const MessageTag any_tag = MessageTag(MPI_ANY_TAG);
279 #else
280 const MessageTag any_tag = MessageTag(-1);
281 #endif
282 
284 
285 
286 //-------------------------------------------------------------------
290 class DataType
291 {
292 public:
293  DataType () : _datatype() {}
294 
295  DataType (const DataType & other) :
296  _datatype(other._datatype)
297  {}
298 
299  DataType (const data_type & type) :
300  _datatype(type)
301  {}
302 
303 #ifdef LIBMESH_HAVE_MPI
304  DataType (const DataType & other, unsigned int count)
305  {
306  // FIXME - if we nest an inner type here will we run into bug
307  // https://github.com/libMesh/libmesh/issues/631 again?
308  MPI_Type_contiguous(count, other._datatype, &_datatype);
309  this->commit();
310  }
311 #else
312  DataType (const DataType &, unsigned int)
313  {
314  }
315 #endif
316 
317  DataType & operator = (const DataType & other)
318  { _datatype = other._datatype; return *this; }
319 
320  DataType & operator = (const data_type & type)
321  { _datatype = type; return *this; }
322 
323  operator const data_type & () const
324  { return _datatype; }
325 
326  operator data_type & ()
327  { return _datatype; }
328 
329  // operator data_type const * () const
330  // { return &_datatype; }
331 
332  // operator data_type * ()
333  // { return &_datatype; }
334 
335  void commit ()
336  {
337 #ifdef LIBMESH_HAVE_MPI
338  MPI_Type_commit (&_datatype);
339 #endif
340  }
341 
342  void free ()
343  {
344 #ifdef LIBMESH_HAVE_MPI
345  MPI_Type_free (&_datatype);
346 #endif
347  }
348 
349 protected:
350 
351  data_type _datatype;
352 };
353 
354 
355 //-------------------------------------------------------------------
356 
357 #ifdef LIBMESH_HAVE_CXX11
358 // A C++03-compatible replacement for std::false_type
360 {
361  static const bool value = false;
362  typedef bool value_type;
363  typedef false_type type;
364  operator value_type() const { return value; }
365 };
366 
367 // Templated helper class to be used with static_assert.
368 template<typename T>
370 {};
371 #endif
372 
380 template <typename T>
381 class StandardType : public DataType
382 {
383 #ifdef LIBMESH_HAVE_CXX11
384  // Get a slightly better compiler diagnostic if we have C++11
385  static_assert(dependent_false<T>::value,
386  "Only specializations of StandardType may be used, did you forget to include a header file (e.g. parallel_algebra.h)?");
387 #endif
388 
389  /*
390  * The unspecialized class is useless, so we make its constructor
391  * private to catch mistakes at compile-time rather than link-time.
392  * Specializations should have a public constructor of the same
393  * form.
394  */
395 private:
396  StandardType(const T * example = libmesh_nullptr);
397 };
398 
406 template <typename T>
408 {
409 #ifdef LIBMESH_HAVE_CXX11
410  // Get a slightly better compiler diagnostic if we have C++11
411  static_assert(dependent_false<T>::value,
412  "Only specializations of OpFunction may be used, did you forget to include a header file (e.g. parallel_algebra.h)?");
413 #endif
414 
415  /*
416  * The unspecialized class defines none of these functions;
417  * specializations will need to define any functions that need to be
418  * usable.
419  *
420  * Most specializations will just return MPI_MIN, etc, but we'll use
421  * a whitelist rather than a default implementation, so that any
422  * attempt to perform a reduction on an unspecialized type will be a
423  * compile-time rather than a run-time failure.
424  */
425  // static MPI_Op max();
426  // static MPI_Op min();
427  // static MPI_Op sum();
428  // static MPI_Op product();
429  // static MPI_Op logical_and();
430  // static MPI_Op bitwise_and();
431  // static MPI_Op logical_or();
432  // static MPI_Op bitwise_or();
433  // static MPI_Op logical_xor();
434  // static MPI_Op bitwise_xor();
435  // static MPI_Op max_loc();
436  // static MPI_Op min_loc();
437 };
438 
439 /*
440  * The unspecialized class gives default, lowest-common-denominator
441  * attributes, for values which can't be used with Parallel min/max.
442  * Specialized classes can set this to true, and should define
443  * the lowest and highest values possible for the type.
444  */
445 template<typename T>
447 {
448  static const bool has_min_max = false;
449  static void set_lowest(T &) {}
450  static void set_highest(T &) {}
451 };
452 
453 
454 
455 //-------------------------------------------------------------------
460 class Status
461 {
462 public:
463  Status ();
464 
465  explicit Status (const data_type & type);
466 
467  explicit Status (const status & status);
468 
469  Status (const status & status,
470  const data_type & type);
471 
472  Status (const Status & status);
473 
474  Status (const Status & status,
475  const data_type & type);
476 
477  status * get() { return &_status; }
478 
479  status const * get() const { return &_status; }
480 
481  int source () const;
482 
483  int tag () const;
484 
485  data_type & datatype () { return _datatype; }
486 
487  const data_type & datatype () const { return _datatype; }
488 
489  unsigned int size (const data_type & type) const;
490 
491  unsigned int size () const;
492 
493 private:
494 
495  status _status;
496  data_type _datatype;
497 };
498 
499 
500 //-------------------------------------------------------------------
505 struct PostWaitWork {
506  virtual ~PostWaitWork() {}
507 
508  virtual void run() {}
509 };
510 
511 
512 //-------------------------------------------------------------------
516 class Request
517 {
518 public:
519  Request ();
520 
521  Request (const request & r);
522 
523  Request (const Request & other);
524 
525  void cleanup();
526 
527  Request & operator = (const Request & other);
528 
529  Request & operator = (const request & r);
530 
531  ~Request ();
532 
533  request * get() { return &_request; }
534 
535  const request * get() const { return &_request; }
536 
537  Status wait ();
538 
539  bool test ();
540 
541  bool test (status & status);
542 
543  void add_prior_request(const Request & req);
544 
545  void add_post_wait_work(PostWaitWork * work);
546 
547 private:
548  request _request;
549 
550  // Breaking non-blocking sends into multiple requests can require chaining
551  // multiple requests into a single Request
552  std::unique_ptr<Request> _prior_request;
553 
554  // post_wait_work->first is a vector of work to do after a wait
555  // finishes; post_wait_work->second is a reference count so that
556  // Request objects will behave roughly like a shared_ptr and be
557  // usable in STL containers
558  std::pair<std::vector <PostWaitWork * >, unsigned int> * post_wait_work;
559 };
560 
564 inline Status wait (Request & r) { return r.wait(); }
565 
569 inline void wait (std::vector<Request> & r)
570 { for (std::size_t i=0; i<r.size(); i++) r[i].wait(); }
571 
572 
580 template <typename T>
581 class Packing {
582 public:
583  // Should be an MPI sendable type in specializations, e.g.
584  // typedef char buffer_type;
585  // typedef unsigned int buffer_type;
586 
587  // Should copy an encoding of the provided object into the provided
588  // output iterator (which is of type buffer_type)
589  template <typename OutputIter, typename Context>
590  static void pack(const T & object,
591  OutputIter data_out,
592  const Context * context);
593 
594  // Should return the number of array entries (of type buffer_type)
595  // required to encode the provided object
596  template <typename Context>
597  static unsigned int packable_size(const T & object,
598  const Context * context);
599 
600  // Should return the number of array entries which were used to
601  // encode the provided serialization of an object which begins at
602  // \p iter
603  template <typename BufferIter>
604  static unsigned int packed_size(BufferIter iter);
605 
606  // Decode a potentially-variable-size object from a subsequence of a
607  // data array, returning a heap-allocated pointer to the result.
608  template <typename BufferIter, typename Context>
609  static T unpack(BufferIter in, Context * ctx);
610 };
611 
612 
617 template <typename Context, typename buffertype,
618  typename OutputIter, typename T>
619 inline void unpack_range (const typename std::vector<buffertype> & buffer,
620  Context * context,
621  OutputIter out,
622  const T * output_type /* used only to infer T */);
623 
632 template <typename Context, typename buffertype, typename Iter>
633 inline Iter pack_range (const Context * context,
634  Iter range_begin,
635  const Iter range_end,
636  typename std::vector<buffertype> & buffer,
637  std::size_t approx_buffer_size = 1000000);
638 
643 template <typename Context, typename Iter>
644 inline std::size_t packed_range_size (const Context * context,
645  Iter range_begin,
646  const Iter range_end);
647 
648 //-------------------------------------------------------------------
657 {
658  // Basic operations:
659 public:
660 
664  Communicator ();
665 
666  /*
667  * Constructor from MPI_Comm
668  */
669  explicit Communicator (const communicator & comm);
670 
671  /*
672  * NON-VIRTUAL destructor
673  */
674  ~Communicator ();
675 
676  /*
677  * Create a new communicator between some subset of \p this
678  */
679  void split(int color, int key, Communicator & target) const;
680 
681  /*
682  * Create a new duplicate of \p this communicator
683  */
684  void duplicate(const Communicator & comm);
685 
686  /*
687  * Create a new duplicate of an MPI communicator
688  */
689  void duplicate(const communicator & comm);
690 
691  communicator & get() { return _communicator; }
692 
693  const communicator & get() const { return _communicator; }
694 
702  MessageTag get_unique_tag(int tagvalue) const;
703 
708  void reference_unique_tag(int tagvalue) const;
709 
714  void dereference_unique_tag(int tagvalue) const;
715 
719  void clear();
720 
721  Communicator & operator= (const communicator & comm);
722 
723  unsigned int rank() const { return _rank; }
724 
725  unsigned int size() const { return _size; }
726 
730  enum SendMode { DEFAULT=0, SYNCHRONOUS };
731 
732 private:
733 
734  // Don't use the copy constructor, just copy by reference or
735  // pointer - it's too hard to keep a common used_tag_values if
736  // each communicator is shared by more than one Communicator
737  explicit Communicator (const Communicator &);
738 
743  void assign(const communicator & comm);
744 
745  communicator _communicator;
746  unsigned int _rank, _size;
748 
749  // mutable used_tag_values - not thread-safe, but then Parallel::
750  // isn't thread-safe in general.
751  mutable std::map<int, unsigned int> used_tag_values;
753 
754  // Communication operations:
755 public:
756 
760  void send_mode (const SendMode sm) { _send_mode = sm; }
761 
765  SendMode send_mode() const { return _send_mode; }
766 
770  void barrier () const;
771 
776  template <typename T>
777  bool verify(const T & r) const;
778 
784  template <typename T>
785  bool semiverify(const T * r) const;
786 
791  template <typename T>
792  void min(T & r) const;
793 
799  template <typename T>
800  void minloc(T & r,
801  unsigned int & min_id) const;
802 
808  template <typename T>
809  void minloc(std::vector<T> & r,
810  std::vector<unsigned int> & min_id) const;
811 
816  template <typename T>
817  void max(T & r) const;
818 
824  template <typename T>
825  void maxloc(T & r,
826  unsigned int & max_id) const;
827 
833  template <typename T>
834  void maxloc(std::vector<T> & r,
835  std::vector<unsigned int> & max_id) const;
836 
841  template <typename T>
842  void sum(T & r) const;
843 
849  template <typename T>
850  void set_union(T & data, const unsigned int root_id) const;
851 
856  template <typename T>
857  void set_union(T & data) const;
858 
863  status probe (const unsigned int src_processor_id,
864  const MessageTag & tag=any_tag) const;
865 
878  template <typename T>
879  Status packed_range_probe (const unsigned int src_processor_id,
880  const MessageTag & tag,
881  bool & flag) const;
882 
886  template <typename T>
887  void send (const unsigned int dest_processor_id,
888  const T & buf,
889  const MessageTag & tag=no_tag) const;
890 
894  template <typename T>
895  void send (const unsigned int dest_processor_id,
896  const T & buf,
897  Request & req,
898  const MessageTag & tag=no_tag) const;
899 
903  template <typename T>
904  void send (const unsigned int dest_processor_id,
905  const T & buf,
906  const DataType & type,
907  const MessageTag & tag=no_tag) const;
908 
912  template <typename T>
913  void send (const unsigned int dest_processor_id,
914  const T & buf,
915  const DataType & type,
916  Request & req,
917  const MessageTag & tag=no_tag) const;
918 
922  template <typename T>
923  Status receive (const unsigned int dest_processor_id,
924  T & buf,
925  const MessageTag & tag=any_tag) const;
926 
930  template <typename T>
931  void receive (const unsigned int dest_processor_id,
932  T & buf,
933  Request & req,
934  const MessageTag & tag=any_tag) const;
935 
939  template <typename T>
940  Status receive (const unsigned int dest_processor_id,
941  T & buf,
942  const DataType & type,
943  const MessageTag & tag=any_tag) const;
944 
948  template <typename T>
949  void receive (const unsigned int dest_processor_id,
950  T & buf,
951  const DataType & type,
952  Request & req,
953  const MessageTag & tag=any_tag) const;
954 
968  template <typename Context, typename Iter>
969  void send_packed_range (const unsigned int dest_processor_id,
970  const Context * context,
971  Iter range_begin,
972  const Iter range_end,
973  const MessageTag & tag=no_tag) const;
974 
988  template <typename Context, typename Iter>
989  void send_packed_range (const unsigned int dest_processor_id,
990  const Context * context,
991  Iter range_begin,
992  const Iter range_end,
993  Request & req,
994  const MessageTag & tag=no_tag) const;
995 
1004  template <typename Context, typename Iter>
1005  void nonblocking_send_packed_range (const unsigned int dest_processor_id,
1006  const Context * context,
1007  Iter range_begin,
1008  const Iter range_end,
1009  Request & req,
1010  const MessageTag & tag=no_tag) const;
1011 
1038  template <typename Context, typename OutputIter, typename T>
1039  void receive_packed_range (const unsigned int dest_processor_id,
1040  Context * context,
1041  OutputIter out,
1042  const T * output_type, // used only to infer T
1043  const MessageTag & tag=any_tag) const;
1044 
1057  template <typename Context, typename OutputIter, typename T>
1058  void nonblocking_receive_packed_range (const unsigned int src_processor_id,
1059  Context * context,
1060  OutputIter out,
1061  const T * output_type,
1062  Request & req,
1063  Status & stat,
1064  const MessageTag & tag=any_tag) const;
1065 
1070  template <typename T1, typename T2>
1071  void send_receive(const unsigned int dest_processor_id,
1072  const T1 & send,
1073  const unsigned int source_processor_id,
1074  T2 & recv,
1075  const MessageTag & send_tag = no_tag,
1076  const MessageTag & recv_tag = any_tag) const;
1077 
1114  template <typename Context1, typename RangeIter, typename Context2,
1115  typename OutputIter, typename T>
1116  void send_receive_packed_range(const unsigned int dest_processor_id,
1117  const Context1 * context1,
1118  RangeIter send_begin,
1119  const RangeIter send_end,
1120  const unsigned int source_processor_id,
1121  Context2 * context2,
1122  OutputIter out,
1123  const T * output_type, // used only to infer T
1124  const MessageTag & send_tag = no_tag,
1125  const MessageTag & recv_tag = any_tag) const;
1126 
1132  template <typename T1, typename T2>
1133  void send_receive(const unsigned int dest_processor_id,
1134  const T1 & send,
1135  const DataType & type1,
1136  const unsigned int source_processor_id,
1137  T2 & recv,
1138  const DataType & type2,
1139  const MessageTag & send_tag = no_tag,
1140  const MessageTag & recv_tag = any_tag) const;
1141 
1146  template <typename T>
1147  inline void gather(const unsigned int root_id,
1148  const T & send,
1149  std::vector<T> & recv) const;
1150 
1154  template <typename T>
1155  inline void gather(const unsigned int root_id,
1156  const std::basic_string<T> & send,
1157  std::vector<std::basic_string<T>> & recv,
1158  const bool identical_buffer_sizes=false) const;
1159 
1182  template <typename T>
1183  inline void gather(const unsigned int root_id,
1184  std::vector<T> & r) const;
1185 
1190  template <typename T>
1191  inline void allgather(const T & send,
1192  std::vector<T> & recv) const;
1193 
1197  template <typename T>
1198  inline void allgather(const std::basic_string<T> & send,
1199  std::vector<std::basic_string<T>> & recv,
1200  const bool identical_buffer_sizes=false) const;
1201 
1226  template <typename T>
1227  inline void allgather(std::vector<T> & r,
1228  const bool identical_buffer_sizes = false) const;
1229 
1233  template <typename T>
1234  inline void allgather(std::vector<std::basic_string<T>> & r,
1235  const bool identical_buffer_sizes = false) const;
1236 
1237  //-------------------------------------------------------------------
1242  template <typename T>
1243  inline void scatter(const std::vector<T> & data,
1244  T & recv,
1245  const unsigned int root_id=0) const;
1246 
1253  template <typename T>
1254  inline void scatter(const std::vector<T> & data,
1255  std::vector<T> & recv,
1256  const unsigned int root_id=0) const;
1257 
1264  template <typename T>
1265  inline void scatter(const std::vector<T> & data,
1266  const std::vector<int> counts,
1267  std::vector<T> & recv,
1268  const unsigned int root_id=0) const;
1269 
1275  template <typename T>
1276  inline void scatter(const std::vector<std::vector<T>> & data,
1277  std::vector<T> & recv,
1278  const unsigned int root_id=0,
1279  const bool identical_buffer_sizes=false) const;
1280 
1281  //-------------------------------------------------------------------
1286  template <typename Context, typename Iter, typename OutputIter>
1287  inline void gather_packed_range (const unsigned int root_id,
1288  Context * context,
1289  Iter range_begin,
1290  const Iter range_end,
1291  OutputIter out) const;
1292 
1297  template <typename Context, typename Iter, typename OutputIter>
1298  inline void allgather_packed_range (Context * context,
1299  Iter range_begin,
1300  const Iter range_end,
1301  OutputIter out) const;
1302 
1308  template <typename T>
1309  inline void alltoall(std::vector<T> & r) const;
1310 
1319  template <typename T>
1320  inline void broadcast(T & data, const unsigned int root_id=0) const;
1321 
1339  template <typename Context, typename OutputContext, typename Iter, typename OutputIter>
1340  inline void broadcast_packed_range (const Context * context1,
1341  Iter range_begin,
1342  const Iter range_end,
1343  OutputContext * context2,
1344  OutputIter out,
1345  const unsigned int root_id = 0) const;
1346 
1354 #include "libmesh/parallel_communicator_specializations"
1355 
1356 }; // class Communicator
1357 
1358 // FakeCommunicator for debugging inappropriate CommWorld uses
1360 {
1361  operator Communicator & ()
1362  {
1363  libmesh_not_implemented();
1364  static Communicator temp;
1365  return temp;
1366  }
1367 };
1368 
1369 // PostWaitWork specialization for copying from temporary to
1370 // output containers
1371 template <typename Container, typename OutputIter>
1373  PostWaitCopyBuffer(const Container & buffer, const OutputIter out)
1374  : _buf(buffer), _out(out) {}
1375 
1376  virtual void run() libmesh_override { std::copy(_buf.begin(), _buf.end(), _out); }
1377 
1378 private:
1379  const Container & _buf;
1380  OutputIter _out;
1381 };
1382 
1383 // PostWaitWork specialization for unpacking received buffers.
1384 template <typename Container, typename Context, typename OutputIter,
1385  typename T>
1387  PostWaitUnpackBuffer(const Container & buffer, Context * context, OutputIter out) :
1388  _buf(buffer), _context(context), _out(out) {}
1389 
1390  virtual void run() libmesh_override {
1391 
1392  Parallel::unpack_range(_buf, _context, _out, (T*)libmesh_nullptr);
1393  }
1394 
1395 private:
1396  const Container & _buf;
1397  Context * _context;
1398  OutputIter _out;
1399 };
1400 
1401 
1402 // PostWaitWork specialization for freeing no-longer-needed buffers.
1403 template <typename Container>
1405  PostWaitDeleteBuffer(Container * buffer) : _buf(buffer) {}
1406 
1407  virtual void run() libmesh_override { delete _buf; }
1408 
1409 private:
1410  Container * _buf;
1411 };
1412 
1413 } // namespace Parallel
1414 
1428 #ifdef LIBMESH_DISABLE_COMMWORLD
1430 #define LIBMESH_CAN_DEFAULT_TO_COMMWORLD
1431 #else
1433 #define LIBMESH_CAN_DEFAULT_TO_COMMWORLD = libMesh::CommWorld
1434 #endif
1435 
1436 } // namespace libMesh
1437 
1438 // Define all the implementations separately; users might want to look
1439 // through this file for APIs, and it's long enough already.
1440 
1442 
1443 #endif // LIBMESH_PARALLEL_H
void alltoall(std::vector< T > &r, const Communicator &comm=Communicator_World)
virtual void run() libmesh_override
Definition: parallel.h:1376
void receive_packed_range(const unsigned int src_processor_id, Context *context, OutputIter out_iter, const T *output_type, const MessageTag &tag=any_tag, const Communicator &comm=Communicator_World)
MessageTag(int tagvalue=invalid_tag)
Definition: parallel.h:241
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Status wait(Request &r)
Definition: parallel.h:564
void send_packed_range(const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, const MessageTag &tag=no_tag, const Communicator &comm=Communicator_World)
void send(const unsigned int dest_processor_id, const T &data, const MessageTag &tag=no_tag, const Communicator &comm=Communicator_World)
const unsigned int any_source
Definition: parallel.h:205
unsigned int size() const
Definition: parallel.h:725
static void set_lowest(T &)
Definition: parallel.h:449
MPI_Comm communicator
Definition: parallel.h:182
void allgather_packed_range(Context *context, Iter range_begin, const Iter range_end, OutputIter out_iter, const Communicator &comm=Communicator_World)
MPI_Request request
Definition: parallel.h:172
const class libmesh_nullptr_t libmesh_nullptr
MPI_Datatype data_type
Definition: parallel.h:167
void send_mode(const SendMode sm)
Definition: parallel.h:760
Status receive(const unsigned int src_processor_id, T &buf, const MessageTag &tag=any_tag, const Communicator &comm=Communicator_World)
std::pair< std::vector< PostWaitWork * >, unsigned int > * post_wait_work
Definition: parallel.h:558
void sum(T &r, const Communicator &comm=Communicator_World)
void gather_packed_range(const unsigned int root_id, Context *context, Iter range_begin, const Iter range_end, OutputIter out_iter, const Communicator &comm=Communicator_World)
void broadcast_packed_range(const Context *context1, Iter range_begin, const Iter range_end, OutputContext *context2, OutputIter out_iter, const unsigned int root_id=0, const Communicator &comm=Communicator_World)
PostWaitDeleteBuffer(Container *buffer)
Definition: parallel.h:1405
std::size_t packed_range_size(const Context *context, Iter range_begin, const Iter range_end)
virtual void run() libmesh_override
Definition: parallel.h:1390
void barrier(const Communicator &comm=Communicator_World)
PostWaitCopyBuffer(const Container &buffer, const OutputIter out)
Definition: parallel.h:1373
PostWaitUnpackBuffer(const Container &buffer, Context *context, OutputIter out)
Definition: parallel.h:1387
void broadcast(T &data, const unsigned int root_id=0, const Communicator &comm=Communicator_World)
void allgather(T send, std::vector< T > &recv, const Communicator &comm=Communicator_World)
MessageTag(int tagvalue, const Communicator *comm)
Definition: parallel.h:265
DataType(const data_type &type)
Definition: parallel.h:299
const data_type & datatype() const
Definition: parallel.h:487
void gather(const unsigned int root_id, T send, std::vector< T > &recv, const Communicator &comm=Communicator_World)
MPI_Status status
Definition: parallel.h:177
DataType(const DataType &other)
Definition: parallel.h:295
virtual void run() libmesh_override
Definition: parallel.h:1407
data_type dataplusint_type()
const MessageTag no_tag
Definition: parallel.h:283
tbb::split split
Definition: threads_tbb.h:79
Parallel::FakeCommunicator CommWorld
Definition: libmesh.C:207
bool verify(const T &r, const Communicator &comm=Communicator_World)
DataType(const DataType &, unsigned int)
Definition: parallel.h:312
void set_union(T &data, const unsigned int root_id, const Communicator &comm=Communicator_World)
void max(T &r, const Communicator &comm=Communicator_World)
static const bool value
Definition: xdr_io.C:108
data_type & datatype()
Definition: parallel.h:485
Iterator & operator=(const Iterator &rhs)
const MessageTag any_tag
Definition: parallel.h:278
DataType(const DataType &other, unsigned int count)
Definition: parallel.h:304
unsigned int rank() const
Definition: parallel.h:723
std::map< int, unsigned int > used_tag_values
Definition: parallel.h:751
IterBase * data
static void set_highest(T &)
Definition: parallel.h:450
OStreamProxy out(std::cout)
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)
void maxloc(T &r, U &max_id, const Communicator &comm=Communicator_World)
void min(T &r, const Communicator &comm=Communicator_World)
void send_receive_packed_range(const unsigned int dest_processor_id, const Context1 *context1, RangeIter send_begin, const RangeIter send_end, const unsigned int source_processor_id, Context2 *context2, OutputIter out_iter, const T *output_type, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag, const Communicator &comm=Communicator_World)
status probe(const unsigned int src_processor_id, const MessageTag &tag=any_tag, const Communicator &comm=Communicator_World)
void minloc(T &r, U &min_id, const Communicator &comm=Communicator_World)
const Communicator * _comm
Definition: parallel.h:262
SendMode send_mode() const
Definition: parallel.h:765
std::unique_ptr< Request > _prior_request
Definition: parallel.h:552
void send_receive(const unsigned int dest_processor_id, T1 &send, const unsigned int source_processor_id, T2 &recv, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag, const Communicator &comm=Communicator_World)