parallel.h
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2017 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 #ifndef LIBMESH_PARALLEL_H
20 #define LIBMESH_PARALLEL_H
21 
22 // libMesh Includes
23 #include "libmesh/libmesh_common.h" // libmesh_assert, cast_int
25 #include "libmesh/auto_ptr.h"
26 
27 // C++ includes
28 #include <cstddef>
29 #include <climits>
30 #include <iterator>
31 #include <limits>
32 #include <map>
33 #include <set>
34 #include <string>
35 #include <vector>
36 
37 namespace libMesh
38 {
39 
40 
41 // Macro to identify and debug functions which should only be called in
42 // parallel on every processor at once
43 
44 #undef parallel_only
45 #ifndef NDEBUG
46 #define parallel_only() do { \
47  libmesh_deprecated(); \
48  libmesh_assert(CommWorld.verify(std::string(__FILE__).size())); \
49  libmesh_assert(CommWorld.verify(std::string(__FILE__))); \
50  libmesh_assert(CommWorld.verify(__LINE__)); } while (0)
51 #else
52 #define parallel_only() ((void) 0)
53 #endif
54 
55 #undef libmesh_parallel_only
56 #ifndef NDEBUG
57 #define libmesh_parallel_only(comm_obj) do { \
58  libmesh_assert((comm_obj).verify(std::string(__FILE__).size())); \
59  libmesh_assert((comm_obj).verify(std::string(__FILE__))); \
60  libmesh_assert((comm_obj).verify(__LINE__)); } while (0)
61 #else
62 #define libmesh_parallel_only(comm_obj) ((void) 0)
63 #endif
64 
65 // Macro to identify and debug functions which should only be called in
66 // parallel on every processor at once
67 
68 #undef parallel_only_on
69 #ifndef NDEBUG
70 #define parallel_only_on(comm_arg) do { \
71  libmesh_deprecated(); \
72  libmesh_assert(CommWorld.verify(std::string(__FILE__).size(), comm_arg)); \
73  libmesh_assert(CommWorld.verify(std::string(__FILE__), comm_arg)); \
74  libmesh_assert(CommWorld.verify(__LINE__), comm_arg); } while (0)
75 #else
76 #define parallel_only_on(comm_arg) ((void) 0)
77 #endif
78 
79 #undef libmesh_parallel_only_on
80 #ifndef NDEBUG
81 #define libmesh_parallel_only_on(comm_obj,comm_arg) do { \
82  libmesh_assert(comm_obj.verify(std::string(__FILE__).size(), comm_arg)); \
83  libmesh_assert(comm_obj.verify(std::string(__FILE__), comm_arg)); \
84  libmesh_assert(comm_obj.verify(__LINE__), comm_arg); } while (0)
85 #else
86 #define libmesh_parallel_only_on(comm_obj,comm_arg) ((void) 0)
87 #endif
88 
96 namespace Parallel
97 {
98 //-------------------------------------------------------------------
102 class Communicator;
103 class DataType;
104 class Request;
105 class Status;
106 
107 #ifdef LIBMESH_HAVE_MPI
108 
112 #ifndef NDEBUG
113 #define libmesh_assert_mpi_success(error_code) \
114  do \
115  { \
116  if (error_code != MPI_SUCCESS) \
117  { \
118  char libmesh_mpi_error_string[MPI_MAX_ERROR_STRING+1]; \
119  int libmesh_mpi_error_string_len; \
120  MPI_Error_string(error_code, libmesh_mpi_error_string, \
121  &libmesh_mpi_error_string_len); \
122  libmesh_assert_equal_to_msg(error_code, MPI_SUCCESS, \
123  libmesh_mpi_error_string); \
124  } \
125  } \
126  while (0)
127 
128 #else
129 
130 #define libmesh_assert_mpi_success(error_code) ((void) 0)
131 
132 #endif
133 
134 
135 
136 // Only catch MPI return values when asserts are active.
137 #ifndef NDEBUG
138 #define libmesh_call_mpi(mpi_call) \
139  do \
140  { \
141  unsigned int libmesh_mpi_error_code = mpi_call; \
142  libmesh_assert_mpi_success (libmesh_mpi_error_code); \
143  } \
144  while (0)
145 
146 #else
147 
148 #define libmesh_call_mpi(mpi_call) \
149  do \
150  { \
151  mpi_call; \
152  } \
153  while (0)
154 #endif
155 
156 
157 
158 //-------------------------------------------------------------------
162 typedef MPI_Datatype data_type;
163 
167 typedef MPI_Request request;
168 
172 typedef MPI_Status status;
173 
177 typedef MPI_Comm communicator;
178 
183 template <typename T>
184 inline data_type dataplusint_type();
185 
189 template <typename T>
191 {
192 public:
193  T val;
194  int rank;
195 };
196 
200 const unsigned int any_source =
201  static_cast<unsigned int>(MPI_ANY_SOURCE);
202 
203 
204 #else
205 
206 // These shouldn't actually be needed, but must be
207 // unique types for function overloading to work
208 // properly.
209 struct data_type { /* unsigned int t; */ };
210 struct request { /* unsigned int r; */ };
211 struct status { /* unsigned int s; */ };
212 typedef int communicator; // Must match petsc-nompi definition
213 
214 const unsigned int any_source=0;
215 #endif // LIBMESH_HAVE_MPI
216 
217 
218 
219 //-------------------------------------------------------------------
224 {
225 public:
226 
230  static const int invalid_tag = INT_MIN;
231 
236  explicit MessageTag(int tagvalue = invalid_tag)
237  : _tagvalue(tagvalue), _comm(libmesh_nullptr) {}
238 
243  MessageTag(const MessageTag & other);
244 
249  ~MessageTag();
250 
251  int value() const {
252  return _tagvalue;
253  }
254 
255 private:
258 
259  // Constructor for reference-counted unique tags
260  MessageTag(int tagvalue, const Communicator * comm)
261  : _tagvalue(tagvalue), _comm(comm) {}
262 
263  // Let Communicator handle the reference counting
264  friend class Communicator;
265 };
266 
267 
268 //-------------------------------------------------------------------
272 #ifdef LIBMESH_HAVE_MPI
273 const MessageTag any_tag = MessageTag(MPI_ANY_TAG);
274 #else
275 const MessageTag any_tag = MessageTag(-1);
276 #endif
277 
279 
280 
281 //-------------------------------------------------------------------
285 class DataType
286 {
287 public:
288  DataType () : _datatype() {}
289 
290  DataType (const DataType & other) :
291  _datatype(other._datatype)
292  {}
293 
294  DataType (const data_type & type) :
295  _datatype(type)
296  {}
297 
298 #ifdef LIBMESH_HAVE_MPI
299  DataType (const DataType & other, unsigned int count)
300  {
301  // FIXME - if we nest an inner type here will we run into bug
302  // https://github.com/libMesh/libmesh/issues/631 again?
303  MPI_Type_contiguous(count, other._datatype, &_datatype);
304  this->commit();
305  }
306 #else
307  DataType (const DataType &, unsigned int)
308  {
309  }
310 #endif
311 
312  DataType & operator = (const DataType & other)
313  { _datatype = other._datatype; return *this; }
314 
315  DataType & operator = (const data_type & type)
316  { _datatype = type; return *this; }
317 
318  operator const data_type & () const
319  { return _datatype; }
320 
321  operator data_type & ()
322  { return _datatype; }
323 
324  // operator data_type const * () const
325  // { return &_datatype; }
326 
327  // operator data_type * ()
328  // { return &_datatype; }
329 
330  void commit ()
331  {
332 #ifdef LIBMESH_HAVE_MPI
333  MPI_Type_commit (&_datatype);
334 #endif
335  }
336 
337  void free ()
338  {
339 #ifdef LIBMESH_HAVE_MPI
340  MPI_Type_free (&_datatype);
341 #endif
342  }
343 
344 protected:
345 
346  data_type _datatype;
347 };
348 
349 
350 //-------------------------------------------------------------------
351 
352 #ifdef LIBMESH_HAVE_CXX11
353 // A C++03-compatible replacement for std::false_type
355 {
356  static const bool value = false;
357  typedef bool value_type;
358  typedef false_type type;
359  operator value_type() const { return value; }
360 };
361 
362 // Templated helper class to be used with static_assert.
363 template<typename T>
365 {};
366 #endif
367 
375 template <typename T>
376 class StandardType : public DataType
377 {
378 #ifdef LIBMESH_HAVE_CXX11
379  // Get a slightly better compiler diagnostic if we have C++11
380  static_assert(dependent_false<T>::value,
381  "Only specializations of StandardType may be used, did you forget to include a header file (e.g. parallel_algebra.h)?");
382 #endif
383 
384  /*
385  * The unspecialized class is useless, so we make its constructor
386  * private to catch mistakes at compile-time rather than link-time.
387  * Specializations should have a public constructor of the same
388  * form.
389  */
390 private:
391  StandardType(const T * example = libmesh_nullptr);
392 };
393 
401 template <typename T>
403 {
404 #ifdef LIBMESH_HAVE_CXX11
405  // Get a slightly better compiler diagnostic if we have C++11
406  static_assert(dependent_false<T>::value,
407  "Only specializations of OpFunction may be used, did you forget to include a header file (e.g. parallel_algebra.h)?");
408 #endif
409 
410  /*
411  * The unspecialized class defines none of these functions;
412  * specializations will need to define any functions that need to be
413  * usable.
414  *
415  * Most specializations will just return MPI_MIN, etc, but we'll use
416  * a whitelist rather than a default implementation, so that any
417  * attempt to perform a reduction on an unspecialized type will be a
418  * compile-time rather than a run-time failure.
419  */
420 /*
421  static MPI_Op max();
422  static MPI_Op min();
423  static MPI_Op sum();
424  static MPI_Op product();
425  static MPI_Op logical_and();
426  static MPI_Op bitwise_and();
427  static MPI_Op logical_or();
428  static MPI_Op bitwise_or();
429  static MPI_Op logical_xor();
430  static MPI_Op bitwise_xor();
431  static MPI_Op max_loc();
432  static MPI_Op min_loc();
433  */
434 };
435 
436 /*
437  * The unspecialized class gives default, lowest-common-denominator
438  * attributes, for values which can't be used with Parallel min/max.
439  * Specialized classes can set this to true, and should define
440  * the lowest and highest values possible for the type.
441  */
442 template<typename T>
444 {
445  static const bool has_min_max = false;
446  static void set_lowest(T &) {}
447  static void set_highest(T &) {}
448 };
449 
450 
451 
452 //-------------------------------------------------------------------
457 class Status
458 {
459 public:
460  Status ();
461 
462  explicit Status (const data_type & type);
463 
464  explicit Status (const status & status);
465 
466  Status (const status & status,
467  const data_type & type);
468 
469  Status (const Status & status);
470 
471  Status (const Status & status,
472  const data_type & type);
473 
474  status * get() { return &_status; }
475 
476  status const * get() const { return &_status; }
477 
478  int source () const;
479 
480  int tag () const;
481 
482  data_type & datatype () { return _datatype; }
483 
484  const data_type & datatype () const { return _datatype; }
485 
486  unsigned int size (const data_type & type) const;
487 
488  unsigned int size () const;
489 
490 private:
491 
492  status _status;
493  data_type _datatype;
494 };
495 
496 
497 //-------------------------------------------------------------------
502 struct PostWaitWork {
503  virtual ~PostWaitWork() {}
504 
505  virtual void run() {}
506 };
507 
508 
509 //-------------------------------------------------------------------
513 class Request
514 {
515 public:
516  Request ();
517 
518  Request (const request & r);
519 
520  Request (const Request & other);
521 
522  void cleanup();
523 
524  Request & operator = (const Request & other);
525 
526  Request & operator = (const request & r);
527 
528  ~Request ();
529 
530  request * get() { return &_request; }
531 
532  const request * get() const { return &_request; }
533 
534  Status wait ();
535 
536  bool test ();
537 
538  bool test (status & status);
539 
540  void add_prior_request(const Request & req);
541 
542  void add_post_wait_work(PostWaitWork * work);
543 
544 private:
545  request _request;
546 
547  // Breaking non-blocking sends into multiple requests can require chaining
548  // multiple requests into a single Request
550 
551  // post_wait_work->first is a vector of work to do after a wait
552  // finishes; post_wait_work->second is a reference count so that
553  // Request objects will behave roughly like a shared_ptr and be
554  // usable in STL containers
555  std::pair<std::vector <PostWaitWork * >, unsigned int> * post_wait_work;
556 };
557 
561 inline Status wait (Request & r) { return r.wait(); }
562 
566 inline void wait (std::vector<Request> & r)
567 { for (std::size_t i=0; i<r.size(); i++) r[i].wait(); }
568 
569 
577 template <typename T>
578 class Packing {
579 public:
580  // Should be an MPI sendable type in specializations, e.g.
581  // typedef char buffer_type;
582  // typedef unsigned int buffer_type;
583 
584  // Should copy an encoding of the provided object into the provided
585  // output iterator (which is of type buffer_type)
586  template <typename OutputIter, typename Context>
587  static void pack(const T & object,
588  OutputIter data_out,
589  const Context * context);
590 
591  // Should return the number of array entries (of type buffer_type)
592  // required to encode the provided object
593  template <typename Context>
594  static unsigned int packable_size(const T & object,
595  const Context * context);
596 
597  // Should return the number of array entries which were used to
598  // encode the provided serialization of an object which begins at
599  // \p iter
600  template <typename BufferIter>
601  static unsigned int packed_size(BufferIter iter);
602 
603  // Decode a potentially-variable-size object from a subsequence of a
604  // data array, returning a heap-allocated pointer to the result.
605  template <typename BufferIter, typename Context>
606  static T unpack(BufferIter in, Context * ctx);
607 };
608 
609 
614 template <typename Context, typename buffertype,
615  typename OutputIter, typename T>
616 inline void unpack_range (const typename std::vector<buffertype> & buffer,
617  Context * context,
618  OutputIter out,
619  const T * output_type /* used only to infer T */);
620 
629 template <typename Context, typename buffertype, typename Iter>
630 inline Iter pack_range (const Context * context,
631  Iter range_begin,
632  const Iter range_end,
633  typename std::vector<buffertype> & buffer,
634  std::size_t approx_buffer_size = 1000000);
635 
640 template <typename Context, typename Iter>
641 inline std::size_t packed_range_size (const Context * context,
642  Iter range_begin,
643  const Iter range_end);
644 
645 //-------------------------------------------------------------------
654 {
655  // Basic operations:
656 public:
657 
661  Communicator ();
662 
663  /*
664  * Constructor from MPI_Comm
665  */
666  explicit Communicator (const communicator & comm);
667 
668  /*
669  * NON-VIRTUAL destructor
670  */
671  ~Communicator ();
672 
673  /*
674  * Create a new communicator between some subset of \p this
675  */
676  void split(int color, int key, Communicator & target) const;
677 
678  /*
679  * Create a new duplicate of \p this communicator
680  */
681  void duplicate(const Communicator & comm);
682 
683  /*
684  * Create a new duplicate of an MPI communicator
685  */
686  void duplicate(const communicator & comm);
687 
688  communicator & get() { return _communicator; }
689 
690  const communicator & get() const { return _communicator; }
691 
699  MessageTag get_unique_tag(int tagvalue) const;
700 
705  void reference_unique_tag(int tagvalue) const;
706 
711  void dereference_unique_tag(int tagvalue) const;
712 
716  void clear();
717 
718  Communicator & operator= (const communicator & comm);
719 
720  unsigned int rank() const { return _rank; }
721 
722  unsigned int size() const { return _size; }
723 
727  enum SendMode { DEFAULT=0, SYNCHRONOUS };
728 
729 private:
730 
731  // Don't use the copy constructor, just copy by reference or
732  // pointer - it's too hard to keep a common used_tag_values if
733  // each communicator is shared by more than one Communicator
734  explicit Communicator (const Communicator &);
735 
740  void assign(const communicator & comm);
741 
742  communicator _communicator;
743  unsigned int _rank, _size;
745 
746  // mutable used_tag_values - not thread-safe, but then Parallel::
747  // isn't thread-safe in general.
748  mutable std::map<int, unsigned int> used_tag_values;
750 
751  // Communication operations:
752 public:
753 
757  void send_mode (const SendMode sm) { _send_mode = sm; }
758 
762  SendMode send_mode() const { return _send_mode; }
763 
767  void barrier () const;
768 
773  template <typename T>
774  bool verify(const T & r) const;
775 
781  template <typename T>
782  bool semiverify(const T * r) const;
783 
788  template <typename T>
789  void min(T & r) const;
790 
796  template <typename T>
797  void minloc(T & r,
798  unsigned int & min_id) const;
799 
805  template <typename T>
806  void minloc(std::vector<T> & r,
807  std::vector<unsigned int> & min_id) const;
808 
813  template <typename T>
814  void max(T & r) const;
815 
821  template <typename T>
822  void maxloc(T & r,
823  unsigned int & max_id) const;
824 
830  template <typename T>
831  void maxloc(std::vector<T> & r,
832  std::vector<unsigned int> & max_id) const;
833 
838  template <typename T>
839  void sum(T & r) const;
840 
846  template <typename T>
847  void set_union(T & data, const unsigned int root_id) const;
848 
853  template <typename T>
854  void set_union(T & data) const;
855 
860  status probe (const unsigned int src_processor_id,
861  const MessageTag & tag=any_tag) const;
862 
875  template <typename T>
876  Status packed_range_probe (const unsigned int src_processor_id,
877  const MessageTag & tag,
878  bool & flag) const;
879 
883  template <typename T>
884  void send (const unsigned int dest_processor_id,
885  const T & buf,
886  const MessageTag & tag=no_tag) const;
887 
891  template <typename T>
892  void send (const unsigned int dest_processor_id,
893  const T & buf,
894  Request & req,
895  const MessageTag & tag=no_tag) const;
896 
900  template <typename T>
901  void send (const unsigned int dest_processor_id,
902  const T & buf,
903  const DataType & type,
904  const MessageTag & tag=no_tag) const;
905 
909  template <typename T>
910  void send (const unsigned int dest_processor_id,
911  const T & buf,
912  const DataType & type,
913  Request & req,
914  const MessageTag & tag=no_tag) const;
915 
919  template <typename T>
920  Status receive (const unsigned int dest_processor_id,
921  T & buf,
922  const MessageTag & tag=any_tag) const;
923 
927  template <typename T>
928  void receive (const unsigned int dest_processor_id,
929  T & buf,
930  Request & req,
931  const MessageTag & tag=any_tag) const;
932 
936  template <typename T>
937  Status receive (const unsigned int dest_processor_id,
938  T & buf,
939  const DataType & type,
940  const MessageTag & tag=any_tag) const;
941 
945  template <typename T>
946  void receive (const unsigned int dest_processor_id,
947  T & buf,
948  const DataType & type,
949  Request & req,
950  const MessageTag & tag=any_tag) const;
951 
965  template <typename Context, typename Iter>
966  void send_packed_range (const unsigned int dest_processor_id,
967  const Context * context,
968  Iter range_begin,
969  const Iter range_end,
970  const MessageTag & tag=no_tag) const;
971 
985  template <typename Context, typename Iter>
986  void send_packed_range (const unsigned int dest_processor_id,
987  const Context * context,
988  Iter range_begin,
989  const Iter range_end,
990  Request & req,
991  const MessageTag & tag=no_tag) const;
992 
1001  template <typename Context, typename Iter>
1002  void nonblocking_send_packed_range (const unsigned int dest_processor_id,
1003  const Context * context,
1004  Iter range_begin,
1005  const Iter range_end,
1006  Request & req,
1007  const MessageTag & tag=no_tag) const;
1008 
1035  template <typename Context, typename OutputIter, typename T>
1036  void receive_packed_range (const unsigned int dest_processor_id,
1037  Context * context,
1038  OutputIter out,
1039  const T * output_type, // used only to infer T
1040  const MessageTag & tag=any_tag) const;
1041 
1054  template <typename Context, typename OutputIter, typename T>
1055  void nonblocking_receive_packed_range (const unsigned int src_processor_id,
1056  Context * context,
1057  OutputIter out,
1058  const T * output_type,
1059  Request & req,
1060  Status & stat,
1061  const MessageTag & tag=any_tag) const;
1062 
1067  template <typename T1, typename T2>
1068  void send_receive(const unsigned int dest_processor_id,
1069  const T1 & send,
1070  const unsigned int source_processor_id,
1071  T2 & recv,
1072  const MessageTag & send_tag = no_tag,
1073  const MessageTag & recv_tag = any_tag) const;
1074 
1111  template <typename Context1, typename RangeIter, typename Context2,
1112  typename OutputIter, typename T>
1113  void send_receive_packed_range(const unsigned int dest_processor_id,
1114  const Context1 * context1,
1115  RangeIter send_begin,
1116  const RangeIter send_end,
1117  const unsigned int source_processor_id,
1118  Context2 * context2,
1119  OutputIter out,
1120  const T * output_type, // used only to infer T
1121  const MessageTag & send_tag = no_tag,
1122  const MessageTag & recv_tag = any_tag) const;
1123 
1129  template <typename T1, typename T2>
1130  void send_receive(const unsigned int dest_processor_id,
1131  const T1 & send,
1132  const DataType & type1,
1133  const unsigned int source_processor_id,
1134  T2 & recv,
1135  const DataType & type2,
1136  const MessageTag & send_tag = no_tag,
1137  const MessageTag & recv_tag = any_tag) const;
1138 
1143  template <typename T>
1144  inline void gather(const unsigned int root_id,
1145  const T & send,
1146  std::vector<T> & recv) const;
1147 
1151  template <typename T>
1152  inline void gather(const unsigned int root_id,
1153  const std::basic_string<T> & send,
1154  std::vector<std::basic_string<T> > & recv,
1155  const bool identical_buffer_sizes=false) const;
1156 
1179  template <typename T>
1180  inline void gather(const unsigned int root_id,
1181  std::vector<T> & r) const;
1182 
1187  template <typename T>
1188  inline void allgather(const T & send,
1189  std::vector<T> & recv) const;
1190 
1194  template <typename T>
1195  inline void allgather(const std::basic_string<T> & send,
1196  std::vector<std::basic_string<T> > & recv,
1197  const bool identical_buffer_sizes=false) const;
1198 
1199 
1224  template <typename T>
1225  inline void allgather(std::vector<T> & r,
1226  const bool identical_buffer_sizes = false) const;
1227 
1228  //-------------------------------------------------------------------
1233  template <typename T>
1234  inline void scatter(const std::vector<T> & data,
1235  T & recv,
1236  const unsigned int root_id=0) const;
1237 
1244  template <typename T>
1245  inline void scatter(const std::vector<T> & data,
1246  std::vector<T> & recv,
1247  const unsigned int root_id=0) const;
1248 
1255  template <typename T>
1256  inline void scatter(const std::vector<T> & data,
1257  const std::vector<int> counts,
1258  std::vector<T> & recv,
1259  const unsigned int root_id=0) const;
1260 
1266  template <typename T>
1267  inline void scatter(const std::vector<std::vector<T> > & data,
1268  std::vector<T> & recv,
1269  const unsigned int root_id=0,
1270  const bool identical_buffer_sizes=false) const;
1271 
1272  //-------------------------------------------------------------------
1277  template <typename Context, typename Iter, typename OutputIter>
1278  inline void gather_packed_range (const unsigned int root_id,
1279  Context * context,
1280  Iter range_begin,
1281  const Iter range_end,
1282  OutputIter out) const;
1283 
1288  template <typename Context, typename Iter, typename OutputIter>
1289  inline void allgather_packed_range (Context * context,
1290  Iter range_begin,
1291  const Iter range_end,
1292  OutputIter out) const;
1293 
1299  template <typename T>
1300  inline void alltoall(std::vector<T> & r) const;
1301 
1310  template <typename T>
1311  inline void broadcast(T & data, const unsigned int root_id=0) const;
1312 
1330  template <typename Context, typename OutputContext, typename Iter, typename OutputIter>
1331  inline void broadcast_packed_range (const Context * context1,
1332  Iter range_begin,
1333  const Iter range_end,
1334  OutputContext * context2,
1335  OutputIter out,
1336  const unsigned int root_id = 0) const;
1337 
1345 #include "libmesh/parallel_communicator_specializations"
1346 
1347 }; // class Communicator
1348 
1349 // FakeCommunicator for debugging inappropriate CommWorld uses
1351 {
1352  operator Communicator & ()
1353  {
1354  libmesh_not_implemented();
1355  static Communicator temp;
1356  return temp;
1357  }
1358 };
1359 
1360 // PostWaitWork specialization for copying from temporary to
1361 // output containers
1362 template <typename Container, typename OutputIter>
1364  PostWaitCopyBuffer(const Container & buffer, const OutputIter out)
1365  : _buf(buffer), _out(out) {}
1366 
1367  virtual void run() libmesh_override { std::copy(_buf.begin(), _buf.end(), _out); }
1368 
1369 private:
1370  const Container & _buf;
1371  OutputIter _out;
1372 };
1373 
1374 // PostWaitWork specialization for unpacking received buffers.
1375 template <typename Container, typename Context, typename OutputIter,
1376  typename T>
1378  PostWaitUnpackBuffer(const Container & buffer, Context * context, OutputIter out) :
1379  _buf(buffer), _context(context), _out(out) {}
1380 
1381  virtual void run() libmesh_override {
1382 
1383  Parallel::unpack_range(_buf, _context, _out, (T*)libmesh_nullptr);
1384  }
1385 
1386 private:
1387  const Container & _buf;
1388  Context * _context;
1389  OutputIter _out;
1390 };
1391 
1392 
1393 // PostWaitWork specialization for freeing no-longer-needed buffers.
1394 template <typename Container>
1396  PostWaitDeleteBuffer(Container * buffer) : _buf(buffer) {}
1397 
1398  virtual void run() libmesh_override { delete _buf; }
1399 
1400 private:
1401  Container * _buf;
1402 };
1403 
1404 } // namespace Parallel
1405 
1419 #ifdef LIBMESH_DISABLE_COMMWORLD
1421 #define LIBMESH_CAN_DEFAULT_TO_COMMWORLD
1422 #else
1424 #define LIBMESH_CAN_DEFAULT_TO_COMMWORLD = libMesh::CommWorld
1425 #endif
1426 
1427 } // namespace libMesh
1428 
1429 // Define all the implementations separately; users might want to look
1430 // through this file for APIs, and it's long enough already.
1431 
1433 
1434 #endif // LIBMESH_PARALLEL_H
void alltoall(std::vector< T > &r, const Communicator &comm=Communicator_World)
virtual void run() libmesh_override
Definition: parallel.h:1367
void receive_packed_range(const unsigned int src_processor_id, Context *context, OutputIter out_iter, const T *output_type, const MessageTag &tag=any_tag, const Communicator &comm=Communicator_World)
MessageTag(int tagvalue=invalid_tag)
Definition: parallel.h:236
Iter pack_range(const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
Status wait(Request &r)
Definition: parallel.h:561
void send_packed_range(const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, const MessageTag &tag=no_tag, const Communicator &comm=Communicator_World)
void send(const unsigned int dest_processor_id, const T &data, const MessageTag &tag=no_tag, const Communicator &comm=Communicator_World)
const unsigned int any_source
Definition: parallel.h:200
unsigned int size() const
Definition: parallel.h:722
static void set_lowest(T &)
Definition: parallel.h:446
MPI_Comm communicator
Definition: parallel.h:177
void allgather_packed_range(Context *context, Iter range_begin, const Iter range_end, OutputIter out_iter, const Communicator &comm=Communicator_World)
MPI_Request request
Definition: parallel.h:167
const class libmesh_nullptr_t libmesh_nullptr
MPI_Datatype data_type
Definition: parallel.h:162
void send_mode(const SendMode sm)
Definition: parallel.h:757
Status receive(const unsigned int src_processor_id, T &buf, const MessageTag &tag=any_tag, const Communicator &comm=Communicator_World)
std::pair< std::vector< PostWaitWork * >, unsigned int > * post_wait_work
Definition: parallel.h:555
void sum(T &r, const Communicator &comm=Communicator_World)
void gather_packed_range(const unsigned int root_id, Context *context, Iter range_begin, const Iter range_end, OutputIter out_iter, const Communicator &comm=Communicator_World)
void broadcast_packed_range(const Context *context1, Iter range_begin, const Iter range_end, OutputContext *context2, OutputIter out_iter, const unsigned int root_id=0, const Communicator &comm=Communicator_World)
PostWaitDeleteBuffer(Container *buffer)
Definition: parallel.h:1396
std::unique_ptr< T > UniquePtr
Definition: auto_ptr.h:46
std::size_t packed_range_size(const Context *context, Iter range_begin, const Iter range_end)
virtual void run() libmesh_override
Definition: parallel.h:1381
void barrier(const Communicator &comm=Communicator_World)
PostWaitCopyBuffer(const Container &buffer, const OutputIter out)
Definition: parallel.h:1364
PostWaitUnpackBuffer(const Container &buffer, Context *context, OutputIter out)
Definition: parallel.h:1378
void broadcast(T &data, const unsigned int root_id=0, const Communicator &comm=Communicator_World)
UniquePtr< Request > _prior_request
Definition: parallel.h:549
void allgather(T send, std::vector< T > &recv, const Communicator &comm=Communicator_World)
MessageTag(int tagvalue, const Communicator *comm)
Definition: parallel.h:260
DataType(const data_type &type)
Definition: parallel.h:294
const data_type & datatype() const
Definition: parallel.h:484
void gather(const unsigned int root_id, T send, std::vector< T > &recv, const Communicator &comm=Communicator_World)
PetscErrorCode Vec Mat Mat void * ctx
MPI_Status status
Definition: parallel.h:172
DataType(const DataType &other)
Definition: parallel.h:290
virtual void run() libmesh_override
Definition: parallel.h:1398
data_type dataplusint_type()
const MessageTag no_tag
Definition: parallel.h:278
tbb::split split
Definition: threads_tbb.h:79
Parallel::FakeCommunicator CommWorld
Definition: libmesh.C:206
bool verify(const T &r, const Communicator &comm=Communicator_World)
DataType(const DataType &, unsigned int)
Definition: parallel.h:307
void set_union(T &data, const unsigned int root_id, const Communicator &comm=Communicator_World)
void max(T &r, const Communicator &comm=Communicator_World)
data_type & datatype()
Definition: parallel.h:482
Iterator & operator=(const Iterator &rhs)
const MessageTag any_tag
Definition: parallel.h:273
DataType(const DataType &other, unsigned int count)
Definition: parallel.h:299
unsigned int rank() const
Definition: parallel.h:720
std::map< int, unsigned int > used_tag_values
Definition: parallel.h:748
IterBase * data
static void set_highest(T &)
Definition: parallel.h:447
OStreamProxy out(std::cout)
void unpack_range(const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)
void maxloc(T &r, U &max_id, const Communicator &comm=Communicator_World)
void min(T &r, const Communicator &comm=Communicator_World)
void send_receive_packed_range(const unsigned int dest_processor_id, const Context1 *context1, RangeIter send_begin, const RangeIter send_end, const unsigned int source_processor_id, Context2 *context2, OutputIter out_iter, const T *output_type, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag, const Communicator &comm=Communicator_World)
status probe(const unsigned int src_processor_id, const MessageTag &tag=any_tag, const Communicator &comm=Communicator_World)
void minloc(T &r, U &min_id, const Communicator &comm=Communicator_World)
const Communicator * _comm
Definition: parallel.h:257
SendMode send_mode() const
Definition: parallel.h:762
void send_receive(const unsigned int dest_processor_id, T1 &send, const unsigned int source_processor_id, T2 &recv, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag, const Communicator &comm=Communicator_World)