communicator.h
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2018 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 #ifndef LIBMESH_COMMUNICATOR_H
20 #define LIBMESH_COMMUNICATOR_H
21 
22 // Parallel includes
23 #include "libmesh/data_type.h"
24 #include "libmesh/message_tag.h"
25 #include "libmesh/request.h"
26 #include "libmesh/status.h"
28 
29 // libMesh Includes
30 #include "libmesh/libmesh_common.h"
31 
32 // C++ includes
33 #include <map>
34 #include <string>
35 #include <vector>
36 
37 // C++ includes needed for parallel_communicator_specializations
38 //
39 // These could be forward declarations if only that wasn't illegal
40 #include <numeric> // complex
41 #include <set>
42 
43 namespace libMesh
44 {
45 
53 namespace Parallel
54 {
55 
56 template <typename T>
57 class Packing;
58 
59 #ifdef LIBMESH_HAVE_MPI
60 
61 //-------------------------------------------------------------------
65 typedef MPI_Comm communicator;
66 
70 const unsigned int any_source =
71  static_cast<unsigned int>(MPI_ANY_SOURCE);
72 
73 #else
74 
75 // These shouldn't actually be needed, but must be
76 // unique types for function overloading to work
77 // properly.
78 typedef int communicator; // Must match petsc-nompi definition
79 
80 const unsigned int any_source=0;
81 
82 #endif // LIBMESH_HAVE_MPI
83 
84 //-------------------------------------------------------------------
93 {
94  // Basic operations:
95 public:
96 
100  Communicator ();
101 
102  /*
103  * Constructor from MPI_Comm
104  */
105  explicit Communicator (const communicator & comm);
106 
107  /*
108  * Don't use copy construction or assignment, just copy by reference
109  * or pointer - it's too hard to keep a common used_tag_values if
110  * each communicator is shared by more than one Communicator
111  */
112  Communicator (const Communicator &) = delete;
113  Communicator & operator= (const Communicator &) = delete;
114 
115  /*
116  * Move constructor and assignment operator
117  */
118  Communicator (Communicator &&) = default;
119  Communicator & operator= (Communicator &&) = default;
120 
121  /*
122  * NON-VIRTUAL destructor
123  */
124  ~Communicator ();
125 
126  /*
127  * Create a new communicator between some subset of \p this
128  */
129  void split(int color, int key, Communicator & target) const;
130 
131  /*
132  * Create a new duplicate of \p this communicator
133  */
134  void duplicate(const Communicator & comm);
135 
136  /*
137  * Create a new duplicate of an MPI communicator
138  */
139  void duplicate(const communicator & comm);
140 
141  communicator & get() { return _communicator; }
142 
143  const communicator & get() const { return _communicator; }
144 
152  MessageTag get_unique_tag(int tagvalue) const;
153 
158  void reference_unique_tag(int tagvalue) const;
159 
164  void dereference_unique_tag(int tagvalue) const;
165 
169  void clear();
170 
171  Communicator & operator= (const communicator & comm);
172 
173  processor_id_type rank() const { return _rank; }
174 
175  processor_id_type size() const { return _size; }
176 
181 
182 private:
183 
188  void assign(const communicator & comm);
189 
193 
194  // mutable used_tag_values - not thread-safe, but then Parallel::
195  // isn't thread-safe in general.
196  mutable std::map<int, unsigned int> used_tag_values;
198 
199  // Communication operations:
200 public:
201 
205  void send_mode (const SendMode sm) { _send_mode = sm; }
206 
210  SendMode send_mode() const { return _send_mode; }
211 
215  void barrier () const;
216 
221  template <typename T>
222  bool verify(const T & r) const;
223 
229  template <typename T>
230  bool semiverify(const T * r) const;
231 
236  template <typename T>
237  void min(T & r) const;
238 
244  template <typename T>
245  void minloc(T & r,
246  unsigned int & min_id) const;
247 
253  template <typename T, typename A1, typename A2>
254  void minloc(std::vector<T,A1> & r,
255  std::vector<unsigned int,A2> & min_id) const;
256 
261  template <typename T>
262  void max(T & r) const;
263 
269  template <typename T>
270  void maxloc(T & r,
271  unsigned int & max_id) const;
272 
278  template <typename T, typename A1, typename A2>
279  void maxloc(std::vector<T,A1> & r,
280  std::vector<unsigned int,A2> & max_id) const;
281 
286  template <typename T>
287  void sum(T & r) const;
288 
294  template <typename T>
295  void set_union(T & data, const unsigned int root_id) const;
296 
301  template <typename T>
302  void set_union(T & data) const;
303 
308  status probe (const unsigned int src_processor_id,
309  const MessageTag & tag=any_tag) const;
310 
323  template <typename T>
324  Status packed_range_probe (const unsigned int src_processor_id,
325  const MessageTag & tag,
326  bool & flag) const;
327 
331  template <typename T>
332  void send (const unsigned int dest_processor_id,
333  const T & buf,
334  const MessageTag & tag=no_tag) const;
335 
339  template <typename T>
340  void send (const unsigned int dest_processor_id,
341  const T & buf,
342  Request & req,
343  const MessageTag & tag=no_tag) const;
344 
352  template <typename T>
353  void send (const unsigned int dest_processor_id,
354  const T & buf,
355  const DataType & type,
356  const MessageTag & tag=no_tag) const;
357 
365  template <typename T>
366  void send (const unsigned int dest_processor_id,
367  const T & buf,
368  const DataType & type,
369  Request & req,
370  const MessageTag & tag=no_tag) const;
371 
375  template <typename T>
376  Status receive (const unsigned int dest_processor_id,
377  T & buf,
378  const MessageTag & tag=any_tag) const;
379 
383  template <typename T>
384  void receive (const unsigned int dest_processor_id,
385  T & buf,
386  Request & req,
387  const MessageTag & tag=any_tag) const;
388 
396  template <typename T>
397  Status receive (const unsigned int dest_processor_id,
398  T & buf,
399  const DataType & type,
400  const MessageTag & tag=any_tag) const;
401 
409  template <typename T>
410  void receive (const unsigned int dest_processor_id,
411  T & buf,
412  const DataType & type,
413  Request & req,
414  const MessageTag & tag=any_tag) const;
415 
429  template <typename Context, typename Iter>
430  void send_packed_range (const unsigned int dest_processor_id,
431  const Context * context,
432  Iter range_begin,
433  const Iter range_end,
434  const MessageTag & tag=no_tag) const;
435 
449  template <typename Context, typename Iter>
450  void send_packed_range (const unsigned int dest_processor_id,
451  const Context * context,
452  Iter range_begin,
453  const Iter range_end,
454  Request & req,
455  const MessageTag & tag=no_tag) const;
456 
465  template <typename Context, typename Iter>
466  void nonblocking_send_packed_range (const unsigned int dest_processor_id,
467  const Context * context,
468  Iter range_begin,
469  const Iter range_end,
470  Request & req,
471  const MessageTag & tag=no_tag) const;
472 
473 
482  template <typename Context, typename Iter>
483  void nonblocking_send_packed_range (const unsigned int dest_processor_id,
484  const Context * context,
485  Iter range_begin,
486  const Iter range_end,
487  Request & req,
488  std::shared_ptr<std::vector<typename Parallel::Packing<typename std::iterator_traits<Iter>::value_type>::buffer_type>> & buffer,
489  const MessageTag & tag=no_tag) const;
490 
517  template <typename Context, typename OutputIter, typename T>
518  void receive_packed_range (const unsigned int dest_processor_id,
519  Context * context,
520  OutputIter out,
521  const T * output_type, // used only to infer T
522  const MessageTag & tag=any_tag) const;
523 
536  template <typename Context, typename OutputIter, typename T>
537  void nonblocking_receive_packed_range (const unsigned int src_processor_id,
538  Context * context,
539  OutputIter out,
540  const T * output_type,
541  Request & req,
542  Status & stat,
543  const MessageTag & tag=any_tag) const;
544 
557  template <typename Context, typename OutputIter, typename T>
558  void nonblocking_receive_packed_range (const unsigned int src_processor_id,
559  Context * context,
560  OutputIter out,
561  const T * output_type,
562  Request & req,
563  Status & stat,
564  std::shared_ptr<std::vector<typename Parallel::Packing<T>::buffer_type>> & buffer,
565  const MessageTag & tag=any_tag
566  ) const;
567 
572  template <typename T1, typename T2>
573  void send_receive(const unsigned int dest_processor_id,
574  const T1 & send,
575  const unsigned int source_processor_id,
576  T2 & recv,
577  const MessageTag & send_tag = no_tag,
578  const MessageTag & recv_tag = any_tag) const;
579 
616  template <typename Context1, typename RangeIter, typename Context2,
617  typename OutputIter, typename T>
618  void send_receive_packed_range(const unsigned int dest_processor_id,
619  const Context1 * context1,
620  RangeIter send_begin,
621  const RangeIter send_end,
622  const unsigned int source_processor_id,
623  Context2 * context2,
624  OutputIter out,
625  const T * output_type, // used only to infer T
626  const MessageTag & send_tag = no_tag,
627  const MessageTag & recv_tag = any_tag) const;
628 
634  template <typename T1, typename T2>
635  void send_receive(const unsigned int dest_processor_id,
636  const T1 & send,
637  const DataType & type1,
638  const unsigned int source_processor_id,
639  T2 & recv,
640  const DataType & type2,
641  const MessageTag & send_tag = no_tag,
642  const MessageTag & recv_tag = any_tag) const;
643 
648  template <typename T, typename A>
649  inline void gather(const unsigned int root_id,
650  const T & send,
651  std::vector<T,A> & recv) const;
652 
658  template <typename T, typename A>
659  inline void gather(const unsigned int root_id,
660  const std::basic_string<T> & send,
661  std::vector<std::basic_string<T>,A> & recv,
662  const bool identical_buffer_sizes=false) const;
663 
686  template <typename T, typename A>
687  inline void gather(const unsigned int root_id,
688  std::vector<T,A> & r) const;
689 
694  template <typename T, typename A>
695  inline void allgather(const T & send,
696  std::vector<T,A> & recv) const;
697 
703  template <typename T, typename A>
704  inline void allgather(const std::basic_string<T> & send,
705  std::vector<std::basic_string<T>,A> & recv,
706  const bool identical_buffer_sizes=false) const;
707 
732  template <typename T, typename A>
733  inline void allgather(std::vector<T,A> & r,
734  const bool identical_buffer_sizes = false) const;
735 
739  template <typename T, typename A>
740  inline void allgather(std::vector<std::basic_string<T>,A> & r,
741  const bool identical_buffer_sizes = false) const;
742 
743  //-------------------------------------------------------------------
748  template <typename T, typename A>
749  inline void scatter(const std::vector<T,A> & data,
750  T & recv,
751  const unsigned int root_id=0) const;
752 
759  template <typename T, typename A>
760  inline void scatter(const std::vector<T,A> & data,
761  std::vector<T,A> & recv,
762  const unsigned int root_id=0) const;
763 
770  template <typename T, typename A1, typename A2>
771  inline void scatter(const std::vector<T,A1> & data,
772  const std::vector<int,A2> counts,
773  std::vector<T,A1> & recv,
774  const unsigned int root_id=0) const;
775 
781  template <typename T, typename A1, typename A2>
782  inline void scatter(const std::vector<std::vector<T,A1>,A2> & data,
783  std::vector<T,A1> & recv,
784  const unsigned int root_id=0,
785  const bool identical_buffer_sizes=false) const;
786 
787  //-------------------------------------------------------------------
792  template <typename Context, typename Iter, typename OutputIter>
793  inline void gather_packed_range (const unsigned int root_id,
794  Context * context,
795  Iter range_begin,
796  const Iter range_end,
797  OutputIter out) const;
798 
803  template <typename Context, typename Iter, typename OutputIter>
804  inline void allgather_packed_range (Context * context,
805  Iter range_begin,
806  const Iter range_end,
807  OutputIter out) const;
808 
814  template <typename T, typename A>
815  inline void alltoall(std::vector<T,A> & r) const;
816 
825  template <typename T>
826  inline void broadcast(T & data, const unsigned int root_id=0) const;
827 
845  template <typename Context, typename OutputContext, typename Iter, typename OutputIter>
846  inline void broadcast_packed_range (const Context * context1,
847  Iter range_begin,
848  const Iter range_end,
849  OutputContext * context2,
850  OutputIter out,
851  const unsigned int root_id = 0) const;
852 
860 #include "libmesh/parallel_communicator_specializations"
861 
862 }; // class Communicator
863 
864 
865 } // namespace Parallel
866 
867 } // namespace libMesh
868 
869 #endif // LIBMESH_COMMUNICATOR_H
void set_union(T &data, const unsigned int root_id) const
void send(const unsigned int dest_processor_id, const T &buf, const MessageTag &tag=no_tag) const
void send_receive(const unsigned int dest_processor_id, const T1 &send, const unsigned int source_processor_id, T2 &recv, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
void maxloc(T &r, unsigned int &max_id) const
const unsigned int any_source
Definition: communicator.h:70
processor_id_type size() const
Definition: communicator.h:175
void send_packed_range(const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, const MessageTag &tag=no_tag) const
MPI_Comm communicator
Definition: communicator.h:57
void allgather(const T &send, std::vector< T, A > &recv) const
uint8_t processor_id_type
Definition: id_types.h:99
void receive_packed_range(const unsigned int dest_processor_id, Context *context, OutputIter out, const T *output_type, const MessageTag &tag=any_tag) const
void gather(const unsigned int root_id, const T &send, std::vector< T, A > &recv) const
void alltoall(std::vector< T, A > &r) const
void nonblocking_send_packed_range(const unsigned int dest_processor_id, const Context *context, Iter range_begin, const Iter range_end, Request &req, const MessageTag &tag=no_tag) const
MessageTag get_unique_tag(int tagvalue) const
Definition: communicator.C:201
void send_mode(const SendMode sm)
Definition: communicator.h:205
void minloc(T &r, unsigned int &min_id) const
processor_id_type rank() const
Definition: communicator.h:173
void send_receive_packed_range(const unsigned int dest_processor_id, const Context1 *context1, RangeIter send_begin, const RangeIter send_end, const unsigned int source_processor_id, Context2 *context2, OutputIter out, const T *output_type, const MessageTag &send_tag=no_tag, const MessageTag &recv_tag=any_tag) const
void assign(const communicator &comm)
Definition: communicator.C:155
void scatter(const std::vector< T, A > &data, T &recv, const unsigned int root_id=0) const
void duplicate(const Communicator &comm)
Definition: communicator.C:108
void nonblocking_receive_packed_range(const unsigned int src_processor_id, Context *context, OutputIter out, const T *output_type, Request &req, Status &stat, const MessageTag &tag=any_tag) const
status probe(const unsigned int src_processor_id, const MessageTag &tag=any_tag) const
void allgather_packed_range(Context *context, Iter range_begin, const Iter range_end, OutputIter out) const
const MessageTag no_tag
Definition: message_tag.h:120
Communicator & operator=(const Communicator &)=delete
Status packed_range_probe(const unsigned int src_processor_id, const MessageTag &tag, bool &flag) const
void dereference_unique_tag(int tagvalue) const
Definition: communicator.C:44
static PetscErrorCode Mat * A
Status receive(const unsigned int dest_processor_id, T &buf, const MessageTag &tag=any_tag) const
const MessageTag any_tag
Definition: message_tag.h:115
std::map< int, unsigned int > used_tag_values
Definition: communicator.h:196
IterBase * data
OStreamProxy out(std::cout)
void reference_unique_tag(int tagvalue) const
Definition: communicator.C:35
void gather_packed_range(const unsigned int root_id, Context *context, Iter range_begin, const Iter range_end, OutputIter out) const
void broadcast(T &data, const unsigned int root_id=0) const
void split(int color, int key, Communicator &target) const
Definition: communicator.C:89
void broadcast_packed_range(const Context *context1, Iter range_begin, const Iter range_end, OutputContext *context2, OutputIter out, const unsigned int root_id=0) const