libMesh::Parallel Namespace Reference

Namespaces
	Utils

Classes
struct	Attributes
class	BinSorter
	Parallel bin sorting object. More...
struct	BuildStandardTypeVector
struct	BuildStandardTypeVector< 0 >
class	Communicator
struct	data_type
class	DataPlusInt
class	DataType
struct	FillDisplacementArray
struct	FillDisplacementArray< 0 >
class	Histogram
	Used in conjunction with a BinSorter for parallel sorting. More...
class	MessageTag
class	OpFunction
class	OpFunction< Point >
class	OpFunction< TypeVector< T > >
class	OpFunction< VectorValue< T > >
struct	opfunction_dependent_false
class	Packing
class	Packing< const Elem * >
class	Packing< const Node * >
class	Packing< Elem * >
class	Packing< Node * >
struct	PostWaitCopyBuffer
struct	PostWaitDeleteBuffer
struct	PostWaitDereferenceSharedPtr
struct	PostWaitDereferenceTag
struct	PostWaitFreeBuffer
struct	PostWaitUnpackBuffer
struct	PostWaitWork
class	Request
struct	request
class	Sort
	Object for performing parallel sorts using MPI. More...
class	StandardType
class	StandardType< Hilbert::HilbertIndices >
class	StandardType< Point >
class	StandardType< std::complex< T > >
class	StandardType< std::pair< T1, T2 > >
class	StandardType< std::tuple< Types... > >
class	StandardType< TensorValue< T > >
class	StandardType< TypeTensor< T > >
class	StandardType< TypeVector< T > >
class	StandardType< VectorValue< T > >
struct	standardtype_dependent_false
class	Status
struct	status
struct	SyncEverything
class	TypeVectorOpFunction

Typedefs
typedef MPI_Comm	communicator
typedef MPI_Datatype	data_type
typedef std::pair< Hilbert::HilbertIndices, unique_id_type >	DofObjectKey
typedef MPI_Request	request
typedef MPI_Status	status

Functions
void	wait (std::vector< Request > &r)
std::size_t	waitany (std::vector< Request > &r)
	LIBMESH_INT_TYPE (char)
	LIBMESH_INT_TYPE (signed char)
	LIBMESH_INT_TYPE (unsigned char)
	LIBMESH_INT_TYPE (short int)
	LIBMESH_INT_TYPE (unsigned short int)
	LIBMESH_INT_TYPE (int)
	LIBMESH_INT_TYPE (long)
	LIBMESH_INT_TYPE (unsigned long long)
	LIBMESH_FLOAT_TYPE (float)
	LIBMESH_FLOAT_TYPE (double)
	LIBMESH_FLOAT_TYPE (long double)
template<typename T , typename C , typename A >
	LIBMESH_CONTAINER_TYPE (std::set< T LIBMESH_ATTRIBUTES_COMMA C LIBMESH_ATTRIBUTES_COMMA A >)
template<typename T , typename A >
	LIBMESH_CONTAINER_TYPE (std::vector< T LIBMESH_ATTRIBUTES_COMMA A >)
	LIBMESH_PARALLEL_INTEGER_OPS (char)
	LIBMESH_PARALLEL_INTEGER_OPS (signed char)
	LIBMESH_PARALLEL_INTEGER_OPS (unsigned char)
	LIBMESH_PARALLEL_INTEGER_OPS (short int)
	LIBMESH_PARALLEL_INTEGER_OPS (unsigned short int)
	LIBMESH_PARALLEL_INTEGER_OPS (int)
	LIBMESH_PARALLEL_INTEGER_OPS (long)
	LIBMESH_PARALLEL_INTEGER_OPS (unsigned long long)
	LIBMESH_PARALLEL_FLOAT_OPS (float)
	LIBMESH_PARALLEL_FLOAT_OPS (double)
	LIBMESH_PARALLEL_FLOAT_OPS (long double)
template<typename Context , typename buffertype , typename OutputIter , typename T >
void	unpack_range (const typename std::vector< buffertype > &buffer, Context *context, OutputIter out, const T *output_type)
template<typename Context , typename buffertype , typename Iter >
Iter	pack_range (const Context *context, Iter range_begin, const Iter range_end, typename std::vector< buffertype > &buffer, std::size_t approx_buffer_size=1000000)
template<typename Context , typename Iter >
std::size_t	packed_range_size (const Context *context, Iter range_begin, const Iter range_end)
template<typename Context , typename buffertype , typename OutputIter , typename T >
void	unpack_range (const std::vector< buffertype > &buffer, Context *context, OutputIter out_iter, const T *)
template<typename Iterator , typename DofObjType , typename SyncFunctor >
void	sync_dofobject_data_by_xyz (const Communicator &comm, const Iterator &range_begin, const Iterator &range_end, LocationMap< DofObjType > *location_map, SyncFunctor &sync)
template<typename Iterator , typename SyncFunctor >
void	sync_dofobject_data_by_id (const Communicator &comm, const Iterator &range_begin, const Iterator &range_end, SyncFunctor &sync)
template<typename Iterator , typename DofObjectCheckFunctor , typename SyncFunctor >
void	sync_dofobject_data_by_id (const Communicator &comm, const Iterator &range_begin, const Iterator &range_end, const DofObjectCheckFunctor &dofobj_check, SyncFunctor &sync)
template<typename Iterator , typename SyncFunctor >
void	sync_element_data_by_parent_id (MeshBase &mesh, const Iterator &range_begin, const Iterator &range_end, SyncFunctor &sync)
template<typename ElemCheckFunctor , typename NodeCheckFunctor , typename SyncFunctor >
bool	sync_node_data_by_element_id_once (MeshBase &mesh, const MeshBase::const_element_iterator &range_begin, const MeshBase::const_element_iterator &range_end, const ElemCheckFunctor &elem_check, const NodeCheckFunctor &node_check, SyncFunctor &sync)
template<typename ElemCheckFunctor , typename NodeCheckFunctor , typename SyncFunctor >
void	sync_node_data_by_element_id (MeshBase &mesh, const MeshBase::const_element_iterator &range_begin, const MeshBase::const_element_iterator &range_end, const ElemCheckFunctor &elem_check, const NodeCheckFunctor &node_check, SyncFunctor &sync)
template<typename Iterator , typename DofObjType , typename SyncFunctor >
void	sync_dofobject_data_by_xyz (const Communicator &comm, const Iterator &range_begin, const Iterator &range_end, LocationMap< DofObjType > &location_map, SyncFunctor &sync)
template<typename T >
data_type	dataplusint_type ()
template<>
data_type	dataplusint_type< short int > ()
template<>
data_type	dataplusint_type< int > ()
template<>
data_type	dataplusint_type< long > ()
template<>
data_type	dataplusint_type< float > ()
template<>
data_type	dataplusint_type< double > ()
template<>
data_type	dataplusint_type< long double > ()
template<typename MapToVectors , typename RequestContainer , typename ActionFunctor >
void	push_parallel_vector_data (const Communicator &comm, const MapToVectors &data, RequestContainer &reqs, ActionFunctor &act_on_data)
template<typename MapToVectors , typename ActionFunctor >
void	push_parallel_vector_data (const Communicator &comm, const MapToVectors &data, ActionFunctor &act_on_data)
template<typename datum , typename MapToVectors , typename RequestContainer , typename GatherFunctor , typename ActionFunctor >
void	pull_parallel_vector_data (const Communicator &comm, const MapToVectors &queries, RequestContainer &reqs, GatherFunctor &gather_data, ActionFunctor &act_on_data, const datum *example)
template<typename datum , typename MapToVectors , typename GatherFunctor , typename ActionFunctor >
void	pull_parallel_vector_data (const Communicator &comm, const MapToVectors &queries, GatherFunctor &gather_data, ActionFunctor &act_on_data, const datum *example)
template<template< typename, typename, typename ... > class MapType, typename KeyType , typename ValueType , typename A1 , typename A2 , typename ... ExtraTypes, typename RequestContainer , typename ActionFunctor >
void	push_parallel_vector_data (const Communicator &comm, const MapType< processor_id_type, std::vector< std::vector< ValueType, A1 >, A2 >, ExtraTypes... > &data, RequestContainer &reqs, ActionFunctor &act_on_data)
template<template< typename, typename, typename ... > class MapType, typename KeyType , typename ValueType , typename A1 , typename A2 , typename ... ExtraTypes, typename ActionFunctor >
void	push_parallel_vector_data (const Communicator &comm, const MapType< processor_id_type, std::vector< std::vector< ValueType, A1 >, A2 >, ExtraTypes... > &data, ActionFunctor &act_on_data)
template<typename datum , typename A , typename MapToVectors , typename RequestContainer , typename GatherFunctor , typename ActionFunctor >
void	pull_parallel_vector_data (const Communicator &comm, const MapToVectors &queries, RequestContainer &reqs, GatherFunctor &gather_data, ActionFunctor &act_on_data, const std::vector< datum, A > *example)
template<template< typename, typename, typename ... > class MapType, typename ValueType , typename A1 , typename A2 , typename ... ExtraTypes, typename RequestContainer , typename ActionFunctor >
void	push_parallel_vector_data (const Communicator &comm, const MapType< processor_id_type, std::vector< std::vector< ValueType, A1 >, A2 >, ExtraTypes... > &data, RequestContainer &reqs, ActionFunctor &act_on_data)
template<template< typename, typename, typename ... > class MapType, typename ValueType , typename A1 , typename A2 , typename ... ExtraTypes, typename ActionFunctor >
void	push_parallel_vector_data (const Communicator &comm, const MapType< processor_id_type, std::vector< std::vector< ValueType, A1 >, A2 >, ExtraTypes... > &data, ActionFunctor &act_on_data)
Status	wait (Request &r)
	LIBMESH_STANDARD_TYPE (char, MPI_CHAR)
	LIBMESH_STANDARD_TYPE (signed char, MPI_SIGNED_CHAR)
	LIBMESH_STANDARD_TYPE (unsigned char, MPI_UNSIGNED_CHAR)
	LIBMESH_STANDARD_TYPE (short int, MPI_SHORT)
	LIBMESH_STANDARD_TYPE (unsigned short int, MPI_UNSIGNED_SHORT)
	LIBMESH_STANDARD_TYPE (int, MPI_INT)
	LIBMESH_STANDARD_TYPE (unsigned int, MPI_UNSIGNED)
	LIBMESH_STANDARD_TYPE (long, MPI_LONG)
	LIBMESH_STANDARD_TYPE (long long, MPI_LONG_LONG_INT)
	LIBMESH_STANDARD_TYPE (unsigned long, MPI_UNSIGNED_LONG)
	LIBMESH_STANDARD_TYPE (unsigned long long, MPI_UNSIGNED_LONG_LONG)
	LIBMESH_STANDARD_TYPE (float, MPI_FLOAT)
	LIBMESH_STANDARD_TYPE (double, MPI_DOUBLE)
	LIBMESH_STANDARD_TYPE (long double, MPI_LONG_DOUBLE)

Variables
const unsigned int	any_source
const MessageTag	any_tag = MessageTag(MPI_ANY_TAG)
const MessageTag	no_tag = MessageTag(0)

Detailed Description

The Parallel namespace is for wrapper functions for common general parallel synchronization tasks.

For MPI 1.1 compatibility, temporary buffers are used instead of MPI 2's MPI_IN_PLACE

The Parallel namespace is for wrapper functions for common general parallel synchronization tasks.

Typedef Documentation

communicator

typedef int libMesh::Parallel::communicator

Communicator object for talking with subsets of processors

Definition at line 57 of file communicator.h.

data_type

typedef MPI_Datatype libMesh::Parallel::data_type

Data types for communication

Definition at line 46 of file data_type.h.

DofObjectKey

typedef Hilbert::HilbertIndices libMesh::Parallel::DofObjectKey

Definition at line 69 of file parallel_hilbert.h.

request

typedef MPI_Request libMesh::Parallel::request

Request object for non-blocking I/O

Definition at line 40 of file request.h.

status

typedef MPI_Status libMesh::Parallel::status

Status object for querying messages

Definition at line 41 of file status.h.

Function Documentation

dataplusint_type()

template<typename T >

data_type libMesh::Parallel::dataplusint_type ( )

inline

Templated function to return the appropriate MPI datatype for use with built-in C types when combined with an int

dataplusint_type< double >()

template<>

data_type libMesh::Parallel::dataplusint_type< double > ( )

inline

Definition at line 167 of file parallel_implementation.h.

{ return MPI_DOUBLE_INT; }

dataplusint_type< float >()

template<>

data_type libMesh::Parallel::dataplusint_type< float > ( )

inline

Definition at line 164 of file parallel_implementation.h.

{ return MPI_FLOAT_INT; }

dataplusint_type< int >()

template<>

data_type libMesh::Parallel::dataplusint_type< int > ( )

inline

Definition at line 158 of file parallel_implementation.h.

{ return MPI_2INT; }

dataplusint_type< long >()

template<>

data_type libMesh::Parallel::dataplusint_type< long > ( )

inline

Definition at line 161 of file parallel_implementation.h.

{ return MPI_LONG_INT; }

dataplusint_type< long double >()

template<>

data_type libMesh::Parallel::dataplusint_type< long double > ( )

inline

Definition at line 170 of file parallel_implementation.h.

{ return MPI_LONG_DOUBLE_INT; }

dataplusint_type< short int >()

template<>

data_type libMesh::Parallel::dataplusint_type< short int > ( )

inline

Definition at line 155 of file parallel_implementation.h.

{ return MPI_SHORT_INT; }

LIBMESH_CONTAINER_TYPE() [1/2]

template<typename T , typename C , typename A >

libMesh::Parallel::LIBMESH_CONTAINER_TYPE

(

std::set< T LIBMESH_ATTRIBUTES_COMMA C LIBMESH_ATTRIBUTES_COMMA A >

)

LIBMESH_CONTAINER_TYPE() [2/2]

template<typename T , typename A >

libMesh::Parallel::LIBMESH_CONTAINER_TYPE

(

std::vector< T LIBMESH_ATTRIBUTES_COMMA A >

)

LIBMESH_FLOAT_TYPE() [1/3]

libMesh::Parallel::LIBMESH_FLOAT_TYPE

(

float

)

LIBMESH_FLOAT_TYPE() [2/3]

libMesh::Parallel::LIBMESH_FLOAT_TYPE

(

double

)

LIBMESH_FLOAT_TYPE() [3/3]

libMesh::Parallel::LIBMESH_FLOAT_TYPE

(

long

double

)

LIBMESH_INT_TYPE() [1/8]

libMesh::Parallel::LIBMESH_INT_TYPE

(

char

)

LIBMESH_INT_TYPE() [2/8]

libMesh::Parallel::LIBMESH_INT_TYPE

(

signed

char

)

LIBMESH_INT_TYPE() [3/8]

libMesh::Parallel::LIBMESH_INT_TYPE

(

unsigned

char

)

LIBMESH_INT_TYPE() [4/8]

libMesh::Parallel::LIBMESH_INT_TYPE

(

short

int

)

LIBMESH_INT_TYPE() [5/8]

libMesh::Parallel::LIBMESH_INT_TYPE

(

unsigned short

int

)

LIBMESH_INT_TYPE() [6/8]

libMesh::Parallel::LIBMESH_INT_TYPE

(

int

)

LIBMESH_INT_TYPE() [7/8]

libMesh::Parallel::LIBMESH_INT_TYPE

(

long

)

LIBMESH_INT_TYPE() [8/8]

libMesh::Parallel::LIBMESH_INT_TYPE

(

unsigned long

long

)

LIBMESH_PARALLEL_FLOAT_OPS() [1/3]

libMesh::Parallel::LIBMESH_PARALLEL_FLOAT_OPS

(

float

)

LIBMESH_PARALLEL_FLOAT_OPS() [2/3]

libMesh::Parallel::LIBMESH_PARALLEL_FLOAT_OPS

(

double

)

LIBMESH_PARALLEL_FLOAT_OPS() [3/3]

libMesh::Parallel::LIBMESH_PARALLEL_FLOAT_OPS

(

long

double

)

LIBMESH_PARALLEL_INTEGER_OPS() [1/8]

libMesh::Parallel::LIBMESH_PARALLEL_INTEGER_OPS

(

char

)

LIBMESH_PARALLEL_INTEGER_OPS() [2/8]

libMesh::Parallel::LIBMESH_PARALLEL_INTEGER_OPS

(

signed

char

)

LIBMESH_PARALLEL_INTEGER_OPS() [3/8]

libMesh::Parallel::LIBMESH_PARALLEL_INTEGER_OPS

(

unsigned

char

)

LIBMESH_PARALLEL_INTEGER_OPS() [4/8]

libMesh::Parallel::LIBMESH_PARALLEL_INTEGER_OPS

(

short

int

)

LIBMESH_PARALLEL_INTEGER_OPS() [5/8]

libMesh::Parallel::LIBMESH_PARALLEL_INTEGER_OPS

(

unsigned short

int

)

LIBMESH_PARALLEL_INTEGER_OPS() [6/8]

libMesh::Parallel::LIBMESH_PARALLEL_INTEGER_OPS

(

int

)

LIBMESH_PARALLEL_INTEGER_OPS() [7/8]

libMesh::Parallel::LIBMESH_PARALLEL_INTEGER_OPS

(

long

)

LIBMESH_PARALLEL_INTEGER_OPS() [8/8]

libMesh::Parallel::LIBMESH_PARALLEL_INTEGER_OPS

(

unsigned long

long

)

LIBMESH_STANDARD_TYPE() [1/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	char	,
		MPI_CHAR
	)

LIBMESH_STANDARD_TYPE() [2/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	signed	char,
		MPI_SIGNED_CHAR
	)

LIBMESH_STANDARD_TYPE() [3/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	unsigned	char,
		MPI_UNSIGNED_CHAR
	)

LIBMESH_STANDARD_TYPE() [4/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	short	int,
		MPI_SHORT
	)

LIBMESH_STANDARD_TYPE() [5/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	unsigned short	int,
		MPI_UNSIGNED_SHORT
	)

LIBMESH_STANDARD_TYPE() [6/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	int	,
		MPI_INT
	)

LIBMESH_STANDARD_TYPE() [7/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	unsigned	int,
		MPI_UNSIGNED
	)

LIBMESH_STANDARD_TYPE() [8/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	long	,
		MPI_LONG
	)

LIBMESH_STANDARD_TYPE() [9/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	long	long,
		MPI_LONG_LONG_INT
	)

LIBMESH_STANDARD_TYPE() [10/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	unsigned	long,
		MPI_UNSIGNED_LONG
	)

LIBMESH_STANDARD_TYPE() [11/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	unsigned long	long,
		MPI_UNSIGNED_LONG_LONG
	)

LIBMESH_STANDARD_TYPE() [12/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	float	,
		MPI_FLOAT
	)

LIBMESH_STANDARD_TYPE() [13/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	double	,
		MPI_DOUBLE
	)

LIBMESH_STANDARD_TYPE() [14/14]

libMesh::Parallel::LIBMESH_STANDARD_TYPE	(	long	double,
		MPI_LONG_DOUBLE
	)

pack_range()

template<typename Context , typename buffertype , typename Iter >

Iter libMesh::Parallel::pack_range ( const Context *  context, Iter  range_begin, const Iter  range_end, typename std::vector< buffertype > &  buffer, std::size_t  approx_buffer_size = 1000000  )

inline

Encode a range of potentially-variable-size objects to a data array.

The data will be buffered in vectors with lengths that do not exceed the sum of approx_buffer_size and the size of an individual packed object.

Helper function for range packing

Definition at line 139 of file packing.h.

References libMesh::Parallel::Packing< T >::pack(), libMesh::Parallel::Packing< T >::packable_size(), and libMesh::Parallel::Packing< T >::packed_size().

Referenced by libMesh::Parallel::Communicator::allgather_packed_range(), libMesh::Parallel::Communicator::broadcast_packed_range(), libMesh::Parallel::Communicator::gather_packed_range(), libMesh::Parallel::Communicator::nonblocking_send_packed_range(), libMesh::Parallel::Communicator::send_packed_range(), and libMesh::Parallel::Communicator::send_receive_packed_range().

{
  typedef typename std::iterator_traits<Iter>::value_type T;

  // Count the total size of and preallocate buffer for efficiency.
  // Prepare to stop early if the buffer would be too large.
  std::size_t buffer_size = 0;
  Iter range_stop = range_begin;
  for (; range_stop != range_end && buffer_size < approx_buffer_size;
       ++range_stop)
    {
      std::size_t next_buffer_size =
        Parallel::Packing<T>::packable_size(*range_stop, context);
      buffer_size += next_buffer_size;
    }
  buffer.reserve(buffer.size() + buffer_size);

  // Pack the objects into the buffer
  for (; range_begin != range_stop; ++range_begin)
    {
#ifndef NDEBUG
      std::size_t old_size = buffer.size();
#endif

      Parallel::Packing<T>::pack
        (*range_begin, back_inserter(buffer), context);

#ifndef NDEBUG
      unsigned int my_packable_size =
        Parallel::Packing<T>::packable_size(*range_begin, context);
      unsigned int my_packed_size =
        Parallel::Packing<T>::packed_size (buffer.begin() + old_size);
      libmesh_assert_equal_to (my_packable_size, my_packed_size);
      libmesh_assert_equal_to (buffer.size(), old_size + my_packable_size);
#endif
    }

  return range_stop;
}

packed_range_size()

template<typename Context , typename Iter >

std::size_t libMesh::Parallel::packed_range_size ( const Context *  context, Iter  range_begin, const Iter  range_end  )

inline

Return the total buffer size needed to encode a range of potentially-variable-size objects to a data array.

Helper function for range packing

Definition at line 118 of file packing.h.

References libMesh::Parallel::Packing< T >::packable_size().

Referenced by libMesh::Parallel::Communicator::send_packed_range().

{
  typedef typename std::iterator_traits<Iter>::value_type T;

  std::size_t buffer_size = 0;
  for (Iter range_count = range_begin;
       range_count != range_end;
       ++range_count)
    {
      buffer_size += Parallel::Packing<T>::packable_size(*range_count, context);
    }
  return buffer_size;
}

pull_parallel_vector_data() [1/3]

template<typename datum , typename MapToVectors , typename RequestContainer , typename GatherFunctor , typename ActionFunctor >

void libMesh::Parallel::pull_parallel_vector_data	(	const Communicator &	comm,
		const MapToVectors &	queries,
		RequestContainer &	reqs,
		GatherFunctor &	gather_data,
		ActionFunctor &	act_on_data,
		const datum *	example
	)

Send query vectors, receive and answer them with vectors of data, then act on those answers.

The data map is indexed by processor ids as keys, and for each processor id in the map there should be a vector of query ids to send.

Query data which is received from other processors will be operated on by gather_data(processor_id_type pid, const std::vector<id> & ids, std::vector<datum> & data)

Answer data which is received from other processors will be operated on by act_on_data(processor_id_type pid, const std::vector<id> & ids, const std::vector<datum> & data);

The example pointer may be null; it merely needs to be of the correct type. It's just here because function overloading in C++ is easy, whereas SFINAE is hard and partial template specialization of functions is impossible.

No guarantee about operation ordering is made - this function will attempt to act on data in the order in which it is received.

All receives and actions are completed before this function returns.

Not all sends may have yet completed. The supplied container of Request objects, req, has more requests inserted, one for each of the data sends. These requests must be waited on before the data map is deleted.

Definition at line 472 of file parallel_sync.h.

References libMesh::Parallel::Communicator::get_unique_tag(), push_parallel_vector_data(), query, libMesh::Parallel::Communicator::rank(), libMesh::Parallel::Communicator::receive(), libMesh::Parallel::Communicator::send(), libMesh::Parallel::Communicator::size(), wait(), and waitany().

Referenced by libMesh::DofMap::allgather_recursive_constraints(), libMesh::MeshCommunication::assign_global_indices(), libMesh::Partitioner::assign_partitioning(), libMesh::MeshCommunication::find_global_indices(), libMesh::DofMap::gather_constraints(), libMesh::DistributedVector< T >::localize(), pull_parallel_vector_data(), libMesh::DistributedMesh::renumber_dof_objects(), libMesh::Partitioner::set_node_processor_ids(), libMesh::DofMap::set_nonlocal_dof_objects(), sync_dofobject_data_by_id(), sync_dofobject_data_by_xyz(), sync_element_data_by_parent_id(), sync_node_data_by_element_id_once(), and libMesh::XdrIO::write_serialized_connectivity().

{
  typedef typename MapToVectors::mapped_type query_type;

  std::map<processor_id_type, std::vector<datum> >
    response_data, received_data;
  std::vector<Request> response_reqs;

  StandardType<datum> datatype;

  // We'll grab a tag so we can overlap request sends and receives
  // without confusing one for the other
  MessageTag tag = comm.get_unique_tag(105);

  auto gather_functor =
    [&comm, &gather_data, &response_data, &response_reqs, &datatype, &tag]
    (processor_id_type pid, query_type query)
    {
      gather_data(pid, query, response_data[pid]);
      libmesh_assert_equal_to(query.size(), response_data[pid].size());

      // Just act on data later if the user requested a send-to-self
      if (pid != comm.rank())
        {
          Request sendreq;
          comm.send(pid, response_data[pid], datatype, sendreq, tag);
          response_reqs.push_back(sendreq);
        }
    };

  push_parallel_vector_data (comm, queries, reqs, gather_functor);

  // Every outgoing query should now have an incoming response.
  // Post all of the receives, non-blocking
  std::vector<Request> receive_reqs;
  std::vector<processor_id_type> receive_procids;
  for (auto & querypair : queries)
    {
      processor_id_type proc_id = querypair.first;
      libmesh_assert_less(proc_id, comm.size());

      if (proc_id == comm.rank())
        {
          libmesh_assert(queries.count(proc_id));
          libmesh_assert_equal_to(queries.at(proc_id).size(),
                                  response_data.at(proc_id).size());
          act_on_data(proc_id, queries.at(proc_id), response_data.at(proc_id));
        }
      else
        {
          auto & querydata = querypair.second;
          Request req;
          auto & incoming_data = received_data[proc_id];
          incoming_data.resize(querydata.size());
          comm.receive(proc_id, incoming_data, datatype, req, tag);
          receive_reqs.push_back(req);
          receive_procids.push_back(proc_id);
        }
    }

  while(receive_reqs.size())
    {
      std::size_t completed = waitany(receive_reqs);
      processor_id_type proc_id = receive_procids[completed];
      receive_reqs.erase(receive_reqs.begin() + completed);
      receive_procids.erase(receive_procids.begin() + completed);

      libmesh_assert(queries.count(proc_id));
      libmesh_assert_equal_to(queries.at(proc_id).size(),
                              received_data[proc_id].size());
      act_on_data(proc_id, queries.at(proc_id), received_data[proc_id]);
      received_data.erase(proc_id);
    }

  wait(response_reqs);
}

pull_parallel_vector_data() [2/3]

template<typename datum , typename MapToVectors , typename GatherFunctor , typename ActionFunctor >

void libMesh::Parallel::pull_parallel_vector_data	(	const Communicator &	comm,
		const MapToVectors &	queries,
		GatherFunctor &	gather_data,
		ActionFunctor &	act_on_data,
		const datum *	example
	)

Send query vectors, receive and answer them with vectors of data, then act on those answers.

The data map is indexed by processor ids as keys, and for each processor id in the map there should be a vector of query ids to send.

Query data which is received from other processors will be operated on by gather_data(processor_id_type pid, const std::vector<id> & ids, std::vector<datum> & data)

Answer data which is received from other processors will be operated on by act_on_data(processor_id_type pid, const std::vector<id> & ids, const std::vector<datum> & data);

The example pointer may be null; it merely needs to be of the correct type. It's just here because function overloading in C++ is easy, whereas SFINAE is hard and partial template specialization of functions is impossible.

No guarantee about operation ordering is made - this function will attempt to act on data in the order in which it is received.

All communication and actions are complete when this function returns.

Definition at line 559 of file parallel_sync.h.

References pull_parallel_vector_data(), and wait().

{
  std::vector<Request> requests;

  pull_parallel_vector_data(comm, queries, requests, gather_data,
                            act_on_data, example);

  wait(requests);
}

pull_parallel_vector_data() [3/3]

template<typename datum , typename A , typename MapToVectors , typename RequestContainer , typename GatherFunctor , typename ActionFunctor >

void libMesh::Parallel::pull_parallel_vector_data	(	const Communicator &	comm,
		const MapToVectors &	queries,
		RequestContainer &	reqs,
		GatherFunctor &	gather_data,
		ActionFunctor &	act_on_data,
		const std::vector< datum, A > *	example
	)

Definition at line 580 of file parallel_sync.h.

References any_source, libMesh::Parallel::Communicator::get_unique_tag(), libMesh::Parallel::Communicator::probe(), push_parallel_vector_data(), query, libMesh::Parallel::Communicator::rank(), libMesh::Parallel::Communicator::receive(), libMesh::Parallel::Communicator::send(), and wait().

{
  typedef typename MapToVectors::mapped_type query_type;

  std::map<processor_id_type, std::vector<std::vector<datum,A>>>
    response_data;
  std::vector<Request> response_reqs;

  // We'll grab a tag so we can overlap request sends and receives
  // without confusing one for the other
  MessageTag tag = comm.get_unique_tag(105);

  auto gather_functor =
    [&comm, &gather_data, &act_on_data,
     &response_data, &response_reqs, &tag]
    (processor_id_type pid, query_type query)
    {
      gather_data(pid, query, response_data[pid]);
      libmesh_assert_equal_to(query.size(),
                              response_data[pid].size());

      // Just act on data if the user requested a send-to-self
      if (pid == comm.rank())
        {
          act_on_data(pid, query, response_data[pid]);
        }
      else
        {
          Request sendreq;
          comm.send(pid, response_data[pid], sendreq, tag);
          response_reqs.push_back(sendreq);
        }
    };

  push_parallel_vector_data (comm, queries, reqs, gather_functor);

  // Every outgoing query should now have an incoming response.
  //
  // Post all of the receives.
  //
  // Use blocking API here since we can't use the pre-sized
  // non-blocking APIs with this data type.
  //
  // FIXME - implement Derek's API from #1684, switch to that!
  std::vector<Request> receive_reqs;
  std::vector<processor_id_type> receive_procids;
  for (std::size_t i = 0,
       n_queries = queries.size() - queries.count(comm.rank());
       i != n_queries; ++i)
    {
      Status stat(comm.probe(any_source, tag));
      const processor_id_type
        proc_id = cast_int<processor_id_type>(stat.source());

      std::vector<std::vector<datum,A>> received_data;
      comm.receive(proc_id, received_data, tag);

      libmesh_assert(queries.count(proc_id));
      auto & querydata = queries.at(proc_id);
      libmesh_assert_equal_to(querydata.size(), received_data.size());
      act_on_data(proc_id, querydata, received_data);
    }

  wait(response_reqs);
}

push_parallel_vector_data() [1/6]

template<typename MapToVectors , typename RequestContainer , typename ActionFunctor >

void libMesh::Parallel::push_parallel_vector_data	(	const Communicator &	comm,
		const MapToVectors &	data,
		RequestContainer &	reqs,
		ActionFunctor &	act_on_data
	)

Send and receive and act on vectors of data.

The data map is indexed by processor ids as keys, and for each processor id in the map there should be a vector of data to send.

Data which is received from other processors will be operated on by act_on_data(processor_id_type pid, const std::vector<datum> & data)

No guarantee about operation ordering is made - this function will attempt to act on data in the order in which it is received.

All receives and actions are completed before this function returns.

Not all sends may have yet completed. The supplied container of Request objects, req, has more requests inserted, one for each of the data sends. These requests must be waited on before the data map is deleted.

Definition at line 239 of file parallel_sync.h.

References libMesh::Parallel::Communicator::alltoall(), data, libMesh::Parallel::Communicator::get_unique_tag(), libMesh::Parallel::Communicator::rank(), libMesh::Parallel::Communicator::receive(), libMesh::Parallel::Communicator::send(), libMesh::Parallel::Communicator::size(), and waitany().

Referenced by libMesh::DofMap::allgather_recursive_constraints(), libMesh::Parallel::Sort< KeyType, IdxType >::communicate_bins(), libMesh::MeshTools::correct_node_proc_ids(), pull_parallel_vector_data(), push_parallel_vector_data(), and libMesh::DofMap::scatter_constraints().

{
  // This function must be run on all processors at once
  libmesh_parallel_only(comm);

  processor_id_type num_procs = comm.size();

  // Size of vectors to send to each procesor
  std::vector<std::size_t> will_send_to(num_procs, 0);
  processor_id_type num_sends = 0;
  for (auto & datapair : data)
    {
      // Don't try to send anywhere that doesn't exist
      libmesh_assert_less(datapair.first, num_procs);

      // Don't give us empty vectors to send
      libmesh_assert_greater(datapair.second.size(), 0);

      will_send_to[datapair.first] = datapair.second.size();
      num_sends++;
    }

  // Tell everyone about where everyone will send to
  comm.alltoall(will_send_to);

  // will_send_to now represents who we'll receive from
  // give it a good name
  auto & will_receive_from = will_send_to;

  // This function only works for "flat" data that we can pre-size
  // receive buffers for: a map to vectors-of-standard-types, not e.g.
  // vectors-of-vectors.
  //
  // Trying to instantiate a StandardType<T> gives us a compiler error
  // where otherwise we would have had a runtime error.
  //
  // Creating a StandardType<T> manually also saves our APIs from
  // having to do a bunch of automatic creations later.
  //
  // This object will be free'd before all non-blocking communications
  // complete, but the MPI standard for MPI_Type_free specifies "Any
  // communication that is currently using this datatype will
  // complete normally." so we're cool.
  typedef decltype(data.begin()->second.front()) ref_type;
  typedef typename std::remove_reference<ref_type>::type nonref_type;
  StandardType<typename std::remove_const<nonref_type>::type> datatype;

  // We'll grab a tag so we can overlap request sends and receives
  // without confusing one for the other
  MessageTag tag = comm.get_unique_tag(1225);

  MapToVectors received_data;

  // Post all of the sends, non-blocking
  for (auto & datapair : data)
    {
      processor_id_type destid = datapair.first;
      libmesh_assert_less(destid, num_procs);
      auto & datum = datapair.second;

      // Just act on data if the user requested a send-to-self
      if (destid == comm.rank())
        act_on_data(destid, datum);
      else
        {
          Request sendreq;
          comm.send(destid, datum, datatype, sendreq, tag);
          reqs.insert(reqs.end(), sendreq);
        }
    }

  // Post all of the receives, non-blocking
  std::vector<Request> receive_reqs;
  std::vector<processor_id_type> receive_procids;
  for (processor_id_type proc_id = 0; proc_id < num_procs; proc_id++)
    if (will_receive_from[proc_id] && proc_id != comm.rank())
      {
        Request req;
        auto & incoming_data = received_data[proc_id];
        incoming_data.resize(will_receive_from[proc_id]);
        comm.receive(proc_id, incoming_data, datatype, req, tag);
        receive_reqs.push_back(req);
        receive_procids.push_back(proc_id);
      }

  while(receive_reqs.size())
    {
      std::size_t completed = waitany(receive_reqs);
      processor_id_type proc_id = receive_procids[completed];
      receive_reqs.erase(receive_reqs.begin() + completed);
      receive_procids.erase(receive_procids.begin() + completed);

      act_on_data(proc_id, received_data[proc_id]);
      received_data.erase(proc_id);
    }
}

push_parallel_vector_data() [2/6]

template<typename MapToVectors , typename ActionFunctor >

void libMesh::Parallel::push_parallel_vector_data	(	const Communicator &	comm,
		const MapToVectors &	data,
		ActionFunctor &	act_on_data
	)

Send and receive and act on vectors of data.

The data map is indexed by processor ids as keys, and for each processor id in the map there should be a vector of data to send.

Data which is received from other processors will be operated on by act_on_data(processor_id_type pid, const std::vector<datum> & data);

No guarantee about operation ordering is made - this function will attempt to act on data in the order in which it is received.

All communication and actions are complete when this function returns.

Definition at line 435 of file parallel_sync.h.

References data, push_parallel_vector_data(), and wait().

{
  std::vector<Request> requests;

  push_parallel_vector_data(comm, data, requests, act_on_data);

  wait(requests);
}

push_parallel_vector_data() [3/6]

template<template< typename, typename, typename ... > class MapType, typename KeyType , typename ValueType , typename A1 , typename A2 , typename ... ExtraTypes, typename RequestContainer , typename ActionFunctor >

void libMesh::Parallel::push_parallel_vector_data	(	const Communicator &	comm,
		const MapType< processor_id_type, std::vector< std::vector< ValueType, A1 >, A2 >, ExtraTypes... > &	data,
		RequestContainer &	reqs,
		ActionFunctor &	act_on_data
	)

Definition at line 348 of file parallel_sync.h.

References libMesh::Parallel::Communicator::alltoall(), any_source, data, libMesh::Parallel::Communicator::get_unique_tag(), libMesh::Parallel::Communicator::probe(), libMesh::Parallel::Communicator::rank(), libMesh::Parallel::Communicator::receive(), libMesh::Parallel::Communicator::send(), and libMesh::Parallel::Communicator::size().

{
  // This function must be run on all processors at once
  libmesh_parallel_only(comm);

  processor_id_type num_procs = comm.size();

  // Size of vectors to send to each procesor
  std::vector<std::size_t> will_send_to(num_procs, 0);
  processor_id_type num_sends = 0;
  for (auto & datapair : data)
    {
      // Don't try to send anywhere that doesn't exist
      libmesh_assert_less(datapair.first, num_procs);

      // Don't give us empty vectors to send
      libmesh_assert_greater(datapair.second.size(), 0);

      will_send_to[datapair.first] = datapair.second.size();
      num_sends++;
    }

  // Tell everyone about where everyone will send to
  comm.alltoall(will_send_to);

  // will_send_to now represents who we'll receive from
  // give it a good name
  auto & will_receive_from = will_send_to;

  processor_id_type n_receives = 0;
  for (processor_id_type proc_id = 0; proc_id < num_procs; proc_id++)
    if (will_receive_from[proc_id])
      n_receives++;

  // We'll construct a datatype once for repeated use
  StandardType<ValueType> datatype;

  // We'll grab a tag so we can overlap request sends and receives
  // without confusing one for the other
  MessageTag tag = comm.get_unique_tag(1225);

  // Post all of the sends, non-blocking
  for (auto & datapair : data)
    {
      processor_id_type destid = datapair.first;
      libmesh_assert_less(destid, num_procs);
      auto & datum = datapair.second;

      // Just act on data if the user requested a send-to-self
      if (destid == comm.rank())
        {
          act_on_data(destid, datum);
          n_receives--;
        }
      else
        {
          Request sendreq;
          comm.send(destid, datum, datatype, sendreq, tag);
          reqs.insert(reqs.end(), sendreq);
        }
    }

  // Post all of the receives.
  //
  // Use blocking API here since we can't use the pre-sized
  // non-blocking APIs with this data type.
  //
  // FIXME - implement Derek's API from #1684, switch to that!
  for (processor_id_type i = 0; i != n_receives; ++i)
    {
      Status stat(comm.probe(any_source, tag));
      const processor_id_type
        proc_id = cast_int<processor_id_type>(stat.source());

      std::vector<std::vector<ValueType,A1>,A2> received_data;
      comm.receive(proc_id, received_data, datatype, tag);
      act_on_data(proc_id, received_data);
    }
}

push_parallel_vector_data() [4/6]

template<template< typename, typename, typename ... > class MapType, typename KeyType , typename ValueType , typename A1 , typename A2 , typename ... ExtraTypes, typename ActionFunctor >

void libMesh::Parallel::push_parallel_vector_data	(	const Communicator &	comm,
		const MapType< processor_id_type, std::vector< std::vector< ValueType, A1 >, A2 >, ExtraTypes... > &	data,
		ActionFunctor &	act_on_data
	)

Definition at line 454 of file parallel_sync.h.

References data, push_parallel_vector_data(), and wait().

{
  std::vector<Request> requests;

  push_parallel_vector_data(comm, data, requests, act_on_data);

  wait(requests);
}

push_parallel_vector_data() [5/6]

template<template< typename, typename, typename ... > class MapType, typename ValueType , typename A1 , typename A2 , typename ... ExtraTypes, typename RequestContainer , typename ActionFunctor >

void libMesh::Parallel::push_parallel_vector_data	(	const Communicator &	comm,
		const MapType< processor_id_type, std::vector< std::vector< ValueType, A1 >, A2 >, ExtraTypes... > &	data,
		RequestContainer &	reqs,
		ActionFunctor &	act_on_data
	)

Definition at line 348 of file parallel_sync.h.

References libMesh::Parallel::Communicator::alltoall(), any_source, data, libMesh::Parallel::Communicator::get_unique_tag(), libMesh::Parallel::Communicator::probe(), libMesh::Parallel::Communicator::rank(), libMesh::Parallel::Communicator::receive(), libMesh::Parallel::Communicator::send(), and libMesh::Parallel::Communicator::size().

{
  // This function must be run on all processors at once
  libmesh_parallel_only(comm);

  processor_id_type num_procs = comm.size();

  // Size of vectors to send to each procesor
  std::vector<std::size_t> will_send_to(num_procs, 0);
  processor_id_type num_sends = 0;
  for (auto & datapair : data)
    {
      // Don't try to send anywhere that doesn't exist
      libmesh_assert_less(datapair.first, num_procs);

      // Don't give us empty vectors to send
      libmesh_assert_greater(datapair.second.size(), 0);

      will_send_to[datapair.first] = datapair.second.size();
      num_sends++;
    }

  // Tell everyone about where everyone will send to
  comm.alltoall(will_send_to);

  // will_send_to now represents who we'll receive from
  // give it a good name
  auto & will_receive_from = will_send_to;

  processor_id_type n_receives = 0;
  for (processor_id_type proc_id = 0; proc_id < num_procs; proc_id++)
    if (will_receive_from[proc_id])
      n_receives++;

  // We'll construct a datatype once for repeated use
  StandardType<ValueType> datatype;

  // We'll grab a tag so we can overlap request sends and receives
  // without confusing one for the other
  MessageTag tag = comm.get_unique_tag(1225);

  // Post all of the sends, non-blocking
  for (auto & datapair : data)
    {
      processor_id_type destid = datapair.first;
      libmesh_assert_less(destid, num_procs);
      auto & datum = datapair.second;

      // Just act on data if the user requested a send-to-self
      if (destid == comm.rank())
        {
          act_on_data(destid, datum);
          n_receives--;
        }
      else
        {
          Request sendreq;
          comm.send(destid, datum, datatype, sendreq, tag);
          reqs.insert(reqs.end(), sendreq);
        }
    }

  // Post all of the receives.
  //
  // Use blocking API here since we can't use the pre-sized
  // non-blocking APIs with this data type.
  //
  // FIXME - implement Derek's API from #1684, switch to that!
  for (processor_id_type i = 0; i != n_receives; ++i)
    {
      Status stat(comm.probe(any_source, tag));
      const processor_id_type
        proc_id = cast_int<processor_id_type>(stat.source());

      std::vector<std::vector<ValueType,A1>,A2> received_data;
      comm.receive(proc_id, received_data, datatype, tag);
      act_on_data(proc_id, received_data);
    }
}

push_parallel_vector_data() [6/6]

template<template< typename, typename, typename ... > class MapType, typename ValueType , typename A1 , typename A2 , typename ... ExtraTypes, typename ActionFunctor >

void libMesh::Parallel::push_parallel_vector_data	(	const Communicator &	comm,
		const MapType< processor_id_type, std::vector< std::vector< ValueType, A1 >, A2 >, ExtraTypes... > &	data,
		ActionFunctor &	act_on_data
	)

Definition at line 454 of file parallel_sync.h.

References data, push_parallel_vector_data(), and wait().

{
  std::vector<Request> requests;

  push_parallel_vector_data(comm, data, requests, act_on_data);

  wait(requests);
}

sync_dofobject_data_by_id() [1/2]

template<typename Iterator , typename SyncFunctor >

void libMesh::Parallel::sync_dofobject_data_by_id	(	const Communicator &	comm,
		const Iterator &	range_begin,
		const Iterator &	range_end,
		SyncFunctor &	sync
	)

Request data about a range of ghost dofobjects uniquely identified by their id. Fulfill requests with sync.gather_data(const std::vector<dof_id_type> & ids, std::vector<sync::datum> & data), by resizing and setting the values of the data vector. Respond to fulfillment with sync.act_on_data(const std::vector<dof_id_type> & ids, std::vector<sync::datum> & data) The user must define Parallel::StandardType<sync::datum> if sync::datum isn't a built-in type.

Definition at line 336 of file parallel_ghost_sync.h.

Referenced by libMesh::Partitioner::_find_global_index_by_pid_map(), libMesh::MeshTools::correct_node_proc_ids(), libMesh::MeshRefinement::make_coarsening_compatible(), libMesh::MeshCommunication::make_elems_parallel_consistent(), libMesh::MeshRefinement::make_flags_parallel_consistent(), libMesh::MeshCommunication::make_node_unique_ids_parallel_consistent(), libMesh::MeshCommunication::make_p_levels_parallel_consistent(), libMesh::FEMSystem::mesh_position_set(), libMesh::LaplaceMeshSmoother::smooth(), and libMesh::MeshRefinement::uniformly_coarsen().

{
  sync_dofobject_data_by_id(comm, range_begin, range_end, SyncEverything(), sync);
}

sync_dofobject_data_by_id() [2/2]

template<typename Iterator , typename DofObjectCheckFunctor , typename SyncFunctor >

void libMesh::Parallel::sync_dofobject_data_by_id	(	const Communicator &	comm,
		const Iterator &	range_begin,
		const Iterator &	range_end,
		const DofObjectCheckFunctor &	dofobj_check,
		SyncFunctor &	sync
	)

Request data about a range of ghost dofobjects uniquely identified by their id.

Elements within the range can be excluded from the request by returning false from dofobj_check(dof_object)

Definition at line 347 of file parallel_ghost_sync.h.

References data, libMesh::DofObject::id(), libMesh::DofObject::invalid_processor_id, libMesh::DofObject::processor_id(), pull_parallel_vector_data(), libMesh::Parallel::Communicator::rank(), and libMesh::Parallel::Communicator::size().

{
  // This function must be run on all processors at once
  libmesh_parallel_only(comm);

  // Count the objects to ask each processor about
  std::vector<dof_id_type>
    ghost_objects_from_proc(comm.size(), 0);

  for (Iterator it = range_begin; it != range_end; ++it)
    {
      DofObject * obj = *it;
      libmesh_assert (obj);

      // We may want to pass Elem* or Node* to the check function, not
      // just DofObject*
      if (!dofobj_check(*it))
        continue;

      processor_id_type obj_procid = obj->processor_id();
      if (obj_procid != DofObject::invalid_processor_id)
        ghost_objects_from_proc[obj_procid]++;
    }

  // Request sets to send to each processor
  std::map<processor_id_type, std::vector<dof_id_type>>
    requested_objs_id;

  // We know how many objects live on each processor, so reserve()
  // space for each.
  for (processor_id_type p=0; p != comm.size(); ++p)
    if (p != comm.rank() && ghost_objects_from_proc[p])
      requested_objs_id[p].reserve(ghost_objects_from_proc[p]);

  for (Iterator it = range_begin; it != range_end; ++it)
    {
      DofObject * obj = *it;

      if (!dofobj_check(*it))
        continue;

      processor_id_type obj_procid = obj->processor_id();
      if (obj_procid == comm.rank() ||
          obj_procid == DofObject::invalid_processor_id)
        continue;

      requested_objs_id[obj_procid].push_back(obj->id());
    }

  auto gather_functor =
    [&sync]
    (processor_id_type, const std::vector<dof_id_type> & ids,
     std::vector<typename SyncFunctor::datum> & data)
    {
      sync.gather_data(ids, data);
    };

  auto action_functor =
    [&sync]
    (processor_id_type, const std::vector<dof_id_type> & ids,
     const std::vector<typename SyncFunctor::datum> & data)
    {
      // Let the user process the results
      sync.act_on_data(ids, data);
    };

  // Trade requests with other processors
  typename SyncFunctor::datum * ex = nullptr;
  pull_parallel_vector_data
    (comm, requested_objs_id, gather_functor, action_functor, ex);
}

sync_dofobject_data_by_xyz() [1/2]

template<typename Iterator , typename DofObjType , typename SyncFunctor >

void libMesh::Parallel::sync_dofobject_data_by_xyz	(	const Communicator &	comm,
		const Iterator &	range_begin,
		const Iterator &	range_end,
		LocationMap< DofObjType > *	location_map,
		SyncFunctor &	sync
	)

Request data about a range of ghost nodes uniquely identified by their xyz location or a range of active ghost elements uniquely identified by their centroids' xyz location. Fulfill requests with sync.gather_data(const std::vector<unsigned int> & ids, std::vector<sync::datum> & data), by resizing and setting the values of the data vector. Respond to fulfillment with sync.act_on_data(const std::vector<unsigned int> & ids, std::vector<sync::datum> & data) The user must define Parallel::StandardType<sync::datum> if sync::datum isn't a built-in type. The user-provided location_map will be used and left unchanged if it is provided, or filled and cleared if it is empty.

sync_dofobject_data_by_xyz() [2/2]

template<typename Iterator , typename DofObjType , typename SyncFunctor >

void libMesh::Parallel::sync_dofobject_data_by_xyz	(	const Communicator &	comm,
		const Iterator &	range_begin,
		const Iterator &	range_end,
		LocationMap< DofObjType > &	location_map,
		SyncFunctor &	sync
	)

Definition at line 232 of file parallel_ghost_sync.h.

References data, libMesh::LocationMap< T >::empty(), libMesh::LocationMap< T >::find(), libMesh::DofObject::invalid_processor_id, libMesh::Parallel::Communicator::max(), libMesh::LocationMap< T >::point_of(), pull_parallel_vector_data(), libMesh::Parallel::Communicator::rank(), and libMesh::Parallel::Communicator::size().

{
  // This function must be run on all processors at once
  libmesh_parallel_only(comm);

  // We need a valid location_map
#ifdef DEBUG
  bool need_map_update = (range_begin != range_end && location_map.empty());
  comm.max(need_map_update);
  libmesh_assert(!need_map_update);
#endif

  // Count the objects to ask each processor about
  std::vector<dof_id_type>
    ghost_objects_from_proc(comm.size(), 0);

  for (Iterator it = range_begin; it != range_end; ++it)
    {
      DofObjType * obj = *it;
      libmesh_assert (obj);
      processor_id_type obj_procid = obj->processor_id();
      if (obj_procid != DofObject::invalid_processor_id)
        ghost_objects_from_proc[obj_procid]++;
    }

  // Request sets to send to each processor
  std::map<processor_id_type, std::vector<Point>>
    requested_objs_pt;
  // Corresponding ids to keep track of
  std::map<processor_id_type, std::vector<dof_id_type>>
    requested_objs_id;

  // We know how many objects live on each processor, so reserve()
  // space for each.
  for (processor_id_type p=0; p != comm.size(); ++p)
    if (p != comm.rank() && ghost_objects_from_proc[p])
      {
        requested_objs_pt[p].reserve(ghost_objects_from_proc[p]);
        requested_objs_id[p].reserve(ghost_objects_from_proc[p]);
      }

  for (Iterator it = range_begin; it != range_end; ++it)
    {
      DofObjType * obj = *it;
      processor_id_type obj_procid = obj->processor_id();
      if (obj_procid == comm.rank() ||
          obj_procid == DofObject::invalid_processor_id)
        continue;

      Point p = location_map.point_of(*obj);
      requested_objs_pt[obj_procid].push_back(p);
      requested_objs_id[obj_procid].push_back(obj->id());
    }

  auto gather_functor =
    [&location_map, &sync]
    (processor_id_type /*pid*/, const std::vector<Point> & pts,
     std::vector<typename SyncFunctor::datum> & data)
    {
      // Find the local id of each requested object
      std::size_t query_size = pts.size();
      std::vector<dof_id_type> query_id(query_size);
      for (std::size_t i=0; i != query_size; ++i)
        {
          Point pt = pts[i];

          // Look for this object in the multimap
          DofObjType * obj = location_map.find(pt);

          // We'd better find every object we're asked for
          libmesh_assert (obj);

          // Return the object's correct processor id,
          // and our (correct if it's local) id for it.
          query_id[i] = obj->id();
        }

      // Gather whatever data the user wants
      sync.gather_data(query_id, data);
    };

  auto action_functor =
    [&sync, &requested_objs_id]
    (processor_id_type pid, const std::vector<Point> &,
     const std::vector<typename SyncFunctor::datum> & data)
    {
      // Let the user process the results
      sync.act_on_data(requested_objs_id[pid], data);
    };

  // Trade requests with other processors
  typename SyncFunctor::datum * ex = nullptr;
  pull_parallel_vector_data
    (comm, requested_objs_pt, gather_functor, action_functor, ex);
}

sync_element_data_by_parent_id()

template<typename Iterator , typename SyncFunctor >

void libMesh::Parallel::sync_element_data_by_parent_id	(	MeshBase &	mesh,
		const Iterator &	range_begin,
		const Iterator &	range_end,
		SyncFunctor &	sync
	)

Request data about a range of ghost elements uniquely identified by their parent id and which child they are. Fulfill requests with sync.gather_data(const std::vector<unsigned int> & ids, std::vector<sync::datum> & data), by resizing and setting the values of the data vector. Respond to fulfillment with sync.act_on_data(const std::vector<unsigned int> & ids, std::vector<sync::datum> & data) The user must define Parallel::StandardType<sync::datum> if sync::datum isn't a built-in type.

Definition at line 429 of file parallel_ghost_sync.h.

References libMesh::Elem::active(), libMesh::Elem::child_ptr(), data, libMesh::Elem::has_children(), libMesh::DofObject::id(), libMesh::DofObject::invalid_processor_id, mesh, libMesh::Elem::parent(), libMesh::DofObject::processor_id(), pull_parallel_vector_data(), and libMesh::Elem::which_child_am_i().

Referenced by libMesh::MeshCommunication::make_elems_parallel_consistent().

{
  const Communicator & comm (mesh.comm());

  // This function must be run on all processors at once
  libmesh_parallel_only(comm);

  // Count the objects to ask each processor about
  std::vector<dof_id_type>
    ghost_objects_from_proc(comm.size(), 0);

  for (Iterator it = range_begin; it != range_end; ++it)
    {
      Elem * elem = *it;
      processor_id_type obj_procid = elem->processor_id();
      if (obj_procid == comm.rank() ||
          obj_procid == DofObject::invalid_processor_id)
        continue;
      const Elem * parent = elem->parent();
      if (!parent || !elem->active())
        continue;

      ghost_objects_from_proc[obj_procid]++;
    }

  // Request sets to send to each processor
  std::map<processor_id_type, std::vector<dof_id_type>>
    requested_objs_id;
  std::map<processor_id_type, std::vector<std::pair<dof_id_type,unsigned char>>>
    requested_objs_parent_id_child_num;

  // We know how many objects live on each processor, so reserve()
  // space for each.
  for (processor_id_type p=0; p != comm.size(); ++p)
    if (p != comm.rank() && ghost_objects_from_proc[p])
      {
        requested_objs_id[p].reserve(ghost_objects_from_proc[p]);
        requested_objs_parent_id_child_num[p].reserve(ghost_objects_from_proc[p]);
      }

  for (Iterator it = range_begin; it != range_end; ++it)
    {
      Elem * elem = *it;
      processor_id_type obj_procid = elem->processor_id();
      if (obj_procid == comm.rank() ||
          obj_procid == DofObject::invalid_processor_id)
        continue;
      const Elem * parent = elem->parent();
      if (!parent || !elem->active())
        continue;

      requested_objs_id[obj_procid].push_back(elem->id());
      requested_objs_parent_id_child_num[obj_procid].push_back
        (std::make_pair
         (parent->id(),
          cast_int<unsigned char>
            (parent->which_child_am_i(elem))));
    }

  auto gather_functor =
    [&mesh, &sync]
    (processor_id_type,
     const std::vector<std::pair<dof_id_type, unsigned char>> & parent_id_child_num,
     std::vector<typename SyncFunctor::datum> & data)
    {
      // Find the id of each requested element
      std::size_t query_size = parent_id_child_num.size();
      std::vector<dof_id_type> query_id(query_size);
      for (std::size_t i=0; i != query_size; ++i)
        {
          Elem & parent = mesh.elem_ref(parent_id_child_num[i].first);
          libmesh_assert(parent.has_children());
          Elem * child = parent.child_ptr(parent_id_child_num[i].second);
          libmesh_assert(child);
          libmesh_assert(child->active());
          query_id[i] = child->id();
        }

      // Gather whatever data the user wants
      sync.gather_data(query_id, data);
    };

  auto action_functor =
    [&sync, &requested_objs_id]
    (processor_id_type pid,
     const std::vector<std::pair<dof_id_type, unsigned char>> &,
     const std::vector<typename SyncFunctor::datum> & data)
    {
      // Let the user process the results
      sync.act_on_data(requested_objs_id[pid], data);
    };

  // Trade requests with other processors
  typename SyncFunctor::datum * ex = nullptr;
  pull_parallel_vector_data
    (comm, requested_objs_parent_id_child_num, gather_functor,
     action_functor, ex);
}

sync_node_data_by_element_id()

template<typename ElemCheckFunctor , typename NodeCheckFunctor , typename SyncFunctor >

void libMesh::Parallel::sync_node_data_by_element_id	(	MeshBase &	mesh,
		const MeshBase::const_element_iterator &	range_begin,
		const MeshBase::const_element_iterator &	range_end,
		const ElemCheckFunctor &	elem_check,
		const NodeCheckFunctor &	node_check,
		SyncFunctor &	sync
	)

Synchronize data about a range of ghost nodes uniquely identified by an element id and local node id, iterating until data is completely in sync and futher synchronization passes cause no changes.

Imagine a vertex surrounded by triangles, each on a different processor, with a ghosting policy that include only face neighbors and not point neighbors. Then the only way for authoritative information to trickle out from that vertex is by being passed along, one neighbor at a time, to processors who mostly don't even see the node's true owner!

Data for all nodes connected to elements in the given range of element iterators will be requested.

Elements can be further excluded from the request by returning false from element_check(elem)

Nodes can be further excluded from the request by returning false from node_check(elem, local_node_num)

Fulfill requests with sync.gather_data(const std::vector<unsigned int> & ids, std::vector<sync::datum> & data), by resizing and setting the values of the data vector. Respond to fulfillment with bool sync.act_on_data(const std::vector<unsigned int> & ids, std::vector<sync::datum> & data) and return true iff the response changed any data.

The user must define Parallel::StandardType<sync::datum> if sync::datum isn't a built-in type.

Definition at line 752 of file parallel_ghost_sync.h.

References mesh, and sync_node_data_by_element_id_once().

Referenced by libMesh::MeshCommunication::make_new_node_proc_ids_parallel_consistent(), libMesh::MeshCommunication::make_node_ids_parallel_consistent(), and libMesh::MeshCommunication::make_node_proc_ids_parallel_consistent().

{
  // This function must be run on all processors at once
  libmesh_parallel_only(mesh.comm());

  bool need_sync = false;

  do
    {
      need_sync =
        sync_node_data_by_element_id_once
          (mesh, range_begin, range_end, elem_check, node_check,
           sync);
    } while (need_sync);
}

sync_node_data_by_element_id_once()

template<typename ElemCheckFunctor , typename NodeCheckFunctor , typename SyncFunctor >

bool libMesh::Parallel::sync_node_data_by_element_id_once	(	MeshBase &	mesh,
		const MeshBase::const_element_iterator &	range_begin,
		const MeshBase::const_element_iterator &	range_end,
		const ElemCheckFunctor &	elem_check,
		const NodeCheckFunctor &	node_check,
		SyncFunctor &	sync
	)

Synchronize data about a range of ghost nodes uniquely identified by an element id and local node id, assuming a single synchronization pass is necessary.

Data for all nodes connected to elements in the given range of element iterators will be requested.

Elements can be further excluded from the request by returning false from element_check(elem)

Nodes can be further excluded from the request by returning false from node_check(elem, local_node_num)

Fulfill requests with sync.gather_data(const std::vector<unsigned int> & ids, std::vector<sync::datum> & data), by resizing and setting the values of the data vector. Respond to fulfillment with bool sync.act_on_data(const std::vector<unsigned int> & ids, std::vector<sync::datum> & data) and return true iff the response changed any data.

The user must define Parallel::StandardType<sync::datum> if sync::datum isn't a built-in type.

This method returns true iff the sync pass changed any data on any processor.

Definition at line 546 of file parallel_ghost_sync.h.

References libMesh::as_range(), data, libMesh::DofObject::id(), libMesh::DofObject::invalid_id, libMesh::DofObject::invalid_processor_id, mesh, libMesh::Elem::n_nodes(), libMesh::Elem::node_ref(), libMesh::DofObject::processor_id(), and pull_parallel_vector_data().

Referenced by libMesh::MeshCommunication::make_new_node_proc_ids_parallel_consistent(), and sync_node_data_by_element_id().

{
  const Communicator & comm (mesh.comm());

  // Count the objects to ask each processor about
  std::vector<dof_id_type>
    ghost_objects_from_proc(comm.size(), 0);

  for (const auto & elem : as_range(range_begin, range_end))
    {
      libmesh_assert (elem);

      if (!elem_check(elem))
        continue;

      const processor_id_type proc_id = elem->processor_id();

      bool i_have_elem =
        (proc_id == comm.rank() ||
         proc_id == DofObject::invalid_processor_id);

      if (elem->active() && i_have_elem)
        continue;

      for (auto n : elem->node_index_range())
        {
          if (!node_check(elem, n))
            continue;

          const processor_id_type node_pid =
            elem->node_ref(n).processor_id();

          if (i_have_elem && (node_pid == comm.rank()))
            continue;

          if (i_have_elem)
            {
              libmesh_assert_not_equal_to
                (node_pid, DofObject::invalid_processor_id);
              ghost_objects_from_proc[node_pid]++;
            }
          else
            {
              const processor_id_type request_pid =
                (node_pid == DofObject::invalid_processor_id) ?
                 proc_id : node_pid;
              ghost_objects_from_proc[request_pid]++;
            }
        }
    }

  // Now repeat that iteration, filling request sets this time.

  // Request sets to send to each processor
  std::map<processor_id_type, std::vector<std::pair<dof_id_type, unsigned char>>>
    requested_objs_elem_id_node_num;

  // We know how many objects live on each processor, so reserve()
  // space for each.
  for (processor_id_type p=0; p != comm.size(); ++p)
    if (p != comm.rank() && ghost_objects_from_proc[p])
      {
        requested_objs_elem_id_node_num[p].reserve(ghost_objects_from_proc[p]);
      }

  for (const auto & elem : as_range(range_begin, range_end))
    {
      libmesh_assert (elem);

      if (!elem_check(elem))
        continue;

      const processor_id_type proc_id = elem->processor_id();

      bool i_have_elem =
        (proc_id == comm.rank() ||
         proc_id == DofObject::invalid_processor_id);

      if (elem->active() && i_have_elem)
        continue;

      const dof_id_type elem_id = elem->id();

      for (auto n : elem->node_index_range())
        {
          if (!node_check(elem, n))
            continue;

          const Node & node = elem->node_ref(n);
          const processor_id_type node_pid = node.processor_id();

          if (i_have_elem && (node_pid == comm.rank()))
            continue;

          if (i_have_elem)
            {
              libmesh_assert_not_equal_to
                (node_pid, DofObject::invalid_processor_id);
              requested_objs_elem_id_node_num[node_pid].push_back
                (std::make_pair
                 (elem_id,
                  cast_int<unsigned char>(n)));
            }
          else
            {
              const processor_id_type request_pid =
                (node_pid == DofObject::invalid_processor_id) ?
                 proc_id : node_pid;
              requested_objs_elem_id_node_num[request_pid].push_back
                (std::make_pair
                 (elem_id,
                  cast_int<unsigned char>(n)));
            }
        }
    }

  auto gather_functor =
    [&mesh, &sync]
    (processor_id_type,
     const std::vector<std::pair<dof_id_type, unsigned char>> & elem_id_node_num,
     std::vector<typename SyncFunctor::datum> & data)
    {
      // Find the id of each requested element
      std::size_t request_size = elem_id_node_num.size();
      std::vector<dof_id_type> query_id(request_size);
      for (std::size_t i=0; i != request_size; ++i)
        {
          // We might now get queries about remote elements, in which
          // case we'll have to ignore them and wait for the query
          // answer to filter to the querier via another source.
          const Elem * elem = mesh.query_elem_ptr(elem_id_node_num[i].first);

          if (elem)
            {
              const unsigned int n = elem_id_node_num[i].second;
              libmesh_assert_less (n, elem->n_nodes());

              const Node & node = elem->node_ref(n);

              // This isn't a safe assertion in the case where we're
              // syncing processor ids
              // libmesh_assert_equal_to (node->processor_id(), comm.rank());

              query_id[i] = node.id();
            }
          else
            query_id[i] = DofObject::invalid_id;
        }

      // Gather whatever data the user wants
      sync.gather_data(query_id, data);
    };

  bool data_changed = false;

  auto action_functor =
    [&sync, &mesh, &requested_objs_elem_id_node_num, &data_changed]
    (processor_id_type pid,
     const std::vector<std::pair<dof_id_type, unsigned char>> &,
     const std::vector<typename SyncFunctor::datum> & data)
    {
      const auto & elem_id_node_num =
        requested_objs_elem_id_node_num[pid];

      const std::size_t data_size = data.size();

      libmesh_assert_equal_to(elem_id_node_num.size(), data_size);

      std::vector<dof_id_type> requested_objs_id(data.size());

      for (auto i : IntRange<std::size_t>(0,data_size))
        {
          const Elem & elem = mesh.elem_ref(elem_id_node_num[i].first);
          const Node & node = elem.node_ref(elem_id_node_num[i].second);
          requested_objs_id[i] = node.id();
        }

      // Let the user process the results.  If any of the results
      // were different than what the user expected, then we may
      // need to sync again just in case this processor has to
      // pass on the changes to yet another processor.
      if (sync.act_on_data(requested_objs_id, data))
        data_changed = true;
    };

  // Trade requests with other processors
  typename SyncFunctor::datum * ex = nullptr;
  pull_parallel_vector_data
    (comm, requested_objs_elem_id_node_num, gather_functor,
     action_functor, ex);

  comm.max(data_changed);

  return data_changed;
}

unpack_range() [1/2]

template<typename Context , typename buffertype , typename OutputIter , typename T >

void libMesh::Parallel::unpack_range ( const typename std::vector< buffertype > &  buffer, Context *  context, OutputIter  out, const T *  output_type  )

inline

Decode a range of potentially-variable-size objects from a data array.

Referenced by libMesh::Parallel::Communicator::allgather_packed_range(), libMesh::Parallel::Communicator::broadcast_packed_range(), libMesh::Parallel::Communicator::gather_packed_range(), libMesh::Parallel::Communicator::receive_packed_range(), libMesh::Parallel::PostWaitUnpackBuffer< Container, Context, OutputIter, T >::run(), and libMesh::Parallel::Communicator::send_receive_packed_range().

unpack_range() [2/2]

template<typename Context , typename buffertype , typename OutputIter , typename T >

void libMesh::Parallel::unpack_range ( const std::vector< buffertype > &  buffer, Context *  context, OutputIter  out_iter, const T *    )

inline

Helper function for range unpacking

Definition at line 194 of file packing.h.

References libMesh::Parallel::Packing< T >::packed_size(), and libMesh::Parallel::Packing< T >::unpack().

{
  // Loop through the buffer and unpack each object, returning the
  // object pointer via the output iterator
  typename std::vector<buffertype>::const_iterator
    next_object_start = buffer.begin();

  while (next_object_start < buffer.end())
    {
      *out_iter++ = Parallel::Packing<T>::unpack(next_object_start, context);
      next_object_start +=
        Parallel::Packing<T>::packed_size(next_object_start);
    }

  // We should have used up the exact amount of data in the buffer
  libmesh_assert (next_object_start == buffer.end());
}

wait() [1/2]

Status libMesh::Parallel::wait ( Request &  r )

inline

Wait for a non-blocking send or receive to finish

Definition at line 129 of file request.h.

References libMesh::Parallel::Request::wait().

{ return r.wait(); }

wait() [2/2]

void libMesh::Parallel::wait

(

std::vector< Request > &

r

)

Wait for all non-blocking operations to finish

Definition at line 213 of file request.C.

Referenced by libMesh::DofMap::allgather_recursive_constraints(), libMesh::BoundaryInfo::build_node_list_from_side_list(), libMesh::MeshTools::correct_node_proc_ids(), libMesh::MeshRefinement::make_coarsening_compatible(), libMesh::SparsityPattern::Build::parallel_sync(), pull_parallel_vector_data(), push_parallel_vector_data(), libMesh::Nemesis_IO::read(), libMesh::System::read_serialized_blocked_dof_objects(), libMesh::DofMap::scatter_constraints(), libMesh::MeshRefinement::uniformly_coarsen(), libMesh::System::write_serialized_blocked_dof_objects(), and libMesh::XdrIO::write_serialized_nodes().

{
  for (std::size_t i=0; i<r.size(); i++)
    r[i].wait();
}

waitany()

std::size_t libMesh::Parallel::waitany

(

std::vector< Request > &

r

)

Wait for at least one non-blocking operation to finish. Return the index of the request which completed.

Definition at line 219 of file request.C.

References libMesh::Parallel::Request::_prior_request, libMesh::Parallel::Request::get(), libMesh::MeshTools::Subdivision::next, and libMesh::Parallel::Request::post_wait_work.

Referenced by pull_parallel_vector_data(), and push_parallel_vector_data().

{
  libmesh_assert(!r.empty());

  int index = 0;
  int r_size = cast_int<int>(r.size());
  std::vector<request> raw(r_size);
  for (int i=0; i != r_size; ++i)
    {
      Request * root = &r[i];
      // If we have prior requests, we need to complete the first one
      // first
      while (root->_prior_request.get())
        root = root->_prior_request.get();
      raw[i] = *root->get();
    }

  bool only_priors_completed = false;

  Request * next;

  do
    {
#ifdef LIBMESH_HAVE_MPI
      libmesh_call_mpi
        (MPI_Waitany(r_size, raw.data(), &index, MPI_STATUS_IGNORE));
#endif

      Request * completed = &r[index];
      next = completed;

      // If we completed a prior request, we're not really done yet,
      // so find the next in that line to try again.
      while (completed->_prior_request.get())
        {
          only_priors_completed = true;
          next = completed;
          completed = completed->_prior_request.get();
        }

      // Do any post-wait work for the completed request
      if (completed->post_wait_work)
        for (auto & item : completed->post_wait_work->first)
          {
            // The user should never try to give us non-existent work or try
            // to wait() twice.
            libmesh_assert (item);
            item->run();
            delete item;
            item = nullptr;
          }

      next->_prior_request.reset(nullptr);
      raw[index] = *next->get();

    } while(only_priors_completed);

  return index;
}

Variable Documentation

any_source

const unsigned int libMesh::Parallel::any_source

Initial value:

=
  static_cast<unsigned int>(MPI_ANY_SOURCE)

Processor id meaning "Accept from any source"

Definition at line 70 of file communicator.h.

Referenced by libMesh::BoundaryInfo::build_node_list_from_side_list(), libMesh::MeshRefinement::make_coarsening_compatible(), pull_parallel_vector_data(), push_parallel_vector_data(), libMesh::Nemesis_IO::read(), libMesh::System::read_serialized_blocked_dof_objects(), libMesh::MeshRefinement::uniformly_coarsen(), and libMesh::System::write_serialized_blocked_dof_objects().

any_tag

const MessageTag libMesh::Parallel::any_tag = MessageTag(MPI_ANY_TAG)

Default message tag ids

Definition at line 115 of file message_tag.h.

no_tag

const MessageTag libMesh::Parallel::no_tag = MessageTag(0)

Definition at line 120 of file message_tag.h.