distributed_mesh.C
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2018 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 
20 // Local includes
21 #include "libmesh/boundary_info.h"
23 #include "libmesh/elem.h"
26 #include "libmesh/parallel.h"
27 #include "libmesh/parallel_sync.h"
29 
30 namespace libMesh
31 {
32 
33 // ------------------------------------------------------------
34 // DistributedMesh class member functions
36  unsigned char d) :
37  UnstructuredMesh (comm_in,d), _is_serial(true),
38  _is_serial_on_proc_0(true),
39  _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
40  _next_free_local_node_id(this->processor_id()),
41  _next_free_local_elem_id(this->processor_id()),
42  _next_free_unpartitioned_node_id(this->n_processors()),
43  _next_free_unpartitioned_elem_id(this->n_processors())
44 #ifdef LIBMESH_ENABLE_UNIQUE_ID
45  , _next_unpartitioned_unique_id(this->n_processors())
46 #endif
47 {
48 #ifdef LIBMESH_ENABLE_UNIQUE_ID
49  _next_unique_id = this->processor_id();
50 #endif
51 
52  // FIXME: give parmetis the communicator!
53  _partitioner = libmesh_make_unique<ParmetisPartitioner>();
54 }
55 
56 
57 
59 {
60  this->clear(); // Free nodes and elements
61 }
62 
63 
64 // This might be specialized later, but right now it's just here to
65 // make sure the compiler doesn't give us a default (non-deep) copy
66 // constructor instead.
68  UnstructuredMesh (other_mesh), _is_serial(other_mesh._is_serial),
69  _is_serial_on_proc_0(other_mesh._is_serial_on_proc_0),
70  _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
71  _next_free_local_node_id(this->processor_id()),
72  _next_free_local_elem_id(this->processor_id()),
73  _next_free_unpartitioned_node_id(this->n_processors()),
74  _next_free_unpartitioned_elem_id(this->n_processors())
75 {
76  this->copy_nodes_and_elements(other_mesh);
77  _n_nodes = other_mesh.n_nodes();
78  _n_elem = other_mesh.n_elem();
79  _max_node_id = other_mesh.max_node_id();
80  _max_elem_id = other_mesh.max_elem_id();
82  other_mesh._next_free_local_node_id;
84  other_mesh._next_free_local_elem_id;
89 #ifdef LIBMESH_ENABLE_UNIQUE_ID
91  other_mesh._next_unique_id;
94 #endif
95  this->get_boundary_info() = other_mesh.get_boundary_info();
96 
97  // Need to copy extra_ghost_elems
98  for (auto & elem : other_mesh._extra_ghost_elems)
99  _extra_ghost_elems.insert(this->elem_ptr(elem->id()));
100 }
101 
102 
103 
105  UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
106  _is_serial_on_proc_0(other_mesh.is_serial()),
107  _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
108  _next_free_local_node_id(this->processor_id()),
109  _next_free_local_elem_id(this->processor_id()),
110  _next_free_unpartitioned_node_id(this->n_processors()),
111  _next_free_unpartitioned_elem_id(this->n_processors())
112 {
113  this->copy_nodes_and_elements(other_mesh);
114  this->get_boundary_info() = other_mesh.get_boundary_info();
115 
116 #ifdef LIBMESH_ENABLE_UNIQUE_ID
118 #endif
120 }
121 
122 
123 // We use cached values for these so they can be called
124 // from one processor without bothering the rest, but
125 // we may need to update those caches before doing a full
126 // renumbering
128 {
129  // This function must be run on all processors at once
130  parallel_object_only();
131 
132  _n_elem = this->parallel_n_elem();
133  _n_nodes = this->parallel_n_nodes();
136 
139  ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
140  (this->n_processors() + 1) + this->n_processors();
143  ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
144  (this->n_processors() + 1) + this->processor_id();
145 
148  ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
149  (this->n_processors() + 1) + this->n_processors();
152  ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
153  (this->n_processors() + 1) + this->processor_id();
154 
155 #ifdef LIBMESH_ENABLE_UNIQUE_ID
158  ((_next_unique_id-1) / (this->n_processors() + 1) + 1) *
159  (this->n_processors() + 1) + this->n_processors();
161  ((_next_unique_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
162  (this->n_processors() + 1) + this->processor_id();
163 #endif
164 }
165 
166 
167 // Or in debug mode we may want to test the uncached values without
168 // changing the cache
170 {
171  // This function must be run on all processors at once
172  parallel_object_only();
173 
174  dof_id_type n_local = this->n_local_elem();
175  this->comm().sum(n_local);
176  n_local += this->n_unpartitioned_elem();
177  return n_local;
178 }
179 
180 
181 
183 {
184  // This function must be run on all processors at once
185  parallel_object_only();
186 
187  dof_id_type max_local = 0;
188 
190  rit = _elements.rbegin();
191 
193  rend = _elements.rend();
194 
195  // Look for the maximum element id. Search backwards through
196  // elements so we can break out early. Beware of nullptr entries that
197  // haven't yet been cleared from _elements.
198  for (; rit != rend; ++rit)
199  if (rit->second)
200  {
201  libmesh_assert_equal_to(rit->second->id(), rit->first);
202  max_local = rit->first + 1;
203  break;
204  }
205 
206  this->comm().max(max_local);
207  return max_local;
208 }
209 
210 
211 
212 #ifdef LIBMESH_ENABLE_UNIQUE_ID
214 {
215  // This function must be run on all processors at once
216  parallel_object_only();
217 
220  this->comm().max(max_local);
221  return max_local;
222 }
223 #endif
224 
225 
226 
228 {
229  // This function must be run on all processors at once
230  parallel_object_only();
231 
232  dof_id_type n_local = this->n_local_nodes();
233  this->comm().sum(n_local);
234  n_local += this->n_unpartitioned_nodes();
235  return n_local;
236 }
237 
238 
239 
241 {
242  // This function must be run on all processors at once
243  parallel_object_only();
244 
245  dof_id_type max_local = 0;
246 
248  rit = _nodes.rbegin();
249 
251  rend = _nodes.rend();
252 
253  // Look for the maximum element id. Search backwards through
254  // elements so we can break out early. Beware of nullptr entries that
255  // haven't yet been cleared from _elements.
256  for (; rit != rend; ++rit)
257  if (rit->second)
258  {
259  libmesh_assert_equal_to(rit->second->id(), rit->first);
260  max_local = rit->first + 1;
261  break;
262  }
263 
264  this->comm().max(max_local);
265  return max_local;
266 }
267 
268 
269 
270 const Point & DistributedMesh::point (const dof_id_type i) const
271 {
272  return this->node_ref(i);
273 }
274 
275 
276 
278 {
279  libmesh_assert(_nodes[i]);
280  libmesh_assert_equal_to (_nodes[i]->id(), i);
281 
282  return _nodes[i];
283 }
284 
285 
286 
287 
289 {
290  libmesh_assert(_nodes[i]);
291  libmesh_assert_equal_to (_nodes[i]->id(), i);
292 
293  return _nodes[i];
294 }
295 
296 
297 
298 
300 {
301  std::map<dof_id_type, Node *>::const_iterator it = _nodes.find(i);
302  if (it != _nodes.end().it)
303  {
304  const Node * n = it->second;
305  libmesh_assert (!n || n->id() == i);
306  return n;
307  }
308 
309  return nullptr;
310 }
311 
312 
313 
314 
316 {
317  std::map<dof_id_type, Node *>::const_iterator it = _nodes.find(i);
318  if (it != _nodes.end().it)
319  {
320  Node * n = it->second;
321  libmesh_assert (!n || n->id() == i);
322  return n;
323  }
324 
325  return nullptr;
326 }
327 
328 
329 
330 
332 {
333  libmesh_assert(_elements[i]);
334  libmesh_assert_equal_to (_elements[i]->id(), i);
335 
336  return _elements[i];
337 }
338 
339 
340 
341 
343 {
344  libmesh_assert(_elements[i]);
345  libmesh_assert_equal_to (_elements[i]->id(), i);
346 
347  return _elements[i];
348 }
349 
350 
351 
352 
354 {
355  std::map<dof_id_type, Elem *>::const_iterator it = _elements.find(i);
356  if (it != _elements.end().it)
357  {
358  const Elem * e = it->second;
359  libmesh_assert (!e || e->id() == i);
360  return e;
361  }
362 
363  return nullptr;
364 }
365 
366 
367 
368 
370 {
371  std::map<dof_id_type, Elem *>::const_iterator it = _elements.find(i);
372  if (it != _elements.end().it)
373  {
374  Elem * e = _elements[i];
375  libmesh_assert (!e || e->id() == i);
376  return e;
377  }
378 
379  return nullptr;
380 }
381 
382 
383 
384 
386 {
387  // Don't try to add nullptrs!
388  libmesh_assert(e);
389 
390  // Trying to add an existing element is a no-op
391  if (e->valid_id() && _elements[e->id()] == e)
392  return e;
393 
394  const processor_id_type elem_procid = e->processor_id();
395 
396  if (!e->valid_id())
397  {
398  // We should only be creating new ids past the end of the range
399  // of existing ids
400  libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
401  _max_elem_id);
402  libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
403 
404  // Use the unpartitioned ids for unpartitioned elems, and
405  // temporarily for ghost elems
407  if (elem_procid == this->processor_id())
408  next_id = &_next_free_local_elem_id;
409  e->set_id (*next_id);
410  }
411 
412  {
413  // Advance next_ids up high enough that each is pointing to an
414  // unused id and any subsequent increments will still point us
415  // to unused ids
417  static_cast<dof_id_type>(e->id()+1));
418 
421  ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
422  (this->n_processors() + 1) + this->n_processors();
425  ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
426  (this->n_processors() + 1) + this->processor_id();
427 
428 #ifndef NDEBUG
429  // We need a const mapvector so we don't inadvertently create
430  // nullptr entries when testing for non-nullptr ones
431  const mapvector<Elem *, dof_id_type> & const_elements = _elements;
432 #endif
433  libmesh_assert(!const_elements[_next_free_unpartitioned_elem_id]);
434  libmesh_assert(!const_elements[_next_free_local_elem_id]);
435  }
436 
437  // Don't try to overwrite existing elems
438  libmesh_assert (!_elements[e->id()]);
439 
440  _elements[e->id()] = e;
441 
442  // Try to make the cached elem data more accurate
443  if (elem_procid == this->processor_id() ||
444  elem_procid == DofObject::invalid_processor_id)
445  _n_elem++;
446 
447 #ifdef LIBMESH_ENABLE_UNIQUE_ID
448  if (!e->valid_unique_id())
449  {
450  if (processor_id() == e->processor_id())
451  {
453  _next_unique_id += this->n_processors() + 1;
454  }
455  else
456  {
459  }
460  }
461 #endif
462 
463  // Unpartitioned elems should be added on every processor
464  // And shouldn't be added in the same batch as ghost elems
465  // But we might be just adding on processor 0 to
466  // broadcast later
467  // #ifdef DEBUG
468  // if (elem_procid == DofObject::invalid_processor_id)
469  // {
470  // dof_id_type elem_id = e->id();
471  // this->comm().max(elem_id);
472  // libmesh_assert_equal_to (elem_id, e->id());
473  // }
474  // #endif
475 
476  return e;
477 }
478 
479 
480 
482 {
483  if (_elements[e->id()])
484  this->delete_elem(_elements[e->id()]);
485 
486 #ifdef LIBMESH_ENABLE_UNIQUE_ID
487  if (!e->valid_unique_id())
488  {
489  if (processor_id() == e->processor_id())
490  {
492  _next_unique_id += this->n_processors() + 1;
493  }
494  else
495  {
498  }
499  }
500 #endif
501 
502  // Try to make the cached elem data more accurate
503  processor_id_type elem_procid = e->processor_id();
504  if (elem_procid == this->processor_id() ||
505  elem_procid == DofObject::invalid_processor_id)
506  _n_elem++;
507 
508  _elements[e->id()] = e;
509 
510  return e;
511 }
512 
513 
514 
516 {
517  libmesh_assert (e);
518 
519  // Try to make the cached elem data more accurate
520  processor_id_type elem_procid = e->processor_id();
521  if (elem_procid == this->processor_id() ||
522  elem_procid == DofObject::invalid_processor_id)
523  _n_elem--;
524 
525  // Delete the element from the BoundaryInfo object
526  this->get_boundary_info().remove(e);
527 
528  // But not yet from the container; we might invalidate
529  // an iterator that way!
530 
531  //_elements.erase(e->id());
532 
533  // Instead, we set it to nullptr for now
534 
535  _elements[e->id()] = nullptr;
536 
537  // delete the element
538  delete e;
539 }
540 
541 
542 
544  const dof_id_type new_id)
545 {
546  Elem * el = _elements[old_id];
547  libmesh_assert (el);
548  libmesh_assert_equal_to (el->id(), old_id);
549 
550  el->set_id(new_id);
551  libmesh_assert (!_elements[new_id]);
552  _elements[new_id] = el;
553  _elements.erase(old_id);
554 }
555 
556 
557 
559  const dof_id_type id,
560  const processor_id_type proc_id)
561 {
562  auto n_it = _nodes.find(id);
563  if (n_it != _nodes.end().it)
564  {
565  Node * n = n_it->second;
566  libmesh_assert (n);
567  libmesh_assert_equal_to (n->id(), id);
568 
569  *n = p;
570  n->processor_id() = proc_id;
571 
572  return n;
573  }
574 
575  Node * n = Node::build(p, id).release();
576  n->processor_id() = proc_id;
577 
578  return DistributedMesh::add_node(n);
579 }
580 
581 
583 {
584  // This had better be a node in our mesh
585  libmesh_assert(_nodes[n.id()] == &n);
586 
587  _nodes[n.id()] = nullptr;
588 
590  n.processor_id() = this->processor_id();
591 
592  this->add_node(&n);
593 }
594 
595 
597 {
598  // Don't try to add nullptrs!
599  libmesh_assert(n);
600 
601  // Trying to add an existing node is a no-op
602  if (n->valid_id() && _nodes[n->id()] == n)
603  return n;
604 
605  const processor_id_type node_procid = n->processor_id();
606 
607  if (!n->valid_id())
608  {
609  // We should only be creating new ids past the end of the range
610  // of existing ids
611  libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
612  _max_node_id);
613  libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
614 
615  // Use the unpartitioned ids for unpartitioned nodes,
616  // and temporarily for ghost nodes
618  if (node_procid == this->processor_id())
619  next_id = &_next_free_local_node_id;
620  n->set_id (*next_id);
621  }
622 
623  {
624  // Advance next_ids up high enough that each is pointing to an
625  // unused id and any subsequent increments will still point us
626  // to unused ids
628  static_cast<dof_id_type>(n->id()+1));
629 
632  ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
633  (this->n_processors() + 1) + this->n_processors();
636  ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
637  (this->n_processors() + 1) + this->processor_id();
638 
639 #ifndef NDEBUG
640  // We need a const mapvector so we don't inadvertently create
641  // nullptr entries when testing for non-nullptr ones
642  const mapvector<Node *,dof_id_type> & const_nodes = _nodes;
643 #endif
644  libmesh_assert(!const_nodes[_next_free_unpartitioned_node_id]);
645  libmesh_assert(!const_nodes[_next_free_local_node_id]);
646  }
647 
648  // Don't try to overwrite existing nodes
649  libmesh_assert (!_nodes[n->id()]);
650 
651  _nodes[n->id()] = n;
652 
653  // Try to make the cached node data more accurate
654  if (node_procid == this->processor_id() ||
655  node_procid == DofObject::invalid_processor_id)
656  _n_nodes++;
657 
658 #ifdef LIBMESH_ENABLE_UNIQUE_ID
659  if (!n->valid_unique_id())
660  {
661  if (processor_id() == n->processor_id())
662  {
664  _next_unique_id += this->n_processors();
665  }
666  else
667  {
670  }
671  }
672 #endif
673 
674 
675  // Unpartitioned nodes should be added on every processor
676  // And shouldn't be added in the same batch as ghost nodes
677  // But we might be just adding on processor 0 to
678  // broadcast later
679  // #ifdef DEBUG
680  // if (node_procid == DofObject::invalid_processor_id)
681  // {
682  // dof_id_type node_id = n->id();
683  // this->comm().max(node_id);
684  // libmesh_assert_equal_to (node_id, n->id());
685  // }
686  // #endif
687 
688  return n;
689 }
690 
691 
692 
694 {
695  return DistributedMesh::add_node(n);
696 }
697 
698 
699 
701 {
702  libmesh_assert(n);
703  libmesh_assert(_nodes[n->id()]);
704 
705  // Try to make the cached elem data more accurate
706  processor_id_type node_procid = n->processor_id();
707  if (node_procid == this->processor_id() ||
708  node_procid == DofObject::invalid_processor_id)
709  _n_nodes--;
710 
711  // Delete the node from the BoundaryInfo object
712  this->get_boundary_info().remove(n);
713 
714  // But not yet from the container; we might invalidate
715  // an iterator that way!
716 
717  //_nodes.erase(n->id());
718 
719  // Instead, we set it to nullptr for now
720 
721  _nodes[n->id()] = nullptr;
722 
723  // delete the node
724  delete n;
725 }
726 
727 
728 
730  const dof_id_type new_id)
731 {
732  Node * nd = _nodes[old_id];
733  libmesh_assert (nd);
734  libmesh_assert_equal_to (nd->id(), old_id);
735 
736  nd->set_id(new_id);
737 
738  // If we have nodes shipped to this processor for NodeConstraints
739  // use, then those nodes will exist in _nodes, but may not be
740  // locatable via a TopologyMap due to the insufficiency of elements
741  // connecting to them. If local refinement then wants to create a
742  // *new* node in the same location, it will initially get a temporary
743  // id, and then make_node_ids_parallel_consistent() will try to move
744  // it to the canonical id. We need to account for this case to
745  // avoid false positives and memory leaks.
746 #ifdef LIBMESH_ENABLE_NODE_CONSTRAINTS
747  if (_nodes[new_id])
748  {
749  libmesh_assert_equal_to (*(Point *)_nodes[new_id],
750  *(Point *)_nodes[old_id]);
751  _nodes.erase(new_id);
752  }
753 #else
754  // If we aren't shipping nodes for NodeConstraints, there should be
755  // no reason for renumbering one node onto another.
756  libmesh_assert (!_nodes[new_id]);
757 #endif
758  _nodes[new_id] = nd;
759  _nodes.erase(old_id);
760 }
761 
762 
763 
765 {
766  // Call parent clear function
767  MeshBase::clear();
768 
769  // Clear our elements and nodes
770  // There is no need to remove the elements from
771  // the BoundaryInfo data structure since we
772  // already cleared it.
773  for (auto & elem : _elements)
774  delete elem;
775 
776  // clear the nodes data structure
777  // There is no need to remove the nodes from
778  // the BoundaryInfo data structure since we
779  // already cleared it.
780  for (auto & node : _nodes)
781  delete node;
782 
783  _elements.clear();
784  _nodes.clear();
785 
786  // We're no longer distributed if we were before
787  _is_serial = true;
788  _is_serial_on_proc_0 = true;
789 
790  // Correct our caches
791  _n_nodes = 0;
792  _n_elem = 0;
793  _max_node_id = 0;
794  _max_elem_id = 0;
799 }
800 
801 
802 
804 {
805  // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
806  if (!this->is_serial())
807  {
808  // Construct a MeshCommunication object to actually redistribute the nodes
809  // and elements according to the partitioner, and then to re-gather the neighbors.
811  mc.redistribute(*this);
812 
814 
815  // We probably had valid neighbors previously, so that a quality
816  // new partitioning could be found, but we might not have valid
817  // neighbor links on the newly-redistributed elements
818  this->find_neighbors();
819 
820  // Is this necessary? If we are called from prepare_for_use(), this will be called
821  // anyway... but users can always call partition directly, in which case we do need
822  // to call delete_remote_elements()...
823  //
824  // Regardless of whether it's necessary, it isn't safe. We
825  // haven't communicated new node processor_ids yet, and we can't
826  // delete nodes until we do.
827  // this->delete_remote_elements();
828  }
829 }
830 
831 
832 
834 {
835  // this->recalculate_n_partitions();
836 
837  // Partitioning changes our numbers of unpartitioned objects
839 }
840 
841 
842 
843 template <typename T>
845 {
846  // This function must be run on all processors at once
847  parallel_object_only();
848 
849  const dof_id_type pmax_node_id = this->parallel_max_node_id();
850  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
851  const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
852 
853  for (dof_id_type i=0; i != pmax_id; ++i)
854  {
855  T * obj = objects[i]; // Returns nullptr if there's no map entry
856 
857  // Local lookups by id should return the requested object
858  libmesh_assert(!obj || obj->id() == i);
859 
860  // All processors with an object should agree on id
861 #ifndef NDEBUG
862  const dof_id_type dofid = obj && obj->valid_id() ?
863  obj->id() : DofObject::invalid_id;
864  libmesh_assert(this->comm().semiverify(obj ? &dofid : nullptr));
865 #endif
866 
867  // All processors with an object should agree on processor id
868  const dof_id_type procid = obj && obj->valid_processor_id() ?
869  obj->processor_id() : DofObject::invalid_processor_id;
870  libmesh_assert(this->comm().semiverify(obj ? &procid : nullptr));
871 
872  dof_id_type min_procid = procid;
873  this->comm().min(min_procid);
874 
875  // Either:
876  // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != nullptr)
877  // or
878  // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
879 
880  // Original assert logic
881  // libmesh_assert (min_procid != this->processor_id() || obj);
882 
883  // More human-understandable logic...
884  libmesh_assert (
885  ((min_procid == this->processor_id()) && obj)
886  ||
887  (min_procid != this->processor_id())
888  );
889 
890 #if defined(LIBMESH_ENABLE_UNIQUE_ID) && !defined(NDEBUG)
891  // All processors with an object should agree on unique id
892  const unique_id_type uniqueid = obj ? obj->unique_id() : 0;
893  libmesh_assert(this->comm().semiverify(obj ? &uniqueid : nullptr));
894 #endif
895  }
896 }
897 
898 
899 
901 {
904 }
905 
906 
907 
909 {
910 #ifndef NDEBUG
911  // This function must be run on all processors at once
912  parallel_object_only();
913 
914  dof_id_type pmax_elem_id = this->parallel_max_elem_id();
915 
916  for (dof_id_type i=0; i != pmax_elem_id; ++i)
917  {
918  Elem * el = _elements[i]; // Returns nullptr if there's no map entry
919 
920  unsigned int p_level = el ? (el->p_level()) : libMesh::invalid_uint;
921 
922  // All processors with an active element should agree on p level
923  libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_level : nullptr));
924  }
925 #endif
926 }
927 
928 
929 
930 
932 {
933 #if defined(LIBMESH_ENABLE_AMR) && !defined(NDEBUG)
934  // This function must be run on all processors at once
935  parallel_object_only();
936 
937  dof_id_type pmax_elem_id = this->parallel_max_elem_id();
938 
939  for (dof_id_type i=0; i != pmax_elem_id; ++i)
940  {
941  Elem * el = _elements[i]; // Returns nullptr if there's no map entry
942 
943  unsigned int refinement_flag = el ?
944  static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
945  unsigned int p_refinement_flag = el ?
946  static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
947 
948  libmesh_assert(this->comm().semiverify(el ? &refinement_flag : nullptr));
949 
950  // p refinement flags aren't always kept correct on inactive
951  // ghost elements
952  libmesh_assert(this->comm().semiverify((el && el->active()) ? &p_refinement_flag : nullptr));
953  }
954 #endif // LIBMESH_ENABLE_AMR
955 }
956 
957 
958 
959 template <typename T>
962 {
963  // This function must be run on all processors at once
964  parallel_object_only();
965 
966  typedef typename mapvector<T *,dof_id_type>::veclike_iterator object_iterator;
967 
968  // In parallel we may not know what objects other processors have.
969  // Start by figuring out how many
970  dof_id_type unpartitioned_objects = 0;
971 
972  std::unordered_map<processor_id_type, dof_id_type>
973  ghost_objects_from_proc;
974 
975  object_iterator it = objects.begin();
976  object_iterator end = objects.end();
977 
978  while (it != end)
979  {
980  T * obj = *it;
981 
982  // Remove any nullptr container entries while we're here.
983  if (!obj)
984  it = objects.erase(it);
985  else
986  {
987  processor_id_type obj_procid = obj->processor_id();
988  if (obj_procid == DofObject::invalid_processor_id)
989  unpartitioned_objects++;
990  else
991  ghost_objects_from_proc[obj_procid]++;
992 
993  // Finally, increment the iterator
994  ++it;
995  }
996  }
997 
998  std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
999  auto this_it = ghost_objects_from_proc.find(this->processor_id());
1000  this->comm().allgather
1001  ((this_it == ghost_objects_from_proc.end()) ? 0 : this_it->second,
1002  objects_on_proc);
1003 
1004 #ifndef NDEBUG
1005  libmesh_assert(this->comm().verify(unpartitioned_objects));
1006  for (processor_id_type p=0, np=this->n_processors(); p != np; ++p)
1007  if (ghost_objects_from_proc.count(p))
1008  libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
1009  else
1010  libmesh_assert_less_equal (0, objects_on_proc[p]);
1011 #endif
1012 
1013  // We'll renumber objects in blocks by processor id
1014  std::vector<dof_id_type> first_object_on_proc(this->n_processors());
1015  for (processor_id_type i=1; i != this->n_processors(); ++i)
1016  first_object_on_proc[i] = first_object_on_proc[i-1] +
1017  objects_on_proc[i-1];
1018  dof_id_type next_id = first_object_on_proc[this->processor_id()];
1019  dof_id_type first_free_id =
1020  first_object_on_proc[this->n_processors()-1] +
1021  objects_on_proc[this->n_processors()-1] +
1022  unpartitioned_objects;
1023 
1024  // First set new local object ids and build request sets
1025  // for non-local object ids
1026 
1027  // Request sets to send to each processor
1028  std::map<processor_id_type, std::vector<dof_id_type>>
1029  requested_ids;
1030 
1031  // We know how many objects live on each processor, so reserve() space for
1032  // each.
1033  auto ghost_end = ghost_objects_from_proc.end();
1034  for (processor_id_type p=0; p != this->n_processors(); ++p)
1035  if (p != this->processor_id())
1036  {
1037  const auto p_it = ghost_objects_from_proc.find(p);
1038  if (p_it != ghost_end)
1039  requested_ids[p].reserve(p_it->second);
1040  }
1041 
1042  end = objects.end();
1043  for (it = objects.begin(); it != end; ++it)
1044  {
1045  T * obj = *it;
1046  if (obj->processor_id() == this->processor_id())
1047  obj->set_id(next_id++);
1048  else if (obj->processor_id() != DofObject::invalid_processor_id)
1049  requested_ids[obj->processor_id()].push_back(obj->id());
1050  }
1051 
1052  // Next set ghost object ids from other processors
1053 
1054  auto gather_functor =
1055  [
1056 #ifndef NDEBUG
1057  this,
1058  &first_object_on_proc,
1059  &objects_on_proc,
1060 #endif
1061  &objects]
1062  (processor_id_type, const std::vector<dof_id_type> & ids,
1063  std::vector<dof_id_type> & new_ids)
1064  {
1065  std::size_t ids_size = ids.size();
1066  new_ids.resize(ids_size);
1067 
1068  for (std::size_t i=0; i != ids_size; ++i)
1069  {
1070  T * obj = objects[ids[i]];
1071  libmesh_assert(obj);
1072  libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1073  new_ids[i] = obj->id();
1074 
1075  libmesh_assert_greater_equal (new_ids[i],
1076  first_object_on_proc[this->processor_id()]);
1077  libmesh_assert_less (new_ids[i],
1078  first_object_on_proc[this->processor_id()] +
1079  objects_on_proc[this->processor_id()]);
1080  }
1081  };
1082 
1083  auto action_functor =
1084  [
1085 #ifndef NDEBUG
1086  &first_object_on_proc,
1087  &objects_on_proc,
1088 #endif
1089  &objects]
1090  (processor_id_type libmesh_dbg_var(pid),
1091  const std::vector<dof_id_type> & ids,
1092  const std::vector<dof_id_type> & data)
1093  {
1094  // Copy the id changes we've now been informed of
1095  for (auto i : index_range(ids))
1096  {
1097  T * obj = objects[ids[i]];
1098  libmesh_assert (obj);
1099  libmesh_assert_equal_to (obj->processor_id(), pid);
1100  libmesh_assert_greater_equal (data[i],
1101  first_object_on_proc[pid]);
1102  libmesh_assert_less (data[i],
1103  first_object_on_proc[pid] +
1104  objects_on_proc[pid]);
1105  obj->set_id(data[i]);
1106  }
1107  };
1108 
1109  const dof_id_type * ex = nullptr;
1111  (this->comm(), requested_ids, gather_functor, action_functor, ex);
1112 
1113 #ifdef LIBMESH_ENABLE_UNIQUE_ID
1114  auto unique_gather_functor =
1115  [
1116 #ifndef NDEBUG
1117  this,
1118 #endif
1119  &objects]
1120  (processor_id_type, const std::vector<dof_id_type> & ids,
1121  std::vector<unique_id_type> & data)
1122  {
1123  std::size_t ids_size = ids.size();
1124  data.resize(ids_size);
1125 
1126  for (std::size_t i=0; i != ids_size; ++i)
1127  {
1128  T * obj = objects[ids[i]];
1129  libmesh_assert(obj);
1130  libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
1131  data[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
1132  }
1133  };
1134 
1135  auto unique_action_functor =
1136  [&objects]
1137  (processor_id_type libmesh_dbg_var(pid),
1138  const std::vector<dof_id_type> & ids,
1139  const std::vector<unique_id_type> & data)
1140  {
1141  for (auto i : index_range(ids))
1142  {
1143  T * obj = objects[ids[i]];
1144  libmesh_assert (obj);
1145  libmesh_assert_equal_to (obj->processor_id(), pid);
1146  if (!obj->valid_unique_id() && data[i] != DofObject::invalid_unique_id)
1147  obj->set_unique_id() = (data[i]);
1148  }
1149  };
1150 
1151  const unique_id_type * unique_ex = nullptr;
1153  (this->comm(), requested_ids, unique_gather_functor,
1154  unique_action_functor, unique_ex);
1155 #endif
1156 
1157  // Next set unpartitioned object ids
1158  next_id = 0;
1159  for (processor_id_type i=0; i != this->n_processors(); ++i)
1160  next_id += objects_on_proc[i];
1161  for (it = objects.begin(); it != end; ++it)
1162  {
1163  T * obj = *it;
1164  if (obj->processor_id() == DofObject::invalid_processor_id)
1165  obj->set_id(next_id++);
1166  }
1167 
1168  // Finally shuffle around objects so that container indices
1169  // match ids
1170  it = objects.begin();
1171  end = objects.end();
1172  while (it != end)
1173  {
1174  T * obj = *it;
1175  if (obj) // don't try shuffling already-nullptr entries
1176  {
1177  T * next = objects[obj->id()];
1178  // If we have to move this object
1179  if (next != obj)
1180  {
1181  // nullptr out its original position for now
1182  // (our shuffling may put another object there shortly)
1183  *it = nullptr;
1184 
1185  // There may already be another object with this id that
1186  // needs to be moved itself
1187  while (next)
1188  {
1189  // We shouldn't be trying to give two objects the
1190  // same id
1191  libmesh_assert_not_equal_to (next->id(), obj->id());
1192  objects[obj->id()] = obj;
1193  obj = next;
1194  next = objects[obj->id()];
1195  }
1196  objects[obj->id()] = obj;
1197  }
1198  }
1199 
1200  // Remove any container entries that were left as nullptr.
1201  if (!obj)
1202  it = objects.erase(it);
1203  else
1204  ++it;
1205  }
1206 
1207  return first_free_id;
1208 }
1209 
1210 
1212 {
1213  parallel_object_only();
1214 
1215 #ifdef DEBUG
1216  // Make sure our ids and flags are consistent
1220 #endif
1221 
1222  LOG_SCOPE("renumber_nodes_and_elements()", "DistributedMesh");
1223 
1224  std::set<dof_id_type> used_nodes;
1225 
1226  // flag the nodes we need
1227  for (auto & elem : this->element_ptr_range())
1228  for (unsigned int n=0; n != elem->n_nodes(); ++n)
1229  used_nodes.insert(elem->node_id(n));
1230 
1231  // Nodes not connected to any local elements, and nullptr node entries
1232  // in our container, are deleted
1233  {
1234  node_iterator_imp it = _nodes.begin();
1235  node_iterator_imp end = _nodes.end();
1236 
1237  while (it != end)
1238  {
1239  Node * nd = *it;
1240  if (!nd)
1241  it = _nodes.erase(it);
1242  else if (!used_nodes.count(nd->id()))
1243  {
1244  // remove any boundary information associated with
1245  // this node
1246  this->get_boundary_info().remove (nd);
1247 
1248  // delete the node
1249  delete nd;
1250 
1251  it = _nodes.erase(it);
1252  }
1253  else
1254  ++it;
1255  }
1256  }
1257 
1259  {
1260  this->update_parallel_id_counts();
1261  return;
1262  }
1263 
1264  // Finally renumber all the elements
1265  _n_elem = this->renumber_dof_objects (this->_elements);
1266 
1267  // and all the remaining nodes
1268  _n_nodes = this->renumber_dof_objects (this->_nodes);
1269 
1270  // And figure out what IDs we should use when adding new nodes and
1271  // new elements
1272  this->update_parallel_id_counts();
1273 
1274  // Make sure our caches are up to date and our
1275  // DofObjects are well packed
1276 #ifdef DEBUG
1277  libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1278  libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1279  const dof_id_type pmax_node_id = this->parallel_max_node_id();
1280  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1281  libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1282  libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1283  libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1284  libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1285 
1286  // Make sure our ids and flags are consistent
1289 
1290  // And make sure we've made our numbering monotonic
1292 #endif
1293 }
1294 
1295 
1296 
1298 {
1299  // We need access to iterators for the underlying containers,
1300  // not the mapvector<> reimplementations.
1303 
1304  // Nodes first
1305  for (auto & pr : nodes)
1306  if (pr.second != nullptr)
1307  pr.second->set_id() = pr.first;
1308 
1309  // Elements next
1310  for (const auto & pr : elems)
1311  if (pr.second != nullptr)
1312  pr.second->set_id() = pr.first;
1313 }
1314 
1315 
1316 
1318 {
1319  parallel_object_only();
1320 
1321  // Get local active elements first
1322  dof_id_type active_elements =
1323  static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1324  this->active_local_elements_end()));
1325  this->comm().sum(active_elements);
1326 
1327  // Then add unpartitioned active elements, which should exist on
1328  // every processor
1329  active_elements +=
1330  static_cast<dof_id_type>(std::distance
1333  return active_elements;
1334 }
1335 
1336 
1337 
1339 {
1340 #ifdef DEBUG
1341  // Make sure our neighbor links are all fine
1343 
1344  // And our child/parent links, and our flags
1346 
1347  // Make sure our ids and flags are consistent
1350 
1351  libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1352  libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1353  const dof_id_type pmax_node_id = this->parallel_max_node_id();
1354  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1355  libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1356  libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1357 #endif
1358 
1359  _is_serial = false;
1360  _is_serial_on_proc_0 = false;
1361 
1363 
1364  libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1365 
1366  // Now make sure the containers actually shrink - strip
1367  // any newly-created nullptr voids out of the element array
1370  while (e_it != e_end)
1371  if (!*e_it)
1372  e_it = _elements.erase(e_it);
1373  else
1374  ++e_it;
1375 
1378  while (n_it != n_end)
1379  if (!*n_it)
1380  n_it = _nodes.erase(n_it);
1381  else
1382  ++n_it;
1383 
1384  // We may have deleted no-longer-connected nodes or coarsened-away
1385  // elements; let's update our caches.
1386  this->update_parallel_id_counts();
1387 
1388 #ifdef DEBUG
1389  // We might not have well-packed objects if the user didn't allow us
1390  // to renumber
1391  // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1392  // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1393 
1394  // Make sure our neighbor links are all fine
1396 
1397  // And our child/parent links, and our flags
1399 
1400  // Make sure our ids and flags are consistent
1403 #endif
1404 }
1405 
1406 
1408 {
1409  // First add the elem like normal
1410  add_elem(e);
1411 
1412  // Now add it to the set that won't be deleted when we call
1413  // delete_remote_elements()
1414  _extra_ghost_elems.insert(e);
1415 }
1416 
1417 
1419 {
1420  if (_is_serial)
1421  return;
1422  MeshCommunication().allgather(*this);
1423  _is_serial = true;
1424  _is_serial_on_proc_0 = true;
1425 
1426  // Make sure our caches are up to date and our
1427  // DofObjects are well packed
1428 #ifdef DEBUG
1429  libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1430  libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1431  const dof_id_type pmax_node_id = this->parallel_max_node_id();
1432  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1433  libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1434  libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1435 
1436  // If we've disabled renumbering we can't be sure we're contiguous
1437  // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1438  // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1439 
1440  // Make sure our neighbor links are all fine
1442 
1443  // Make sure our ids and flags are consistent
1446 #endif
1447 }
1448 
1450 {
1452  return;
1453 
1454  _is_serial_on_proc_0 = true;
1455  MeshCommunication().gather(0, *this);
1456 }
1457 
1458 
1459 } // namespace libMesh
virtual void copy_nodes_and_elements(const UnstructuredMesh &other_mesh, const bool skip_find_neighbors=false, dof_id_type element_id_offset=0, dof_id_type node_id_offset=0, unique_id_type unique_id_offset=0)
RefinementState refinement_flag() const
Definition: elem.h:2638
std::map< index_t, Val > maptype
Definition: mapvector.h:42
virtual element_iterator active_local_elements_end() override
virtual void update_parallel_id_counts() override
unique_id_type & set_unique_id()
Definition: dof_object.h:685
virtual dof_id_type parallel_n_nodes() const override
bool _skip_renumber_nodes_and_elements
Definition: mesh_base.h:1431
mapvector< Elem *, dof_id_type > _elements
virtual const Elem * elem(const dof_id_type i) const
Definition: mesh_base.h:537
A geometric point in (x,y,z) space associated with a DOF.
Definition: node.h:52
void libmesh_assert_valid_parallel_object_ids(const mapvector< T *, dof_id_type > &) const
virtual unique_id_type parallel_max_unique_id() const =0
dof_id_type n_unpartitioned_nodes() const
Definition: mesh_base.h:292
virtual element_iterator active_pid_elements_begin(processor_id_type proc_id) override
const unsigned int invalid_uint
Definition: libmesh.h:245
virtual void libmesh_assert_valid_parallel_ids() const override
virtual void fix_broken_node_and_element_numbering() override
virtual const Node * query_node_ptr(const dof_id_type i) const override
virtual void renumber_elem(dof_id_type old_id, dof_id_type new_id) override
virtual dof_id_type max_node_id() const override
virtual void clear() override
void allgather(const T &send, std::vector< T, A > &recv) const
virtual void renumber_nodes_and_elements() override
RefinementState p_refinement_flag() const
Definition: elem.h:2654
void libmesh_assert_valid_refinement_tree(const MeshBase &mesh)
Definition: mesh_tools.C:2033
dof_id_type _next_free_local_node_id
void remove(const Node *node)
virtual void allgather() override
virtual void own_node(Node &n) override
virtual dof_id_type n_active_elem() const override
The base class for all geometric element types.
Definition: elem.h:100
IntRange< std::size_t > index_range(const std::vector< T > &vec)
Definition: int_range.h:104
uint8_t processor_id_type
Definition: id_types.h:99
dof_id_type n_local_nodes() const
Definition: mesh_base.h:286
const Parallel::Communicator & comm() const
unsigned int p_level() const
Definition: elem.h:2555
IterBase * end
virtual void find_neighbors(const bool reset_remote_elements=false, const bool reset_current_list=true) override
void libmesh_assert_valid_parallel_flags() const
const BoundaryInfo & get_boundary_info() const
Definition: mesh_base.h:131
virtual void add_extra_ghost_elem(Elem *e)
dof_id_type n_unpartitioned_elem() const
Definition: mesh_base.h:392
long double max(long double a, double b)
virtual unique_id_type parallel_max_unique_id() const override
dof_id_type n_local_elem() const
Definition: mesh_base.h:386
virtual void delete_remote_elements() override
void libmesh_assert_valid_parallel_p_levels() const
dof_id_type & set_id()
Definition: dof_object.h:664
virtual Elem * add_elem(Elem *e) override
unique_id_type _next_unique_id
Definition: mesh_base.h:1418
dof_id_type renumber_dof_objects(mapvector< T *, dof_id_type > &)
processor_id_type n_processors() const
virtual dof_id_type parallel_n_elem() const override
dof_id_type id() const
Definition: dof_object.h:655
mapvector< Node *, dof_id_type > _nodes
virtual Elem * insert_elem(Elem *e) override
void pull_parallel_vector_data(const Communicator &comm, const MapToVectors &queries, RequestContainer &reqs, GatherFunctor &gather_data, ActionFunctor &act_on_data, const datum *example)
virtual unsigned int n_nodes() const =0
static const unique_id_type invalid_unique_id
Definition: dof_object.h:352
static const processor_id_type invalid_processor_id
Definition: dof_object.h:358
std::set< Elem * > _extra_ghost_elems
Base class for Replicated and Distributed meshes.
virtual Node * insert_node(Node *n) override
virtual dof_id_type n_nodes() const override
void erase(index_t i)
Definition: mapvector.h:117
std::unique_ptr< Partitioner > _partitioner
Definition: mesh_base.h:1412
static const dof_id_type invalid_id
Definition: dof_object.h:347
virtual void renumber_node(dof_id_type old_id, dof_id_type new_id) override
virtual Node * add_point(const Point &p, const dof_id_type id=DofObject::invalid_id, const processor_id_type proc_id=DofObject::invalid_processor_id) override
bool valid_unique_id() const
Definition: dof_object.h:705
dof_id_type parallel_max_node_id() const
veclike_iterator end()
Definition: mapvector.h:133
void allgather(DistributedMesh &mesh) const
Mesh data structure which is distributed across all processors.
virtual void clear()
Definition: mesh_base.C:260
virtual const Point & point(const dof_id_type i) const override
dof_id_type _next_free_unpartitioned_node_id
virtual const Elem * elem_ptr(const dof_id_type i) const override
virtual const Node & node(const dof_id_type i) const
Definition: mesh_base.h:454
virtual element_iterator active_pid_elements_end(processor_id_type proc_id) override
static std::unique_ptr< Node > build(const Node &n)
Definition: node.h:304
static const unsigned int next[3]
virtual void redistribute() override
virtual element_iterator active_local_elements_begin() override
dof_id_type _next_free_unpartitioned_elem_id
bool valid_id() const
Definition: dof_object.h:697
virtual const Node * node_ptr(const dof_id_type i) const override
DistributedMesh(const Parallel::Communicator &comm_in, unsigned char dim=1)
veclike_iterator begin()
Definition: mapvector.h:125
virtual void gather_to_zero() override
virtual dof_id_type n_elem() const override
virtual Node * add_node(Node *n) override
virtual SimpleRange< element_iterator > element_ptr_range() override
virtual const Node & node_ref(const dof_id_type i) const
Definition: mesh_base.h:434
virtual void update_post_partitioning() override
virtual const Elem * query_elem_ptr(const dof_id_type i) const override
unique_id_type _next_unpartitioned_unique_id
void libmesh_assert_valid_neighbors(const MeshBase &mesh, bool assert_valid_remote_elems=true)
Definition: mesh_tools.C:2068
virtual void delete_node(Node *n) override
IterBase * data
processor_id_type processor_id() const
void gather(const processor_id_type root_id, DistributedMesh &) const
bool active() const
Definition: elem.h:2390
dof_id_type _next_free_local_elem_id
processor_id_type processor_id() const
Definition: dof_object.h:717
void delete_remote_elements(DistributedMesh &, const std::set< Elem *> &) const
virtual dof_id_type max_elem_id() const override
A geometric point in (x,y,z) space.
Definition: point.h:38
dof_id_type node_id(const unsigned int i) const
Definition: elem.h:1914
uint8_t unique_id_type
Definition: id_types.h:79
void redistribute(DistributedMesh &mesh, bool newly_coarsened_only=false) const
virtual bool is_serial() const override
dof_id_type parallel_max_elem_id() const
virtual void delete_elem(Elem *e) override
uint8_t dof_id_type
Definition: id_types.h:64
void libmesh_assert_valid_elem_ids(const MeshBase &mesh)
Definition: mesh_tools.C:1297