libMesh::LibMeshInit Class Reference

#include <libmesh.h>

Public Member Functions

 LibMeshInit (int argc, const char *const *argv, MPI_Comm COMM_WORLD_IN=MPI_COMM_WORLD)
 
 LibMeshInit (int argc, const char *const *argv)
 
virtual ~LibMeshInit ()
 
const Parallel::Communicatorcomm () const
 
Parallel::Communicatorcomm ()
 

Private Attributes

Parallel::Communicator _comm
 
vtkMPIController * _vtk_mpi_controller
 

Detailed Description

The LibMeshInit class, when constructed, initializes the dependent libraries (e.g. MPI or PETSC) and does the command line parsing needed by libMesh. The LibMeshInit destructor closes those libraries properly.

For most users, a single LibMeshInit object should be created at the start of your main() function. This object replaces the previous

libMesh::init()/libMesh::close()

methods, which are now deprecated.

Definition at line 62 of file libmesh.h.

Constructor & Destructor Documentation

libMesh::LibMeshInit::LibMeshInit ( int  argc,
const char *const *  argv,
MPI_Comm  COMM_WORLD_IN = MPI_COMM_WORLD 
)

Initialize the library for use, with the command line options provided. This will e.g. call PetscInitialize if PETSC is available. You must create a LibMeshInit object before using any of the library functionality. This method may take an optional parameter to use a user-specified MPI communicator.

libMesh::LibMeshInit::LibMeshInit ( int  argc,
const char *const *  argv 
)

Definition at line 344 of file libmesh.C.

References libMesh::libMeshPrivateData::_is_initialized, libMesh::libMeshPrivateData::_n_processors, libMesh::libMeshPrivateData::_processor_id, libMesh::Singleton::cleanup(), libMesh::PerfLog::clear(), libMesh::Parallel::Communicator::clear(), libMesh::closed(), libMesh::COMM_WORLD, libMesh::command_line_value(), libMesh::PerfLog::disable_logging(), libMesh::ReferenceCounter::disable_print_counter_info(), DMCreate_libMesh(), libMesh::enableFPE(), libMesh::enableSEGV(), libMesh::err, libMesh::BasicOStreamProxy< charT, traits >::get(), libMesh::GLOBAL_COMM_WORLD, libMesh::global_processor_id(), ierr, libMesh::initialized(), libMesh_MPI_Handler(), libmesh_nullptr, libMesh::libmesh_terminate_handler(), libMesh::ReferenceCounter::n_objects(), libMesh::n_threads(), libMesh::on_command_line(), libMesh::out, libMesh::perflog, libMesh::ReferenceCounter::print_info(), libMesh::PerfLog::print_log(), libMesh::BasicOStreamProxy< charT, traits >::rdbuf(), libMesh::remote_elem, libMesh::BasicOStreamProxy< charT, traits >::reset(), libMesh::Singleton::setup(), and ~LibMeshInit().

349 {
350  // should _not_ be initialized already.
351  libmesh_assert (!libMesh::initialized());
352 
353  // Build a command-line parser.
354  command_line.reset (new GetPot (argc, argv));
355 
356  // Disable performance logging upon request
357  {
358  if (libMesh::on_command_line ("--disable-perflog"))
360  }
361 
362  // Build a task scheduler
363  {
364  // Get the requested number of threads, defaults to 1 to avoid MPI and
365  // multithreading competition. If you would like to use MPI and multithreading
366  // at the same time then (n_mpi_processes_per_node)x(n_threads) should be the
367  // number of processing cores per node.
368  std::vector<std::string> n_threads(2);
369  n_threads[0] = "--n_threads";
370  n_threads[1] = "--n-threads";
373 
374  // If there's no threading model active, force _n_threads==1
375 #if !LIBMESH_USING_THREADS
377  {
379  libmesh_warning("Warning: You requested --n-threads>1 but no threading model is active!\n"
380  << "Forcing --n-threads==1 instead!");
381  }
382 #endif
383 
384  // Set the number of OpenMP threads to the same as the number of threads libMesh is going to use
385 #ifdef LIBMESH_HAVE_OPENMP
386  omp_set_num_threads(libMesh::libMeshPrivateData::_n_threads);
387 #endif
388 
389  task_scheduler.reset (new Threads::task_scheduler_init(libMesh::n_threads()));
390  }
391 
392  // Construct singletons who may be at risk of the
393  // "static initialization order fiasco"
395 
396  // Make sure the construction worked
397  libmesh_assert(remote_elem);
398 
399 #if defined(LIBMESH_HAVE_MPI)
400 
401  // Allow the user to bypass MPI initialization
402  if (!libMesh::on_command_line ("--disable-mpi"))
403  {
404  // Check whether the calling program has already initialized
405  // MPI, and avoid duplicate Init/Finalize
406  int flag;
407  libmesh_call_mpi(MPI_Initialized (&flag));
408 
409  if (!flag)
410  {
411 #if MPI_VERSION > 1
412  int mpi_thread_provided;
413  const int mpi_thread_requested = libMesh::n_threads() > 1 ?
414  MPI_THREAD_FUNNELED :
415  MPI_THREAD_SINGLE;
416 
417  libmesh_call_mpi
418  (MPI_Init_thread (&argc, const_cast<char ***>(&argv),
419  mpi_thread_requested, &mpi_thread_provided));
420 
421  if ((libMesh::n_threads() > 1) &&
422  (mpi_thread_provided < MPI_THREAD_FUNNELED))
423  {
424  libmesh_warning("Warning: MPI failed to guarantee MPI_THREAD_FUNNELED\n"
425  << "for a threaded run.\n"
426  << "Be sure your library is funneled-thread-safe..."
427  << std::endl);
428 
429  // Ideally, if an MPI stack tells us it's unsafe for us
430  // to use threads, we shouldn't use threads.
431  // In practice, we've encountered one MPI stack (an
432  // mvapich2 configuration) that returned
433  // MPI_THREAD_SINGLE as a proper warning, two stacks
434  // that handle MPI_THREAD_FUNNELED properly, and two
435  // current stacks plus a couple old stacks that return
436  // MPI_THREAD_SINGLE but support libMesh threaded runs
437  // anyway.
438 
439  // libMesh::libMeshPrivateData::_n_threads = 1;
440  // task_scheduler.reset (new Threads::task_scheduler_init(libMesh::n_threads()));
441  }
442 #else
444  {
445  libmesh_warning("Warning: using MPI1 for threaded code.\n" <<
446  "Be sure your library is funneled-thread-safe..." <<
447  std::endl);
448  }
449 
450  libmesh_call_mpi
451  (MPI_Init (&argc, const_cast<char ***>(&argv)));
452 #endif
453  libmesh_initialized_mpi = true;
454  }
455 
456  // Duplicate the input communicator for internal use
457  // And get a Parallel::Communicator copy too, to use
458  // as a default for that API
459  this->_comm = COMM_WORLD_IN;
460 
461  libMesh::GLOBAL_COMM_WORLD = COMM_WORLD_IN;
462 
463 #ifndef LIBMESH_DISABLE_COMMWORLD
464  libMesh::COMM_WORLD = COMM_WORLD_IN;
465  Parallel::Communicator_World = COMM_WORLD_IN;
466 #endif
467 
468  //MPI_Comm_set_name not supported in at least SGI MPT's MPI implementation
469  //MPI_Comm_set_name (libMesh::COMM_WORLD, "libMesh::COMM_WORLD");
470 
472  cast_int<processor_id_type>(this->comm().rank());
474  cast_int<processor_id_type>(this->comm().size());
475 
476  // Set up an MPI error handler if requested. This helps us get
477  // into a debugger with a proper stack when an MPI error occurs.
478  if (libMesh::on_command_line ("--handle-mpi-errors"))
479  {
480 #if MPI_VERSION > 1
481  libmesh_call_mpi
482  (MPI_Comm_create_errhandler(libMesh_MPI_Handler, &libmesh_errhandler));
483  libmesh_call_mpi
484  (MPI_Comm_set_errhandler(libMesh::GLOBAL_COMM_WORLD, libmesh_errhandler));
485  libmesh_call_mpi
486  (MPI_Comm_set_errhandler(MPI_COMM_WORLD, libmesh_errhandler));
487 #else
488  libmesh_call_mpi
489  (MPI_Errhandler_create(libMesh_MPI_Handler, &libmesh_errhandler));
490  libmesh_call_mpi
491  (MPI_Errhandler_set(libMesh::GLOBAL_COMM_WORLD, libmesh_errhandler));
492  libmesh_call_mpi
493  (MPI_Errhandler_set(MPI_COMM_WORLD, libmesh_errhandler));
494 #endif // #if MPI_VERSION > 1
495  }
496  }
497 
498  // Could we have gotten bad values from the above calls?
499  libmesh_assert_greater (libMeshPrivateData::_n_processors, 0);
500 
501  // The cast_int already tested _processor_id>=0
502  // libmesh_assert_greater_equal (libMeshPrivateData::_processor_id, 0);
503 
504  // Let's be sure we properly initialize on every processor at once:
505  libmesh_parallel_only(this->comm());
506 
507 #endif
508 
509 #if defined(LIBMESH_HAVE_PETSC)
510 
511  // Allow the user to bypass PETSc initialization
512  if (!libMesh::on_command_line ("--disable-petsc")
513 
514 #if defined(LIBMESH_HAVE_MPI)
515  // If the user bypassed MPI, we'd better be safe and assume that
516  // PETSc was built to require it; otherwise PETSc initialization
517  // dies.
518  && !libMesh::on_command_line ("--disable-mpi")
519 #endif
520  )
521  {
522  int ierr=0;
523 
524  PETSC_COMM_WORLD = libMesh::GLOBAL_COMM_WORLD;
525 
526  // Check whether the calling program has already initialized
527  // PETSc, and avoid duplicate Initialize/Finalize
528  PetscBool petsc_already_initialized;
529  ierr = PetscInitialized(&petsc_already_initialized);
530  CHKERRABORT(libMesh::GLOBAL_COMM_WORLD,ierr);
531  if (petsc_already_initialized != PETSC_TRUE)
532  libmesh_initialized_petsc = true;
533 # if defined(LIBMESH_HAVE_SLEPC)
534 
535  // If SLEPc allows us to check whether the calling program
536  // has already initialized it, we do that, and avoid
537  // duplicate Initialize/Finalize.
538  // We assume that SLEPc will handle PETSc appropriately,
539  // which it does in the versions we've checked.
540  if (!SlepcInitializeCalled)
541  {
542  ierr = SlepcInitialize (&argc, const_cast<char ***>(&argv), libmesh_nullptr, libmesh_nullptr);
543  CHKERRABORT(libMesh::GLOBAL_COMM_WORLD,ierr);
544  libmesh_initialized_slepc = true;
545  }
546 # else
547  if (libmesh_initialized_petsc)
548  {
549  ierr = PetscInitialize (&argc, const_cast<char ***>(&argv), libmesh_nullptr, libmesh_nullptr);
550  CHKERRABORT(libMesh::GLOBAL_COMM_WORLD,ierr);
551  }
552 # endif
553 #if !PETSC_RELEASE_LESS_THAN(3,3,0)
554  // Register the reference implementation of DMlibMesh
555 #if PETSC_RELEASE_LESS_THAN(3,4,0)
556  ierr = DMRegister(DMLIBMESH, PETSC_NULL, "DMCreate_libMesh", DMCreate_libMesh); CHKERRABORT(libMesh::GLOBAL_COMM_WORLD,ierr);
557 #else
558  ierr = DMRegister(DMLIBMESH, DMCreate_libMesh); CHKERRABORT(libMesh::GLOBAL_COMM_WORLD,ierr);
559 #endif
560 
561 #endif
562  }
563 #endif
564 
565 #if defined(LIBMESH_HAVE_MPI) && defined(LIBMESH_HAVE_VTK)
566  // Do MPI initialization for VTK.
567  _vtk_mpi_controller = vtkMPIController::New();
568  _vtk_mpi_controller->Initialize(&argc, const_cast<char ***>(&argv), /*initialized_externally=*/1);
569  _vtk_mpi_controller->SetGlobalController(_vtk_mpi_controller);
570 #endif
571 
572  // Re-parse the command-line arguments. Note that PETSc and MPI
573  // initialization above may have removed command line arguments
574  // that are not relevant to this application in the above calls.
575  // We don't want a false-positive by detecting those arguments.
576  //
577  // Note: this seems overly paranoid/like it should be unnecessary,
578  // plus we were doing it wrong for many years and not clearing the
579  // existing GetPot object before re-parsing the command line, so all
580  // the command line arguments appeared twice in the GetPot object...
581  command_line.reset (new GetPot (argc, argv));
582 
583  // The following line is an optimization when simultaneous
584  // C and C++ style access to output streams is not required.
585  // The amount of benefit which occurs is probably implementation
586  // defined, and may be nothing. On the other hand, I have seen
587  // some IO tests where IO performance improves by a factor of two.
588  if (!libMesh::on_command_line ("--sync-with-stdio"))
589  std::ios::sync_with_stdio(false);
590 
591  // Honor the --separate-libmeshout command-line option.
592  // When this is specified, the library uses an independent ostream
593  // for libMesh::out/libMesh::err messages, and
594  // std::cout and std::cerr are untouched by any other options
595  if (libMesh::on_command_line ("--separate-libmeshout"))
596  {
597  // Redirect. We'll share streambufs with cout/cerr for now, but
598  // presumably anyone using this option will want to replace the
599  // bufs later.
600  std::ostream * newout = new std::ostream(std::cout.rdbuf());
601  libMesh::out = *newout;
602  std::ostream * newerr = new std::ostream(std::cerr.rdbuf());
603  libMesh::err = *newerr;
604  }
605 
606  // Process command line arguments for redirecting stdout/stderr.
607  bool
608  cmdline_has_redirect_stdout = libMesh::on_command_line ("--redirect-stdout"),
609  cmdline_has_redirect_output = libMesh::on_command_line ("--redirect-output");
610 
611  // The --redirect-stdout command-line option has been deprecated in
612  // favor of "--redirect-output basename".
613  if (cmdline_has_redirect_stdout)
614  libmesh_warning("The --redirect-stdout command line option has been deprecated. "
615  "Use '--redirect-output basename' instead.");
616 
617  // Honor the "--redirect-stdout" and "--redirect-output basename"
618  // command-line options. When one of these is specified, each
619  // processor sends libMesh::out/libMesh::err messages to
620  // stdout.processor.#### (default) or basename.processor.####.
621  if (cmdline_has_redirect_stdout || cmdline_has_redirect_output)
622  {
623  std::string basename = "stdout";
624 
625  // Look for following argument if using new API
626  if (cmdline_has_redirect_output)
627  {
628  // Set the cursor to the correct location in the list of command line arguments.
629  command_line->search(1, "--redirect-output");
630 
631  // Get the next option on the command line as a string.
632  std::string next_string = "";
633  next_string = command_line->next(next_string);
634 
635  // If the next string starts with a dash, we assume it's
636  // another flag and not a file basename requested by the
637  // user.
638  if (next_string.size() > 0 && next_string.find_first_of("-") != 0)
639  basename = next_string;
640  }
641 
642  std::ostringstream filename;
643  filename << basename << ".processor." << libMesh::global_processor_id();
644  _ofstream.reset (new std::ofstream (filename.str().c_str()));
645 
646  // Redirect, saving the original streambufs!
647  out_buf = libMesh::out.rdbuf (_ofstream->rdbuf());
648  err_buf = libMesh::err.rdbuf (_ofstream->rdbuf());
649  }
650 
651  // redirect libMesh::out to nothing on all
652  // other processors unless explicitly told
653  // not to via the --keep-cout command-line argument.
654  if (libMesh::global_processor_id() != 0)
655  if (!libMesh::on_command_line ("--keep-cout"))
657 
658  // Similarly, the user can request to drop cerr on all non-0 ranks.
659  // By default, errors are printed on all ranks, but this can lead to
660  // interleaved/unpredictable outputs when doing parallel regression
661  // testing, which this option is designed to support.
662  if (libMesh::global_processor_id() != 0)
663  if (libMesh::on_command_line ("--drop-cerr"))
665 
666  // Check command line to override printing
667  // of reference count information.
668  if (libMesh::on_command_line("--disable-refcount-printing"))
670 
671 #ifdef LIBMESH_ENABLE_EXCEPTIONS
672  // Set our terminate handler to write stack traces in the event of a
673  // crash
674  old_terminate_handler = std::set_terminate(libmesh_terminate_handler);
675 #endif
676 
677 
678  if (libMesh::on_command_line("--enable-fpe"))
679  libMesh::enableFPE(true);
680 
681  if (libMesh::on_command_line("--enable-segv"))
682  libMesh::enableSEGV(true);
683 
684  // The library is now ready for use
686 
687 
688  // Make sure these work. Library methods
689  // depend on these being implemented properly,
690  // so this is a good time to test them!
691  libmesh_assert (libMesh::initialized());
692  libmesh_assert (!libMesh::closed());
693 }
694 
695 
696 
698 {
699  // Every processor had better be ready to exit at the same time.
700  // This would be a libmesh_parallel_only() function, except that
701  // libmesh_parallel_only() uses libmesh_assert() which throws an
702  // exception() which causes compilers to scream about exceptions
703  // inside destructors.
704 
705  // Even if we're not doing parallel_only debugging, we don't want
706  // one processor to try to exit until all others are done working.
707  this->comm().barrier();
708 
709  // We can't delete, finalize, etc. more than once without
710  // reinitializing in between
711  libmesh_exceptionless_assert(!libMesh::closed());
712 
713  // Delete reference counted singleton(s)
715 
716  // Clear the thread task manager we started
717  task_scheduler.reset();
718 
719  // Force the \p ReferenceCounter to print
720  // its reference count information. This allows
721  // us to find memory leaks. By default the
722  // \p ReferenceCounter only prints its information
723  // when the last created object has been destroyed.
724  // That does no good if we are leaking memory!
726 
727 
728  // Print an informative message if we detect a memory leak
729  if (ReferenceCounter::n_objects() != 0)
730  {
731  libMesh::err << "Memory leak detected!"
732  << std::endl;
733 
734 #if !defined(LIBMESH_ENABLE_REFERENCE_COUNTING) || defined(NDEBUG)
735 
736  libMesh::err << "Compile in DEBUG mode with --enable-reference-counting"
737  << std::endl
738  << "for more information"
739  << std::endl;
740 #endif
741 
742  }
743 
744  // print the perflog to individual processor's file.
746 
747  // Now clear the logging object, we don't want it to print
748  // a second time during the PerfLog destructor.
750 
751  // Reconnect the output streams
752  // (don't do this, or we will get messages from objects
753  // that go out of scope after the following return)
754  //std::cout.rdbuf(std::cerr.rdbuf());
755 
756 
757  // Set the initialized() flag to false
759 
760  if (libMesh::on_command_line ("--redirect-stdout") ||
761  libMesh::on_command_line ("--redirect-output"))
762  {
763  // If stdout/stderr were redirected to files, reset them now.
764  libMesh::out.rdbuf (out_buf);
765  libMesh::err.rdbuf (err_buf);
766  }
767 
768  // If we built our own output streams, we want to clean them up.
769  if (libMesh::on_command_line ("--separate-libmeshout"))
770  {
771  delete libMesh::out.get();
772  delete libMesh::err.get();
773 
774  libMesh::out.reset(std::cout);
775  libMesh::err.reset(std::cerr);
776  }
777 
778 #ifdef LIBMESH_ENABLE_EXCEPTIONS
779  // Reset the old terminate handler; maybe the user code wants to
780  // keep doing C++ stuff after closing libMesh stuff.
781  std::set_terminate(old_terminate_handler);
782 #endif
783 
784 
785  if (libMesh::on_command_line("--enable-fpe"))
786  libMesh::enableFPE(false);
787 
788 #if defined(LIBMESH_HAVE_PETSC)
789  // Allow the user to bypass PETSc finalization
790  if (!libMesh::on_command_line ("--disable-petsc")
791 #if defined(LIBMESH_HAVE_MPI)
792  && !libMesh::on_command_line ("--disable-mpi")
793 #endif
794  )
795  {
796 # if defined(LIBMESH_HAVE_SLEPC)
797  if (libmesh_initialized_slepc)
798  SlepcFinalize();
799 # else
800  if (libmesh_initialized_petsc)
801  PetscFinalize();
802 # endif
803  }
804 #endif
805 
806 #if defined(LIBMESH_HAVE_MPI) && defined(LIBMESH_HAVE_VTK)
807  _vtk_mpi_controller->Finalize(/*finalized_externally=*/1);
808  _vtk_mpi_controller->Delete();
809 #endif
810 
811 #if defined(LIBMESH_HAVE_MPI)
812  // Allow the user to bypass MPI finalization
813  if (!libMesh::on_command_line ("--disable-mpi"))
814  {
815  this->_comm.clear();
816 #ifndef LIBMESH_DISABLE_COMMWORLD
818 #endif
819 
820  if (libmesh_initialized_mpi)
821  {
822  // We can't just libmesh_assert here because destructor,
823  // but we ought to report any errors
824  unsigned int error_code = MPI_Finalize();
825  if (error_code != MPI_SUCCESS)
826  {
827  char error_string[MPI_MAX_ERROR_STRING+1];
828  int error_string_len;
829  MPI_Error_string(error_code, error_string,
830  &error_string_len);
831  std::cerr << "Failure from MPI_Finalize():\n"
832  << error_string << std::endl;
833  }
834  }
835  }
836 #endif
837 }
bool closed()
Definition: libmesh.C:280
static unsigned int n_objects()
unsigned int n_threads()
Definition: libmesh_base.h:125
FakeCommunicator & Communicator_World
unsigned int size() const
Definition: parallel.h:725
streambufT * rdbuf() const
void enableSEGV(bool on)
Definition: libmesh.C:890
processor_id_type _n_processors
Definition: libmesh.C:250
Parallel::Communicator _comm
Definition: libmesh.h:86
const class libmesh_nullptr_t libmesh_nullptr
MPI_Comm GLOBAL_COMM_WORLD
Definition: libmesh.C:198
MPI_Comm COMM_WORLD
Definition: libmesh.C:196
void reset(streamT &target)
vtkMPIController * _vtk_mpi_controller
Definition: libmesh.h:92
PetscErrorCode DMCreate_libMesh(DM dm)
tbb::task_scheduler_init task_scheduler_init
Definition: threads_tbb.h:73
std::terminate_handler old_terminate_handler
Definition: libmesh.C:287
processor_id_type _processor_id
Definition: libmesh.C:251
const Parallel::Communicator & comm() const
Definition: libmesh.h:81
OStreamProxy err(std::cerr)
static void print_info(std::ostream &out=libMesh::out)
PerfLog perflog("libMesh",#ifdef LIBMESH_ENABLE_PERFORMANCE_LOGGING true#else false#endif)
MPI_Errhandler libmesh_errhandler
Definition: libmesh.C:248
void clear()
Definition: perf_log.C:77
void enableFPE(bool on)
Definition: libmesh.C:844
PetscTruth PetscBool
Definition: petsc_macro.h:64
PetscErrorCode ierr
bool on_command_line(const std::string &arg)
Definition: libmesh.C:920
processor_id_type global_processor_id()
Definition: libmesh_base.h:114
bool initialized()
Definition: libmesh.C:273
void libMesh_MPI_Handler(MPI_Comm *, int *,...)
Definition: libmesh.C:161
unsigned int rank() const
Definition: parallel.h:723
static void cleanup()
void print_log() const
Definition: perf_log.C:684
T command_line_value(const std::string &name, T value)
Definition: libmesh.C:931
static void disable_print_counter_info()
OStreamProxy out(std::cout)
void disable_logging()
Definition: perf_log.h:156
void libmesh_terminate_handler()
Definition: libmesh.C:289
const RemoteElem * remote_elem
Definition: remote_elem.C:57
virtual libMesh::LibMeshInit::~LibMeshInit ( )
virtual

Referenced by LibMeshInit().

Member Function Documentation

const Parallel::Communicator& libMesh::LibMeshInit::comm ( ) const
inline

Definition at line 81 of file libmesh.h.

References _comm.

81 { return _comm; }
Parallel::Communicator _comm
Definition: libmesh.h:86
Parallel::Communicator& libMesh::LibMeshInit::comm ( )
inline

Definition at line 83 of file libmesh.h.

References _comm.

83 { return _comm; }
Parallel::Communicator _comm
Definition: libmesh.h:86

Member Data Documentation

Parallel::Communicator libMesh::LibMeshInit::_comm
private

Definition at line 86 of file libmesh.h.

Referenced by comm().

vtkMPIController* libMesh::LibMeshInit::_vtk_mpi_controller
private

Definition at line 92 of file libmesh.h.


The documentation for this class was generated from the following files: