libMesh::PetscDMWrapper Class Reference

#include <petsc_dm_wrapper.h>

Public Member Functions
	PetscDMWrapper ()=default
	~PetscDMWrapper ()
void	clear ()
	Destroys and clears all build DM-related data. More...
void	init_and_attach_petscdm (System &system, SNES &snes)

Private Member Functions
void	init_dm_data (unsigned int n_levels)
	Init all the n_mesh_level dependent data structures. More...
DM &	get_dm (unsigned int level)
	Get reference to DM for the given mesh level. More...
PetscSection &	get_section (unsigned int level)
	Get reference to PetscSection for the given mesh level. More...
PetscSF &	get_star_forest (unsigned int level)
	Get reference to PetscSF for the given mesh level. More...
void	build_section (const System &system, PetscSection &section)
	Takes System, empty PetscSection and populates the PetscSection. More...
void	build_sf (const System &system, PetscSF &star_forest)
	Takes System, empty PetscSF and populates the PetscSF. More...
void	set_point_range_in_section (const System &system, PetscSection &section, std::unordered_map< dof_id_type, dof_id_type > &node_map, std::unordered_map< dof_id_type, dof_id_type > &elem_map, std::map< dof_id_type, unsigned int > &scalar_map)
	Helper function for build_section. More...
void	add_dofs_to_section (const System &system, PetscSection &section, const std::unordered_map< dof_id_type, dof_id_type > &node_map, const std::unordered_map< dof_id_type, dof_id_type > &elem_map, const std::map< dof_id_type, unsigned int > &scalar_map)
	Helper function for build_section. More...
dof_id_type	check_section_n_dofs (const System &system, PetscSection &section)
	Helper function to sanity check PetscSection construction. More...
void	add_dofs_helper (const System &system, const DofObject &dof_object, dof_id_type local_id, PetscSection &section)
	Helper function to reduce code duplication when setting dofs in section. More...

Private Attributes
std::vector< std::unique_ptr< DM > >	_dms
	Vector of DMs for all grid levels. More...
std::vector< std::unique_ptr< PetscSection > >	_sections
	Vector of PETScSections for all grid levels. More...
std::vector< std::unique_ptr< PetscSF > >	_star_forests
	Vector of star forests for all grid levels. More...

Detailed Description

This class defines a wrapper around the PETSc DM infrastructure. By coordinating DM data structures with libMesh, we can use libMesh mesh hierarchies for geometric multigrid. Additionally, by setting the DM data, we can additionally (with or without multigrid) define recursive fieldsplits of our variables.

Author

Paul T. Bauman, Boris Boutkov

Date

2018

Definition at line 51 of file petsc_dm_wrapper.h.

Constructor & Destructor Documentation

PetscDMWrapper()

libMesh::PetscDMWrapper::PetscDMWrapper ( )

default

~PetscDMWrapper()

libMesh::PetscDMWrapper::~PetscDMWrapper

(

)

Definition at line 36 of file petsc_dm_wrapper.C.

References clear().

{
  this->clear();
}

Member Function Documentation

add_dofs_helper()

void libMesh::PetscDMWrapper::add_dofs_helper ( const System &  system, const DofObject &  dof_object, dof_id_type  local_id, PetscSection &  section  )

private

Helper function to reduce code duplication when setting dofs in section.

Definition at line 407 of file petsc_dm_wrapper.C.

References libMesh::ParallelObject::comm(), libMesh::Parallel::Communicator::get(), ierr, libMesh::DofObject::n_dofs(), libMesh::System::n_vars(), and libMesh::System::number().

Referenced by add_dofs_to_section().

{
  unsigned int total_n_dofs_at_dofobject = 0;

  // We are assuming variables are also numbered 0 to n_vars()-1
  for( unsigned int v = 0; v < system.n_vars(); v++ )
    {
      unsigned int n_dofs_at_dofobject = dof_object.n_dofs(system.number(), v);

      if( n_dofs_at_dofobject > 0 )
        {
          PetscErrorCode ierr = PetscSectionSetFieldDof( section,
                                                         local_id,
                                                         v,
                                                         n_dofs_at_dofobject );

          CHKERRABORT(system.comm().get(),ierr);

          total_n_dofs_at_dofobject += n_dofs_at_dofobject;
        }
    }

  libmesh_assert_equal_to(total_n_dofs_at_dofobject, dof_object.n_dofs(system.number()));

  PetscErrorCode ierr =
    PetscSectionSetDof( section, local_id, total_n_dofs_at_dofobject );
  CHKERRABORT(system.comm().get(),ierr);
}

add_dofs_to_section()

void libMesh::PetscDMWrapper::add_dofs_to_section ( const System &  system, PetscSection &  section, const std::unordered_map< dof_id_type, dof_id_type > &  node_map, const std::unordered_map< dof_id_type, dof_id_type > &  elem_map, const std::map< dof_id_type, unsigned int > &  scalar_map  )

private

Helper function for build_section.

This function will set the DoF info for each "point" in the PetscSection.

Definition at line 340 of file petsc_dm_wrapper.C.

References add_dofs_helper(), libMesh::ParallelObject::comm(), libMesh::Parallel::Communicator::get(), libMesh::System::get_mesh(), libMesh::OrderWrapper::get_order(), ierr, mesh, libMesh::ParallelObject::n_processors(), libMesh::FEType::order, libMesh::ParallelObject::processor_id(), libMesh::Variable::type(), and libMesh::System::variable().

Referenced by build_section().

{
  const MeshBase & mesh = system.get_mesh();

  PetscErrorCode ierr;

  // Now we go through and add dof information for each "point".
  //
  // In libMesh, for most finite elements, we just associate those DoFs with the
  // geometric nodes. So can we loop over the nodes we cached in the node_map and
  // the DoFs for each field for that node. We need to give PETSc the local id
  // we built up in the node map.
  for (const auto & nmap : node_map )
    {
      const dof_id_type global_node_id = nmap.first;
      const dof_id_type local_node_id = nmap.second;

      libmesh_assert( mesh.query_node_ptr(global_node_id) );

      const Node & node = mesh.node_ref(global_node_id);

      this->add_dofs_helper(system,node,local_node_id,section);
    }

  // Some finite element types associate dofs with the element. So now we go through
  // any of those with the Elem as the point we add to the PetscSection with accompanying
  // dofs
  for (const auto & emap : elem_map )
    {
      const dof_id_type global_elem_id = emap.first;
      const dof_id_type local_elem_id = emap.second;

      libmesh_assert( mesh.query_elem_ptr(global_elem_id) );

      const Elem & elem = mesh.elem_ref(global_elem_id);

      this->add_dofs_helper(system,elem,local_elem_id,section);
    }

  // Now add any SCALAR dofs to the PetscSection
  // SCALAR dofs live on the "last" processor, so only work there if there are any
  if (system.processor_id() == (system.n_processors()-1))
    {
      for (const auto & smap : scalar_map )
        {
          const dof_id_type local_id = smap.first;
          const unsigned int scalar_var = smap.second;

          // The number of SCALAR dofs comes from the variable order
          const int n_dofs = system.variable(scalar_var).type().order.get_order();

          ierr = PetscSectionSetFieldDof( section, local_id, scalar_var, n_dofs );
          CHKERRABORT(system.comm().get(),ierr);

          // In the SCALAR case, there are no other variables associate with the "point"
          // the total number of dofs on the point is the same as that for the field
          ierr = PetscSectionSetDof( section, local_id, n_dofs );
          CHKERRABORT(system.comm().get(),ierr);
        }
    }

}

build_section()

void libMesh::PetscDMWrapper::build_section ( const System &  system, PetscSection &  section  )

private

Takes System, empty PetscSection and populates the PetscSection.

Take the System in its current state and an empty PetscSection and then populate the PetscSection. The PetscSection is comprised of global "point" numbers, where a "point" in PetscDM parlance is a geometric entity: node, edge, face, or element. Then, we also add the DoF numbering for each variable for each of the "points". The PetscSection, together the with PetscSF will allow for recursive fieldsplits from the command line using PETSc.

Definition at line 98 of file petsc_dm_wrapper.C.

References add_dofs_to_section(), check_section_n_dofs(), libMesh::ParallelObject::comm(), libMesh::Parallel::Communicator::get(), ierr, libMesh::System::n_local_dofs(), libMesh::System::n_vars(), libMesh::on_command_line(), set_point_range_in_section(), and libMesh::System::variable_name().

Referenced by init_and_attach_petscdm().

{
  START_LOG ("build_section()", "PetscDMWrapper");

  PetscErrorCode ierr;
  ierr = PetscSectionCreate(system.comm().get(),&section);
  CHKERRABORT(system.comm().get(),ierr);

  // Tell the PetscSection about all of our System variables
  ierr = PetscSectionSetNumFields(section,system.n_vars());
  CHKERRABORT(system.comm().get(),ierr);

  // Set the actual names of all the field variables
  for( unsigned int v = 0; v < system.n_vars(); v++ )
    {
      ierr = PetscSectionSetFieldName( section, v, system.variable_name(v).c_str() );
      CHKERRABORT(system.comm().get(),ierr);
    }

  // For building the section, we need to create local-to-global map
  // of local "point" ids to the libMesh global id of that point.
  // A "point" in PETSc nomenclature is a geometric object that can have
  // dofs associated with it, e.g. Node, Edge, Face, Elem.
  // The numbering PETSc expects is continuous for the local numbering.
  // Since we're only using this interface for solvers, then we can just
  // assign whatever local id to any of the global ids. But it is local
  // so we don't need to worry about processor numbering for the local
  // point ids.
  std::unordered_map<dof_id_type,dof_id_type> node_map;
  std::unordered_map<dof_id_type,dof_id_type> elem_map;
  std::map<dof_id_type,unsigned int> scalar_map;

  // First we tell the PetscSection about all of our points that have
  // dofs associated with them.
  this->set_point_range_in_section(system, section, node_map, elem_map, scalar_map);

  // Now we can build up the dofs per "point" in the PetscSection
  this->add_dofs_to_section(system, section, node_map, elem_map, scalar_map);

  // Final setup of PetscSection
  // Until Matt Knepley finishes implementing the commented out function
  // below, the PetscSection will be assuming node-major ordering
  // so let's throw and error if the user tries to use this without
  // node-major order
  if (!libMesh::on_command_line("--node-major-dofs"))
    libmesh_error_msg("ERROR: Must use --node-major-dofs with PetscSection!");

  //else if (!system.identify_variable_groups())
  //  ierr = PetscSectionSetUseFieldOffsets(section,PETSC_TRUE);CHKERRABORT(system.comm().get(),ierr);
  //else
  //  {
  //    std::string msg = "ERROR: Only node-major or var-major ordering supported for PetscSection!\n";
  //    msg += "       var-group-major ordering not supported!\n";
  //    msg += "       Must use --node-major-dofs or set System::identify_variable_groups() = false!\n";
  //    libmesh_error_msg(msg);
  //  }

  ierr = PetscSectionSetUp(section);CHKERRABORT(system.comm().get(),ierr);

  // Sanity checking at least that local_n_dofs match
  libmesh_assert_equal_to(system.n_local_dofs(),this->check_section_n_dofs(system, section));

  STOP_LOG ("build_section()", "PetscDMWrapper");
}

build_sf()

void libMesh::PetscDMWrapper::build_sf ( const System &  system, PetscSF &  star_forest  )

private

Takes System, empty PetscSF and populates the PetscSF.

The PetscSF (star forest) is a cousin of PetscSection. PetscSection has the DoF info, and PetscSF gives the parallel distribution of the DoF info. So PetscSF should only be necessary when we have more than one MPI rank. Essentially, we are copying the DofMap.send_list(): we are specifying the local dofs, what rank communicates that dof info (for off-processor dofs that are communicated) and the dofs local index on that rank.

https://jedbrown.org/files/StarForest.pdf

Definition at line 163 of file petsc_dm_wrapper.C.

References libMesh::ParallelObject::comm(), libMesh::DofMap::dof_owner(), libMesh::DofMap::first_dof(), libMesh::Parallel::Communicator::get(), libMesh::System::get_dof_map(), libMesh::DofMap::get_send_list(), ierr, libMesh::DofMap::n_local_dofs(), and PETSC_COPY_VALUES.

Referenced by init_and_attach_petscdm().

{
  START_LOG ("build_sf()", "PetscDMWrapper");

  const DofMap & dof_map = system.get_dof_map();

  const std::vector<dof_id_type> & send_list = dof_map.get_send_list();

  // Number of ghost dofs that send information to this processor
  const PetscInt n_leaves = cast_int<PetscInt>(send_list.size());

  // Number of local dofs, including ghosts dofs
  const PetscInt n_roots = dof_map.n_local_dofs() + n_leaves;

  // This is the vector of dof indices coming from other processors
  // We need to give this to the PetscSF
  // We'll be extra paranoid about this ugly double cast
  static_assert(sizeof(PetscInt) == sizeof(dof_id_type),"PetscInt is not a dof_id_type!");
  PetscInt * local_dofs = reinterpret_cast<PetscInt *>(const_cast<dof_id_type *>(send_list.data()));

  // This is the vector of PetscSFNode's for the local_dofs.
  // For each entry in local_dof, we have to supply the rank from which
  // that dof stems and its local index on that rank.
  // PETSc documentation here:
  // http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/PetscSF/PetscSFNode.html
  std::vector<PetscSFNode> sf_nodes(send_list.size());

  for( unsigned int i = 0; i < send_list.size(); i++ )
    {
      dof_id_type incoming_dof = send_list[i];

      const processor_id_type rank = dof_map.dof_owner(incoming_dof);

      // Dofs are sorted and continuous on the processor so local index
      // is counted up from the first dof on the processor.
      PetscInt index = incoming_dof - dof_map.first_dof(rank);

      sf_nodes[i].rank  = rank; /* Rank of owner */
      sf_nodes[i].index = index;/* Index of dof on rank */
    }

  PetscSFNode * remote_dofs = sf_nodes.data();

  PetscErrorCode ierr;
  ierr = PetscSFCreate(system.comm().get(), &star_forest);CHKERRABORT(system.comm().get(),ierr);

  // TODO: We should create pointers to arrays so we don't have to copy
  //       and then can use PETSC_OWN_POINTER where PETSc will take ownership
  //       and delete the memory for us. But then we'd have to use PetscMalloc.
  ierr = PetscSFSetGraph(star_forest,
                         n_roots,
                         n_leaves,
                         local_dofs,
                         PETSC_COPY_VALUES,
                         remote_dofs,
                         PETSC_COPY_VALUES);
  CHKERRABORT(system.comm().get(),ierr);

  STOP_LOG ("build_sf()", "PetscDMWrapper");
}

check_section_n_dofs()

dof_id_type libMesh::PetscDMWrapper::check_section_n_dofs ( const System &  system, PetscSection &  section  )

private

Helper function to sanity check PetscSection construction.

The PetscSection contains local dof information. This helper function just facilitates sanity checking that in fact it only has n_local_dofs.

Definition at line 440 of file petsc_dm_wrapper.C.

References libMesh::ParallelObject::comm(), libMesh::Parallel::Communicator::get(), and ierr.

Referenced by build_section().

{
  PetscInt n_local_dofs = 0;

  // Grap the starting and ending points from the section
  PetscInt pstart, pend;
  PetscErrorCode ierr = PetscSectionGetChart(section, &pstart, &pend);
  CHKERRABORT(system.comm().get(),ierr);

  // Count up the n_dofs for each point from the section
  for( PetscInt p = pstart; p < pend; p++ )
    {
      PetscInt n_dofs;
      ierr = PetscSectionGetDof(section,p,&n_dofs);CHKERRABORT(system.comm().get(),ierr);
      n_local_dofs += n_dofs;
    }

  static_assert(sizeof(PetscInt) == sizeof(dof_id_type),"PetscInt is not a dof_id_type!");
  return n_local_dofs;
}

clear()

void libMesh::PetscDMWrapper::clear

(

)

Destroys and clears all build DM-related data.

Definition at line 41 of file petsc_dm_wrapper.C.

References _dms, _sections, and _star_forests.

Referenced by libMesh::PetscDiffSolver::clear(), and ~PetscDMWrapper().

{
  // This will also Destroy the attached PetscSection and PetscSF as well
  // Destroy doesn't free the memory, but just resets points internally
  // in the struct, so we'd still need to wipe out the memory on our side
  for( auto dm_it = _dms.begin(); dm_it < _dms.end(); ++dm_it )
    DMDestroy( dm_it->get() );

  _dms.clear();
  _sections.clear();
  _star_forests.clear();
}

get_dm()

DM& libMesh::PetscDMWrapper::get_dm ( unsigned int  level )

inlineprivate

Get reference to DM for the given mesh level.

init_dm_data() should be called before this function.

Definition at line 82 of file petsc_dm_wrapper.h.

References _dms.

Referenced by init_and_attach_petscdm().

  { libmesh_assert(level < _dms.size());
    return *(_dms[level].get()); }

get_section()

PetscSection& libMesh::PetscDMWrapper::get_section ( unsigned int  level )

inlineprivate

Get reference to PetscSection for the given mesh level.

init_dm_data() should be called before this function.

Definition at line 90 of file petsc_dm_wrapper.h.

References _sections.

Referenced by init_and_attach_petscdm().

  { libmesh_assert(level < _sections.size());
    return *(_sections[level].get()); }

get_star_forest()

PetscSF& libMesh::PetscDMWrapper::get_star_forest ( unsigned int  level )

inlineprivate

Get reference to PetscSF for the given mesh level.

init_dm_data() should be called before this function.

Definition at line 98 of file petsc_dm_wrapper.h.

References _star_forests.

Referenced by init_and_attach_petscdm().

  { libmesh_assert(level < _star_forests.size());
    return *(_star_forests[level].get()); }

init_and_attach_petscdm()

void libMesh::PetscDMWrapper::init_and_attach_petscdm	(	System &	system,
		SNES &	snes
	)

Definition at line 54 of file petsc_dm_wrapper.C.

References build_section(), build_sf(), libMesh::ParallelObject::comm(), libMesh::Parallel::Communicator::get(), get_dm(), get_section(), get_star_forest(), ierr, init_dm_data(), libMesh::MeshTools::n_levels(), and libMesh::ParallelObject::n_processors().

Referenced by libMesh::PetscDiffSolver::setup_petsc_data().

{
  START_LOG ("init_and_attach_petscdm()", "PetscDMWrapper");

  PetscErrorCode ierr;

  // Eventually, we'll traverse the mesh hierarchy and cache
  // for each grid level, but for now, we're just using the
  // finest level
  unsigned int n_levels = 1;
  this->init_dm_data(n_levels);

  for(unsigned int level = 0; level < n_levels; level++)
    {
      DM & dm = this->get_dm(level);
      PetscSection & section = this->get_section(level);
      PetscSF & star_forest = this->get_star_forest(level);

      ierr = DMShellCreate(system.comm().get(), &dm);
      CHKERRABORT(system.comm().get(),ierr);

      // Build the PetscSection and attach it to the DM
      this->build_section(system, section);
      ierr = DMSetDefaultSection(dm, section);
      CHKERRABORT(system.comm().get(),ierr);

      // We only need to build the star forest if we're in a parallel environment
      if (system.n_processors() > 1)
        {
          // Build the PetscSF and attach it to the DM
          this->build_sf(system, star_forest);
          ierr = DMSetDefaultSF(dm, star_forest);
          CHKERRABORT(system.comm().get(),ierr);
        }
    }

  // We need to set only the finest level DM in the SNES
  DM & dm = this->get_dm(0);
  ierr = SNESSetDM(snes, dm);
  CHKERRABORT(system.comm().get(),ierr);

  STOP_LOG ("init_and_attach_petscdm()", "PetscDMWrapper");
}

init_dm_data()

void libMesh::PetscDMWrapper::init_dm_data ( unsigned int  n_levels )

private

Init all the n_mesh_level dependent data structures.

Definition at line 461 of file petsc_dm_wrapper.C.

References _dms, _sections, _star_forests, and libMesh::MeshTools::n_levels().

Referenced by init_and_attach_petscdm().

{
  _dms.resize(n_levels);
  _sections.resize(n_levels);
  _star_forests.resize(n_levels);

  for( unsigned int i = 0; i < n_levels; i++ )
    {
      _dms[i] = libmesh_make_unique<DM>();
      _sections[i] = libmesh_make_unique<PetscSection>();
      _star_forests[i] = libmesh_make_unique<PetscSF>();
    }
}

set_point_range_in_section()

void libMesh::PetscDMWrapper::set_point_range_in_section ( const System &  system, PetscSection &  section, std::unordered_map< dof_id_type, dof_id_type > &  node_map, std::unordered_map< dof_id_type, dof_id_type > &  elem_map, std::map< dof_id_type, unsigned int > &  scalar_map  )

private

Helper function for build_section.

This function will count how many "points" on the current processor have DoFs associated with them and give that count to PETSc. We need to cache a mapping between the global node id and our local count that we do in this function because we will need the local number again in the add_dofs_to_section function.

Definition at line 224 of file petsc_dm_wrapper.C.

References libMesh::ParallelObject::comm(), libMesh::DofMap::dof_indices(), libMesh::FEType::family, libMesh::Parallel::Communicator::get(), libMesh::System::get_dof_map(), libMesh::System::get_mesh(), libMesh::DofObject::id(), ierr, libMesh::DofMap::local_index(), mesh, libMesh::DofMap::n_local_dofs(), libMesh::ParallelObject::n_processors(), libMesh::DofMap::n_SCALAR_dofs(), libMesh::System::n_vars(), libMesh::System::number(), libMesh::ParallelObject::processor_id(), libMesh::SCALAR, libMesh::Variable::type(), and libMesh::System::variable().

Referenced by build_section().

{
  // We're expecting this to be empty coming in
  libmesh_assert(node_map.empty());

  // We need to count up the number of active "points" on this processor.
  // Nominally, a "point" in PETSc parlance is a geometric object that can
  // hold DoFs, i.e node, edge, face, elem. Since we handle the mesh and are only
  // interested in solvers, then the only thing PETSc needs is a unique *local* number
  // for each "point" that has active DoFs; note however this local numbering
  // we construct must be continuous.
  //
  // In libMesh, for most finite elements, we just associate those DoFs with the
  // geometric nodes. So can we loop over the nodes on this processor and check
  // if any of the fields are have active DoFs on that node.
  // If so, then we tell PETSc about that "point". At this stage, we just need
  // to count up how many active "points" we have and cache the local number to global id
  // mapping.


  // These will be our local counters. pstart should always be zero.
  // pend will track our local "point" count.
  // If we're on a processor who coarsened the mesh to have no local elements,
  // we should make an empty PetscSection. An empty PetscSection is specified
  // by passing [0,0) to the PetscSectionSetChart call at the end. So, if we
  // have nothing on this processor, these are the correct values to pass to
  // PETSc.
  dof_id_type pstart = 0;
  dof_id_type pend = 0;

  const MeshBase & mesh = system.get_mesh();

  const DofMap & dof_map = system.get_dof_map();

  // If we don't have any local dofs, then there's nothing to tell to the PetscSection
  if (dof_map.n_local_dofs() > 0)
    {
      // Conservative estimate of space needed so we don't thrash
      node_map.reserve(mesh.n_local_nodes());
      elem_map.reserve(mesh.n_active_local_elem());

      // We loop over active elements and then cache the global/local node mapping to make sure
      // we only count active nodes. For example, if we're calling this function and we're
      // not the finest level in the Mesh tree, we don't want to include nodes of child elements
      // that aren't active on this level.
      for (const auto & elem : mesh.active_local_element_ptr_range())
        {
          for (unsigned int n = 0; n < elem->n_nodes(); n++)
            {
              // get the global id number of local node n
              const Node & node = elem->node_ref(n);

              // Only register nodes with the PetscSection if they have dofs that belong to
              // this processor. Even though we're active local elements, the dofs associated
              // with the node may belong to a different processor. The processor who owns
              // those dofs will register that node with the PetscSection on that processor.
              std::vector<dof_id_type> node_dof_indices;
              dof_map.dof_indices( &node, node_dof_indices );
              if( !node_dof_indices.empty() && dof_map.local_index(node_dof_indices[0]) )
                {
#ifndef NDEBUG
                  // We're assuming that if the first dof for this node belongs to this processor,
                  // then all of them do.
                  for( auto dof : node_dof_indices )
                    libmesh_assert(dof_map.local_index(dof));
#endif
                  // Cache the global/local mapping if we haven't already
                  // Then increment our local count
                  dof_id_type node_id = node.id();
                  if( node_map.count(node_id) == 0 )
                    {
                      node_map.insert(std::make_pair(node_id,pend));
                      pend++;
                    }
                }
            }

          // Some finite elements, e.g. Hierarchic, associate element interior DoFs with the element
          // rather than the node (since we ought to be able to use Hierachic elements on a QUAD4,
          // which has no interior node). Thus, we also need to check element interiors for DoFs
          // as well and, if the finite element has them, we also need to count the Elem in our
          // "point" accounting.
          if( elem->n_dofs(system.number()) > 0 )
            {
              dof_id_type elem_id = elem->id();
              elem_map.insert(std::make_pair(elem_id,pend));
              pend++;
            }
        }

      // SCALAR dofs live on the "last" processor, so only work there if there are any
      if( dof_map.n_SCALAR_dofs() > 0 && (system.processor_id() == (system.n_processors()-1)) )
        {
          // Loop through all the variables and cache the scalar ones. We cache the
          // SCALAR variable index along with the local point to make it easier when
          // we have to register dofs with the PetscSection
          for( unsigned int v = 0; v < system.n_vars(); v++ )
            {
              if( system.variable(v).type().family == SCALAR )
                {
                  scalar_map.insert(std::make_pair(pend,v));
                  pend++;
                }
            }
        }

    }

  PetscErrorCode ierr = PetscSectionSetChart(section, pstart, pend);
  CHKERRABORT(system.comm().get(),ierr);
}

Member Data Documentation

_dms

std::vector<std::unique_ptr<DM> > libMesh::PetscDMWrapper::_dms

private

Vector of DMs for all grid levels.

Definition at line 67 of file petsc_dm_wrapper.h.

Referenced by clear(), get_dm(), and init_dm_data().

_sections

std::vector<std::unique_ptr<PetscSection> > libMesh::PetscDMWrapper::_sections

private

Vector of PETScSections for all grid levels.

Definition at line 70 of file petsc_dm_wrapper.h.

Referenced by clear(), get_section(), and init_dm_data().

_star_forests

std::vector<std::unique_ptr<PetscSF> > libMesh::PetscDMWrapper::_star_forests

private

Vector of star forests for all grid levels.

Definition at line 73 of file petsc_dm_wrapper.h.

Referenced by clear(), get_star_forest(), and init_dm_data().

---

The documentation for this class was generated from the following files:

• petsc_dm_wrapper.h
• petsc_dm_wrapper.C