Date: 2021-03-03T22:26:54+00:00

Actual source code: dmmoab.cxx
petsc-3.14.5 2021-03-03
Report Typos and Errors

      1: #include <petsc/private/dmmbimpl.h>

      3: #include <petscdmmoab.h>
      4: #include <MBTagConventions.hpp>
      5: #include <moab/NestedRefine.hpp>
      6: #include <moab/Skinner.hpp>

      8: /*MC
      9:   DMMOAB = "moab" - A DM object that encapsulates an unstructured mesh described by the MOAB mesh database.
     10:                     Direct access to the MOAB Interface and other mesh manipulation related objects are available
     11:                     through public API. Ability to create global and local representation of Vecs containing all
     12:                     unknowns in the interior and shared boundary via a transparent tag-data wrapper is provided
     13:                     along with utility functions to traverse the mesh and assemble a discrete system via
     14:                     field-based/blocked Vec(Get/Set) methods. Input from and output to different formats are
     15:                     available.

     17:   Reference: https://www.mcs.anl.gov/~fathom/moab-docs/html/contents.html

     19:   Level: intermediate

     21: .seealso: DMType, DMMoabCreate(), DMCreate(), DMSetType(), DMMoabCreateMoab()
     22: M*/

     24: /* External function declarations here */
     25: PETSC_EXTERN PetscErrorCode DMCreateInterpolation_Moab(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling);
     26: PETSC_EXTERN PetscErrorCode DMCreateDefaultConstraints_Moab(DM dm);
     27: PETSC_EXTERN PetscErrorCode DMCreateMatrix_Moab(DM dm,  Mat *J);
     28: PETSC_EXTERN PetscErrorCode DMCreateCoordinateDM_Moab(DM dm, DM *cdm);
     29: PETSC_EXTERN PetscErrorCode DMRefine_Moab(DM dm, MPI_Comm comm, DM *dmRefined);
     30: PETSC_EXTERN PetscErrorCode DMCoarsen_Moab(DM dm, MPI_Comm comm, DM *dmCoarsened);
     31: PETSC_EXTERN PetscErrorCode DMRefineHierarchy_Moab(DM dm, PetscInt nlevels, DM dmRefined[]);
     32: PETSC_EXTERN PetscErrorCode DMCoarsenHierarchy_Moab(DM dm, PetscInt nlevels, DM dmCoarsened[]);
     33: PETSC_EXTERN PetscErrorCode DMClone_Moab(DM dm, DM *newdm);
     34: PETSC_EXTERN PetscErrorCode DMCreateGlobalVector_Moab(DM, Vec *);
     35: PETSC_EXTERN PetscErrorCode DMCreateLocalVector_Moab(DM, Vec *);
     36: PETSC_EXTERN PetscErrorCode DMCreateMatrix_Moab(DM dm, Mat *J);
     37: PETSC_EXTERN PetscErrorCode DMGlobalToLocalBegin_Moab(DM, Vec, InsertMode, Vec);
     38: PETSC_EXTERN PetscErrorCode DMGlobalToLocalEnd_Moab(DM, Vec, InsertMode, Vec);
     39: PETSC_EXTERN PetscErrorCode DMLocalToGlobalBegin_Moab(DM, Vec, InsertMode, Vec);
     40: PETSC_EXTERN PetscErrorCode DMLocalToGlobalEnd_Moab(DM, Vec, InsertMode, Vec);

     43: /* Un-implemented routines */
     44: /*
     45: PETSC_EXTERN PetscErrorCode DMCreatelocalsection_Moab(DM dm);
     46: PETSC_EXTERN PetscErrorCode DMCreateInjection_Moab(DM dmCoarse, DM dmFine, Mat *mat);
     47: PETSC_EXTERN PetscErrorCode DMLoad_Moab(DM dm, PetscViewer viewer);
     48: PETSC_EXTERN PetscErrorCode DMGetDimPoints_Moab(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd);
     49: PETSC_EXTERN PetscErrorCode DMCreateSubDM_Moab(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm);
     50: PETSC_EXTERN PetscErrorCode DMLocatePoints_Moab(DM dm, Vec v, IS *cellIS);
     51: */

     53: /*@C
     54:   DMMoabCreate - Creates a DMMoab object, which encapsulates a moab instance

     56:   Collective

     58:   Input Parameter:
     59: . comm - The communicator for the DMMoab object

     61:   Output Parameter:
     62: . dmb  - The DMMoab object

     64:   Level: beginner

     66: @*/
     67: PetscErrorCode DMMoabCreate(MPI_Comm comm, DM *dmb)
     68: {

     73:   DMCreate(comm, dmb);
     74:   DMSetType(*dmb, DMMOAB);
     75:   return(0);
     76: }

     78: /*@C
     79:   DMMoabCreateMoab - Creates a DMMoab object, optionally from an instance and other data

     81:   Collective

     83:   Input Parameter:
     84: + comm - The communicator for the DMMoab object
     85: . mbiface - (ptr to) the MOAB Instance; if passed in NULL, MOAB instance is created inside PETSc, and destroyed
     86:          along with the DMMoab
     87: . pcomm - (ptr to) a ParallelComm; if NULL, creates one internally for the whole communicator
     88: . ltog_tag - A tag to use to retrieve global id for an entity; if 0, will use GLOBAL_ID_TAG_NAME/tag
     89: - range - If non-NULL, contains range of entities to which DOFs will be assigned

     91:   Output Parameter:
     92: . dmb  - The DMMoab object

     94:   Level: intermediate

     96: @*/
     97: PetscErrorCode DMMoabCreateMoab(MPI_Comm comm, moab::Interface *mbiface, moab::Tag *ltog_tag, moab::Range *range, DM *dmb)
     98: {
    100:   moab::ErrorCode merr;
    101:   DM             dmmb;
    102:   DM_Moab        *dmmoab;

    107:   DMMoabCreate(comm, &dmmb);
    108:   dmmoab = (DM_Moab*)(dmmb)->data;

    110:   if (!mbiface) {
    111:     dmmoab->mbiface = new moab::Core();
    112:     dmmoab->icreatedinstance = PETSC_TRUE;
    113:   }
    114:   else {
    115:     dmmoab->mbiface = mbiface;
    116:     dmmoab->icreatedinstance = PETSC_FALSE;
    117:   }

    119:   /* by default the fileset = root set. This set stores the hierarchy of entities belonging to current DM */
    120:   dmmoab->fileset = 0;
    121:   dmmoab->hlevel = 0;
    122:   dmmoab->nghostrings = 0;

    124: #ifdef MOAB_HAVE_MPI
    125:   moab::EntityHandle partnset;

    127:   /* Create root sets for each mesh.  Then pass these
    128:       to the load_file functions to be populated. */
    129:   merr = dmmoab->mbiface->create_meshset(moab::MESHSET_SET, partnset); MBERR("Creating partition set failed", merr);

    131:   /* Create the parallel communicator object with the partition handle associated with MOAB */
    132:   dmmoab->pcomm = moab::ParallelComm::get_pcomm(dmmoab->mbiface, partnset, &comm);
    133: #endif

    135:   /* do the remaining initializations for DMMoab */
    136:   dmmoab->bs = 1;
    137:   dmmoab->numFields = 1;
    138:   PetscMalloc(dmmoab->numFields * sizeof(char*), &dmmoab->fieldNames);
    139:   PetscStrallocpy("DEFAULT", (char**) &dmmoab->fieldNames[0]);
    140:   dmmoab->rw_dbglevel = 0;
    141:   dmmoab->partition_by_rank = PETSC_FALSE;
    142:   dmmoab->extra_read_options[0] = '\0';
    143:   dmmoab->extra_write_options[0] = '\0';
    144:   dmmoab->read_mode = READ_PART;
    145:   dmmoab->write_mode = WRITE_PART;

    147:   /* set global ID tag handle */
    148:   if (ltog_tag && *ltog_tag) {
    149:     DMMoabSetLocalToGlobalTag(dmmb, *ltog_tag);
    150:   }
    151:   else {
    152:     merr = dmmoab->mbiface->tag_get_handle(GLOBAL_ID_TAG_NAME, dmmoab->ltog_tag); MBERRNM(merr);
    153:     if (ltog_tag) *ltog_tag = dmmoab->ltog_tag;
    154:   }

    156:   merr = dmmoab->mbiface->tag_get_handle(MATERIAL_SET_TAG_NAME, dmmoab->material_tag); MBERRNM(merr);

    158:   /* set the local range of entities (vertices) of interest */
    159:   if (range) {
    160:     DMMoabSetLocalVertices(dmmb, range);
    161:   }
    162:   *dmb = dmmb;
    163:   return(0);
    164: }

    167: #ifdef MOAB_HAVE_MPI

    169: /*@C
    170:   DMMoabGetParallelComm - Get the ParallelComm used with this DMMoab

    172:   Collective

    174:   Input Parameter:
    175: . dm    - The DMMoab object being set

    177:   Output Parameter:
    178: . pcomm - The ParallelComm for the DMMoab

    180:   Level: beginner

    182: @*/
    183: PetscErrorCode DMMoabGetParallelComm(DM dm, moab::ParallelComm **pcomm)
    184: {
    187:   *pcomm = ((DM_Moab*)(dm)->data)->pcomm;
    188:   return(0);
    189: }

    191: #endif /* MOAB_HAVE_MPI */

    194: /*@C
    195:   DMMoabSetInterface - Set the MOAB instance used with this DMMoab

    197:   Collective

    199:   Input Parameter:
    200: + dm      - The DMMoab object being set
    201: - mbiface - The MOAB instance being set on this DMMoab

    203:   Level: beginner

    205: @*/
    206: PetscErrorCode DMMoabSetInterface(DM dm, moab::Interface *mbiface)
    207: {
    208:   DM_Moab        *dmmoab = (DM_Moab*)(dm)->data;

    213: #ifdef MOAB_HAVE_MPI
    214:   dmmoab->pcomm = NULL;
    215: #endif
    216:   dmmoab->mbiface = mbiface;
    217:   dmmoab->icreatedinstance = PETSC_FALSE;
    218:   return(0);
    219: }

    222: /*@C
    223:   DMMoabGetInterface - Get the MOAB instance used with this DMMoab

    225:   Collective

    227:   Input Parameter:
    228: . dm      - The DMMoab object being set

    230:   Output Parameter:
    231: . mbiface - The MOAB instance set on this DMMoab

    233:   Level: beginner

    235: @*/
    236: PetscErrorCode DMMoabGetInterface(DM dm, moab::Interface **mbiface)
    237: {
    238:   PetscErrorCode   ierr;
    239:   static PetscBool cite = PETSC_FALSE;

    243:   PetscCitationsRegister("@techreport{tautges_moab:_2004,\n  type = {{SAND2004-1592}},\n  title = {{MOAB:} A Mesh-Oriented Database},  institution = {Sandia National Laboratories},\n  author = {Tautges, T. J. and Meyers, R. and Merkley, K. and Stimpson, C. and Ernst, C.},\n  year = {2004},  note = {Report}\n}\n", &cite);
    244:   *mbiface = ((DM_Moab*)dm->data)->mbiface;
    245:   return(0);
    246: }

    249: /*@C
    250:   DMMoabSetLocalVertices - Set the entities having DOFs on this DMMoab

    252:   Collective

    254:   Input Parameter:
    255: + dm    - The DMMoab object being set
    256: - range - The entities treated by this DMMoab

    258:   Level: beginner

    260: @*/
    261: PetscErrorCode DMMoabSetLocalVertices(DM dm, moab::Range *range)
    262: {
    263:   moab::Range     tmpvtxs;
    264:   DM_Moab        *dmmoab = (DM_Moab*)(dm)->data;

    268:   dmmoab->vlocal->clear();
    269:   dmmoab->vowned->clear();

    271:   dmmoab->vlocal->insert(range->begin(), range->end());

    273: #ifdef MOAB_HAVE_MPI
    274:   moab::ErrorCode merr;
    275:   /* filter based on parallel status */
    276:   merr = dmmoab->pcomm->filter_pstatus(*dmmoab->vlocal, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, dmmoab->vowned); MBERRNM(merr);

    278:   /* filter all the non-owned and shared entities out of the list */
    279:   tmpvtxs = moab::subtract(*dmmoab->vlocal, *dmmoab->vowned);
    280:   merr = dmmoab->pcomm->filter_pstatus(tmpvtxs, PSTATUS_INTERFACE, PSTATUS_OR, -1, dmmoab->vghost); MBERRNM(merr);
    281:   tmpvtxs = moab::subtract(tmpvtxs, *dmmoab->vghost);
    282:   *dmmoab->vlocal = moab::subtract(*dmmoab->vlocal, tmpvtxs);
    283: #else
    284:   *dmmoab->vowned = *dmmoab->vlocal;
    285: #endif

    287:   /* compute and cache the sizes of local and ghosted entities */
    288:   dmmoab->nloc = dmmoab->vowned->size();
    289:   dmmoab->nghost = dmmoab->vghost->size();
    290: #ifdef MOAB_HAVE_MPI
    291:   PetscErrorCode  ierr;
    292:   MPIU_Allreduce(&dmmoab->nloc, &dmmoab->n, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
    293: #else
    294:   dmmoab->n = dmmoab->nloc;
    295: #endif
    296:   return(0);
    297: }

    300: /*@C
    301:   DMMoabGetAllVertices - Get the entities having DOFs on this DMMoab

    303:   Collective

    305:   Input Parameter:
    306: . dm    - The DMMoab object being set

    308:   Output Parameter:
    309: . owned - The local vertex entities in this DMMoab = (owned+ghosted)

    311:   Level: beginner

    313: @*/
    314: PetscErrorCode DMMoabGetAllVertices(DM dm, moab::Range *local)
    315: {
    318:   if (local) *local = *((DM_Moab*)dm->data)->vlocal;
    319:   return(0);
    320: }

    324: /*@C
    325:   DMMoabGetLocalVertices - Get the entities having DOFs on this DMMoab

    327:   Collective

    329:   Input Parameter:
    330: . dm    - The DMMoab object being set

    332:   Output Parameters:
    333: + owned - The owned vertex entities in this DMMoab
    334: - ghost - The ghosted entities (non-owned) stored locally in this partition

    336:   Level: beginner

    338: @*/
    339: PetscErrorCode DMMoabGetLocalVertices(DM dm, const moab::Range **owned, const moab::Range **ghost)
    340: {
    343:   if (owned) *owned = ((DM_Moab*)dm->data)->vowned;
    344:   if (ghost) *ghost = ((DM_Moab*)dm->data)->vghost;
    345:   return(0);
    346: }

    348: /*@C
    349:   DMMoabGetLocalElements - Get the higher-dimensional entities that are locally owned

    351:   Collective

    353:   Input Parameter:
    354: . dm    - The DMMoab object being set

    356:   Output Parameter:
    357: . range - The entities owned locally

    359:   Level: beginner

    361: @*/
    362: PetscErrorCode DMMoabGetLocalElements(DM dm, const moab::Range **range)
    363: {
    366:   if (range) *range = ((DM_Moab*)dm->data)->elocal;
    367:   return(0);
    368: }

    371: /*@C
    372:   DMMoabSetLocalElements - Set the entities having DOFs on this DMMoab

    374:   Collective

    376:   Input Parameters:
    377: + dm    - The DMMoab object being set
    378: - range - The entities treated by this DMMoab

    380:   Level: beginner

    382: @*/
    383: PetscErrorCode DMMoabSetLocalElements(DM dm, moab::Range *range)
    384: {
    385:   DM_Moab        *dmmoab = (DM_Moab*)(dm)->data;

    389:   dmmoab->elocal->clear();
    390:   dmmoab->eghost->clear();
    391:   dmmoab->elocal->insert(range->begin(), range->end());
    392: #ifdef MOAB_HAVE_MPI
    393:   moab::ErrorCode merr;
    394:   merr = dmmoab->pcomm->filter_pstatus(*dmmoab->elocal, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    395:   *dmmoab->eghost = moab::subtract(*range, *dmmoab->elocal);
    396: #endif
    397:   dmmoab->neleloc = dmmoab->elocal->size();
    398:   dmmoab->neleghost = dmmoab->eghost->size();
    399: #ifdef MOAB_HAVE_MPI
    400:   PetscErrorCode  ierr;
    401:   MPIU_Allreduce(&dmmoab->neleloc, &dmmoab->nele, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
    402:   PetscInfo2(dm, "Created %D local and %D global elements.\n", dmmoab->neleloc, dmmoab->nele);
    403: #else
    404:   dmmoab->nele = dmmoab->neleloc;
    405: #endif
    406:   return(0);
    407: }

    410: /*@C
    411:   DMMoabSetLocalToGlobalTag - Set the tag used for local to global numbering

    413:   Collective

    415:   Input Parameters:
    416: + dm      - The DMMoab object being set
    417: - ltogtag - The MOAB tag used for local to global ids

    419:   Level: beginner

    421: @*/
    422: PetscErrorCode DMMoabSetLocalToGlobalTag(DM dm, moab::Tag ltogtag)
    423: {
    426:   ((DM_Moab*)dm->data)->ltog_tag = ltogtag;
    427:   return(0);
    428: }

    431: /*@C
    432:   DMMoabGetLocalToGlobalTag - Get the tag used for local to global numbering

    434:   Collective

    436:   Input Parameter:
    437: . dm      - The DMMoab object being set

    439:   Output Parameter:
    440: . ltogtag - The MOAB tag used for local to global ids

    442:   Level: beginner

    444: @*/
    445: PetscErrorCode DMMoabGetLocalToGlobalTag(DM dm, moab::Tag *ltog_tag)
    446: {
    449:   *ltog_tag = ((DM_Moab*)dm->data)->ltog_tag;
    450:   return(0);
    451: }

    454: /*@C
    455:   DMMoabSetBlockSize - Set the block size used with this DMMoab

    457:   Collective

    459:   Input Parameter:
    460: + dm - The DMMoab object being set
    461: - bs - The block size used with this DMMoab

    463:   Level: beginner

    465: @*/
    466: PetscErrorCode DMMoabSetBlockSize(DM dm, PetscInt bs)
    467: {
    470:   ((DM_Moab*)dm->data)->bs = bs;
    471:   return(0);
    472: }

    475: /*@C
    476:   DMMoabGetBlockSize - Get the block size used with this DMMoab

    478:   Collective

    480:   Input Parameter:
    481: . dm - The DMMoab object being set

    483:   Output Parameter:
    484: . bs - The block size used with this DMMoab

    486:   Level: beginner

    488: @*/
    489: PetscErrorCode DMMoabGetBlockSize(DM dm, PetscInt *bs)
    490: {
    493:   *bs = ((DM_Moab*)dm->data)->bs;
    494:   return(0);
    495: }

    498: /*@C
    499:   DMMoabGetSize - Get the global vertex size used with this DMMoab

    501:   Collective on dm

    503:   Input Parameter:
    504: . dm - The DMMoab object being set

    506:   Output Parameter:
    507: + neg - The number of global elements in the DMMoab instance
    508: - nvg - The number of global vertices in the DMMoab instance

    510:   Level: beginner

    512: @*/
    513: PetscErrorCode DMMoabGetSize(DM dm, PetscInt *neg, PetscInt *nvg)
    514: {
    517:   if (neg) *neg = ((DM_Moab*)dm->data)->nele;
    518:   if (nvg) *nvg = ((DM_Moab*)dm->data)->n;
    519:   return(0);
    520: }

    523: /*@C
    524:   DMMoabGetLocalSize - Get the local and ghosted vertex size used with this DMMoab

    526:   Collective on dm

    528:   Input Parameter:
    529: . dm - The DMMoab object being set

    531:   Output Parameter:
    532: + nel - The number of owned elements in this processor
    533: . neg - The number of ghosted elements in this processor
    534: . nvl - The number of owned vertices in this processor
    535: - nvg - The number of ghosted vertices in this processor

    537:   Level: beginner

    539: @*/
    540: PetscErrorCode DMMoabGetLocalSize(DM dm, PetscInt *nel, PetscInt *neg, PetscInt *nvl, PetscInt *nvg)
    541: {
    544:   if (nel) *nel = ((DM_Moab*)dm->data)->neleloc;
    545:   if (neg) *neg = ((DM_Moab*)dm->data)->neleghost;
    546:   if (nvl) *nvl = ((DM_Moab*)dm->data)->nloc;
    547:   if (nvg) *nvg = ((DM_Moab*)dm->data)->nghost;
    548:   return(0);
    549: }

    552: /*@C
    553:   DMMoabGetOffset - Get the local offset for the global vector

    555:   Collective

    557:   Input Parameter:
    558: . dm - The DMMoab object being set

    560:   Output Parameter:
    561: . offset - The local offset for the global vector

    563:   Level: beginner

    565: @*/
    566: PetscErrorCode DMMoabGetOffset(DM dm, PetscInt *offset)
    567: {
    570:   *offset = ((DM_Moab*)dm->data)->vstart;
    571:   return(0);
    572: }

    575: /*@C
    576:   DMMoabGetDimension - Get the dimension of the DM Mesh

    578:   Collective

    580:   Input Parameter:
    581: . dm - The DMMoab object

    583:   Output Parameter:
    584: . dim - The dimension of DM

    586:   Level: beginner

    588: @*/
    589: PetscErrorCode DMMoabGetDimension(DM dm, PetscInt *dim)
    590: {
    593:   *dim = ((DM_Moab*)dm->data)->dim;
    594:   return(0);
    595: }

    598: /*@C
    599:   DMMoabGetHierarchyLevel - Get the current level of the mesh hierarchy
    600:   generated through uniform refinement.

    602:   Collective on dm

    604:   Input Parameter:
    605: . dm - The DMMoab object being set

    607:   Output Parameter:
    608: . nvg - The current mesh hierarchy level

    610:   Level: beginner

    612: @*/
    613: PetscErrorCode DMMoabGetHierarchyLevel(DM dm, PetscInt *nlevel)
    614: {
    617:   if (nlevel) *nlevel = ((DM_Moab*)dm->data)->hlevel;
    618:   return(0);
    619: }

    622: /*@C
    623:   DMMoabGetMaterialBlock - Get the material ID corresponding to the current entity of the DM Mesh

    625:   Collective

    627:   Input Parameter:
    628: + dm - The DMMoab object
    629: - ehandle - The element entity handle

    631:   Output Parameter:
    632: . mat - The material ID for the current entity

    634:   Level: beginner

    636: @*/
    637: PetscErrorCode DMMoabGetMaterialBlock(DM dm, const moab::EntityHandle ehandle, PetscInt *mat)
    638: {
    639:   DM_Moab         *dmmoab;

    643:   if (*mat) {
    644:     dmmoab = (DM_Moab*)(dm)->data;
    645:     *mat = dmmoab->materials[dmmoab->elocal->index(ehandle)];
    646:   }
    647:   return(0);
    648: }

    651: /*@C
    652:   DMMoabGetVertexCoordinates - Get the coordinates corresponding to the requested vertex entities

    654:   Collective

    656:   Input Parameter:
    657: + dm - The DMMoab object
    658: . nconn - Number of entities whose coordinates are needed
    659: - conn - The vertex entity handles

    661:   Output Parameter:
    662: . vpos - The coordinates of the requested vertex entities

    664:   Level: beginner

    666: .seealso: DMMoabGetVertexConnectivity()
    667: @*/
    668: PetscErrorCode DMMoabGetVertexCoordinates(DM dm, PetscInt nconn, const moab::EntityHandle *conn, PetscReal *vpos)
    669: {
    670:   DM_Moab         *dmmoab;
    671:   moab::ErrorCode merr;

    677:   dmmoab = (DM_Moab*)(dm)->data;

    679:   /* Get connectivity information in MOAB canonical ordering */
    680:   if (dmmoab->hlevel) {
    681:     merr = dmmoab->hierarchy->get_coordinates(const_cast<moab::EntityHandle*>(conn), nconn, dmmoab->hlevel, vpos);MBERRNM(merr);
    682:   }
    683:   else {
    684:     merr = dmmoab->mbiface->get_coords(conn, nconn, vpos);MBERRNM(merr);
    685:   }
    686:   return(0);
    687: }

    690: /*@C
    691:   DMMoabGetVertexConnectivity - Get the vertex adjacency for the given entity

    693:   Collective

    695:   Input Parameter:
    696: + dm - The DMMoab object
    697: - vhandle - Vertex entity handle

    699:   Output Parameter:
    700: + nconn - Number of entities whose coordinates are needed
    701: - conn - The vertex entity handles

    703:   Level: beginner

    705: .seealso: DMMoabGetVertexCoordinates(), DMMoabRestoreVertexConnectivity()
    706: @*/
    707: PetscErrorCode DMMoabGetVertexConnectivity(DM dm, moab::EntityHandle vhandle, PetscInt* nconn, moab::EntityHandle **conn)
    708: {
    709:   DM_Moab        *dmmoab;
    710:   std::vector<moab::EntityHandle> adj_entities, connect;
    711:   PetscErrorCode  ierr;
    712:   moab::ErrorCode merr;

    717:   dmmoab = (DM_Moab*)(dm)->data;

    719:   /* Get connectivity information in MOAB canonical ordering */
    720:   merr = dmmoab->mbiface->get_adjacencies(&vhandle, 1, 1, true, adj_entities, moab::Interface::UNION); MBERRNM(merr);
    721:   merr = dmmoab->mbiface->get_connectivity(&adj_entities[0], adj_entities.size(), connect); MBERRNM(merr);

    723:   if (conn) {
    724:     PetscMalloc(sizeof(moab::EntityHandle) * connect.size(), conn);
    725:     PetscArraycpy(*conn, &connect[0], connect.size());
    726:   }
    727:   if (nconn) *nconn = connect.size();
    728:   return(0);
    729: }

    732: /*@C
    733:   DMMoabRestoreVertexConnectivity - Restore the vertex connectivity for the given entity

    735:   Collective

    737:   Input Parameter:
    738: + dm - The DMMoab object
    739: . vhandle - Vertex entity handle
    740: . nconn - Number of entities whose coordinates are needed
    741: - conn - The vertex entity handles

    743:   Level: beginner

    745: .seealso: DMMoabGetVertexCoordinates(), DMMoabGetVertexConnectivity()
    746: @*/
    747: PetscErrorCode DMMoabRestoreVertexConnectivity(DM dm, moab::EntityHandle ehandle, PetscInt* nconn, moab::EntityHandle **conn)
    748: {
    749:   PetscErrorCode  ierr;

    755:   if (conn) {
    756:     PetscFree(*conn);
    757:   }
    758:   if (nconn) *nconn = 0;
    759:   return(0);
    760: }

    763: /*@C
    764:   DMMoabGetElementConnectivity - Get the vertex adjacency for the given entity

    766:   Collective

    768:   Input Parameter:
    769: + dm - The DMMoab object
    770: - ehandle - Vertex entity handle

    772:   Output Parameter:
    773: + nconn - Number of entities whose coordinates are needed
    774: - conn - The vertex entity handles

    776:   Level: beginner

    778: .seealso: DMMoabGetVertexCoordinates(), DMMoabGetVertexConnectivity(), DMMoabRestoreVertexConnectivity()
    779: @*/
    780: PetscErrorCode DMMoabGetElementConnectivity(DM dm, moab::EntityHandle ehandle, PetscInt* nconn, const moab::EntityHandle **conn)
    781: {
    782:   DM_Moab        *dmmoab;
    783:   const moab::EntityHandle *connect;
    784:   std::vector<moab::EntityHandle> vconn;
    785:   moab::ErrorCode merr;
    786:   PetscInt nnodes;

    791:   dmmoab = (DM_Moab*)(dm)->data;

    793:   /* Get connectivity information in MOAB canonical ordering */
    794:   merr = dmmoab->mbiface->get_connectivity(ehandle, connect, nnodes); MBERRNM(merr);
    795:   if (conn) *conn = connect;
    796:   if (nconn) *nconn = nnodes;
    797:   return(0);
    798: }

    801: /*@C
    802:   DMMoabIsEntityOnBoundary - Check whether a given entity is on the boundary (vertex, edge, face, element)

    804:   Collective

    806:   Input Parameter:
    807: + dm - The DMMoab object
    808: - ent - Entity handle

    810:   Output Parameter:
    811: . ent_on_boundary - PETSC_TRUE if entity on boundary; PETSC_FALSE otherwise

    813:   Level: beginner

    815: .seealso: DMMoabCheckBoundaryVertices()
    816: @*/
    817: PetscErrorCode DMMoabIsEntityOnBoundary(DM dm, const moab::EntityHandle ent, PetscBool* ent_on_boundary)
    818: {
    819:   moab::EntityType etype;
    820:   DM_Moab         *dmmoab;
    821:   PetscInt         edim;

    826:   dmmoab = (DM_Moab*)(dm)->data;

    828:   /* get the entity type and handle accordingly */
    829:   etype = dmmoab->mbiface->type_from_handle(ent);
    830:   if (etype >= moab::MBPOLYHEDRON) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_OUTOFRANGE, "Entity type on the boundary skin is invalid. EntityType = %D\n", etype);

    832:   /* get the entity dimension */
    833:   edim = dmmoab->mbiface->dimension_from_handle(ent);

    835:   *ent_on_boundary = PETSC_FALSE;
    836:   if (etype == moab::MBVERTEX && edim == 0) {
    837:     *ent_on_boundary = ((dmmoab->bndyvtx->index(ent) >= 0) ? PETSC_TRUE : PETSC_FALSE);
    838:   }
    839:   else {
    840:     if (edim == dmmoab->dim) {  /* check the higher-dimensional elements first */
    841:       if (dmmoab->bndyelems->index(ent) >= 0) *ent_on_boundary = PETSC_TRUE;
    842:     }
    843:     else {                      /* next check the lower-dimensional faces */
    844:       if (dmmoab->bndyfaces->index(ent) >= 0) *ent_on_boundary = PETSC_TRUE;
    845:     }
    846:   }
    847:   return(0);
    848: }

    851: /*@C
    852:   DMMoabIsEntityOnBoundary - Check whether a given entity is on the boundary (vertex, edge, face, element)

    854:   Input Parameter:
    855: + dm - The DMMoab object
    856: . nconn - Number of handles
    857: - cnt - Array of entity handles

    859:   Output Parameter:
    860: . isbdvtx - Array of boundary markers - PETSC_TRUE if entity on boundary; PETSC_FALSE otherwise

    862:   Level: beginner

    864: .seealso: DMMoabIsEntityOnBoundary()
    865: @*/
    866: PetscErrorCode DMMoabCheckBoundaryVertices(DM dm, PetscInt nconn, const moab::EntityHandle *cnt, PetscBool* isbdvtx)
    867: {
    868:   DM_Moab        *dmmoab;
    869:   PetscInt       i;

    875:   dmmoab = (DM_Moab*)(dm)->data;

    877:   for (i = 0; i < nconn; ++i) {
    878:     isbdvtx[i] = (dmmoab->bndyvtx->index(cnt[i]) >= 0 ? PETSC_TRUE : PETSC_FALSE);
    879:   }
    880:   return(0);
    881: }

    884: /*@C
    885:   DMMoabGetBoundaryMarkers - Return references to the vertices, faces, elements on the boundary

    887:   Input Parameter:
    888: . dm - The DMMoab object

    890:   Output Parameter:
    891: + bdvtx - Boundary vertices
    892: . bdelems - Boundary elements
    893: - bdfaces - Boundary faces

    895:   Level: beginner

    897: .seealso: DMMoabCheckBoundaryVertices(), DMMoabIsEntityOnBoundary()
    898: @*/
    899: PetscErrorCode DMMoabGetBoundaryMarkers(DM dm, const moab::Range **bdvtx, const moab::Range** bdelems, const moab::Range** bdfaces)
    900: {
    901:   DM_Moab        *dmmoab;

    905:   dmmoab = (DM_Moab*)(dm)->data;

    907:   if (bdvtx)  *bdvtx = dmmoab->bndyvtx;
    908:   if (bdfaces)  *bdfaces = dmmoab->bndyfaces;
    909:   if (bdelems)  *bdfaces = dmmoab->bndyelems;
    910:   return(0);
    911: }

    914: PETSC_EXTERN PetscErrorCode DMDestroy_Moab(DM dm)
    915: {
    916:   PetscErrorCode  ierr;
    917:   PetscInt        i;
    918:   moab::ErrorCode merr;
    919:   DM_Moab        *dmmoab = (DM_Moab*)dm->data;

    924:   dmmoab->refct--;
    925:   if (!dmmoab->refct) {
    926:     delete dmmoab->vlocal;
    927:     delete dmmoab->vowned;
    928:     delete dmmoab->vghost;
    929:     delete dmmoab->elocal;
    930:     delete dmmoab->eghost;
    931:     delete dmmoab->bndyvtx;
    932:     delete dmmoab->bndyfaces;
    933:     delete dmmoab->bndyelems;

    935:     PetscFree(dmmoab->gsindices);
    936:     PetscFree2(dmmoab->gidmap, dmmoab->lidmap);
    937:     PetscFree(dmmoab->dfill);
    938:     PetscFree(dmmoab->ofill);
    939:     PetscFree(dmmoab->materials);
    940:     if (dmmoab->fieldNames) {
    941:       for (i = 0; i < dmmoab->numFields; i++) {
    942:         PetscFree(dmmoab->fieldNames[i]);
    943:       }
    944:       PetscFree(dmmoab->fieldNames);
    945:     }

    947:     if (dmmoab->nhlevels) {
    948:       PetscFree(dmmoab->hsets);
    949:       dmmoab->nhlevels = 0;
    950:       if (!dmmoab->hlevel && dmmoab->icreatedinstance) delete dmmoab->hierarchy;
    951:       dmmoab->hierarchy = NULL;
    952:     }

    954:     if (dmmoab->icreatedinstance) {
    955:       delete dmmoab->pcomm;
    956:       merr = dmmoab->mbiface->delete_mesh(); MBERRNM(merr);
    957:       delete dmmoab->mbiface;
    958:     }
    959:     dmmoab->mbiface = NULL;
    960: #ifdef MOAB_HAVE_MPI
    961:     dmmoab->pcomm = NULL;
    962: #endif
    963:     VecScatterDestroy(&dmmoab->ltog_sendrecv);
    964:     ISLocalToGlobalMappingDestroy(&dmmoab->ltog_map);
    965:     PetscFree(dm->data);
    966:   }
    967:   return(0);
    968: }

    971: PETSC_EXTERN PetscErrorCode DMSetFromOptions_Moab(PetscOptionItems *PetscOptionsObject, DM dm)
    972: {
    974:   DM_Moab        *dmmoab = (DM_Moab*)dm->data;

    978:   PetscOptionsHead(PetscOptionsObject, "DMMoab Options");
    979:   PetscOptionsBoundedInt("-dm_moab_rw_dbg", "The verbosity level for reading and writing MOAB meshes", "DMView", dmmoab->rw_dbglevel, &dmmoab->rw_dbglevel, NULL,0);
    980:   PetscOptionsBool("-dm_moab_partiton_by_rank", "Use partition by rank when reading MOAB meshes from file", "DMView", dmmoab->partition_by_rank, &dmmoab->partition_by_rank, NULL);
    981:   /* TODO: typically, the read options are needed before a DM is completely created and available in which case, the options wont be available ?? */
    982:   PetscOptionsString("-dm_moab_read_opts", "Extra options to enable MOAB reader to load DM from file", "DMView", dmmoab->extra_read_options, dmmoab->extra_read_options, sizeof(dmmoab->extra_read_options), NULL);
    983:   PetscOptionsString("-dm_moab_write_opts", "Extra options to enable MOAB writer to serialize DM to file", "DMView", dmmoab->extra_write_options, dmmoab->extra_write_options, sizeof(dmmoab->extra_write_options), NULL);
    984:   PetscOptionsEnum("-dm_moab_read_mode", "MOAB parallel read mode", "DMView", MoabReadModes, (PetscEnum)dmmoab->read_mode, (PetscEnum*)&dmmoab->read_mode, NULL);
    985:   PetscOptionsEnum("-dm_moab_write_mode", "MOAB parallel write mode", "DMView", MoabWriteModes, (PetscEnum)dmmoab->write_mode, (PetscEnum*)&dmmoab->write_mode, NULL);
    986:   return(0);
    987: }

    990: PETSC_EXTERN PetscErrorCode DMSetUp_Moab(DM dm)
    991: {
    992:   PetscErrorCode          ierr;
    993:   moab::ErrorCode         merr;
    994:   Vec                     local, global;
    995:   IS                      from, to;
    996:   moab::Range::iterator   iter;
    997:   PetscInt                i, j, f, bs, vent, totsize, *lgmap;
    998:   DM_Moab                *dmmoab = (DM_Moab*)dm->data;
    999:   moab::Range             adjs;

    1003:   /* Get the local and shared vertices and cache it */
    1004:   if (dmmoab->mbiface == NULL) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ORDER, "Set the MOAB Interface before calling SetUp.");
    1005: #ifdef MOAB_HAVE_MPI
    1006:   if (dmmoab->pcomm == NULL) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ORDER, "Set the MOAB ParallelComm object before calling SetUp.");
    1007: #endif

    1009:   /* Get the entities recursively in the current part of the mesh, if user did not set the local vertices explicitly */
    1010:   if (dmmoab->vlocal->empty())
    1011:   {
    1012:     //merr = dmmoab->mbiface->get_entities_by_type(dmmoab->fileset,moab::MBVERTEX,*dmmoab->vlocal,true);MBERRNM(merr);
    1013:     merr = dmmoab->mbiface->get_entities_by_dimension(dmmoab->fileset, 0, *dmmoab->vlocal, false); MBERRNM(merr);

    1015: #ifdef MOAB_HAVE_MPI
    1016:     /* filter based on parallel status */
    1017:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->vlocal, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, dmmoab->vowned); MBERRNM(merr);

    1019:     /* filter all the non-owned and shared entities out of the list */
    1020:     // *dmmoab->vghost = moab::subtract(*dmmoab->vlocal, *dmmoab->vowned);
    1021:     adjs = moab::subtract(*dmmoab->vlocal, *dmmoab->vowned);
    1022:     merr = dmmoab->pcomm->filter_pstatus(adjs, PSTATUS_GHOST | PSTATUS_INTERFACE, PSTATUS_OR, -1, dmmoab->vghost); MBERRNM(merr);
    1023:     adjs = moab::subtract(adjs, *dmmoab->vghost);
    1024:     *dmmoab->vlocal = moab::subtract(*dmmoab->vlocal, adjs);
    1025: #else
    1026:     *dmmoab->vowned = *dmmoab->vlocal;
    1027: #endif

    1029:     /* compute and cache the sizes of local and ghosted entities */
    1030:     dmmoab->nloc = dmmoab->vowned->size();
    1031:     dmmoab->nghost = dmmoab->vghost->size();

    1033: #ifdef MOAB_HAVE_MPI
    1034:     MPIU_Allreduce(&dmmoab->nloc, &dmmoab->n, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
    1035:     PetscInfo4(NULL, "Filset ID: %u, Vertices: local - %D, owned - %D, ghosted - %D.\n", dmmoab->fileset, dmmoab->vlocal->size(), dmmoab->nloc, dmmoab->nghost);
    1036: #else
    1037:     dmmoab->n = dmmoab->nloc;
    1038: #endif
    1039:   }

    1041:   {
    1042:     /* get the information about the local elements in the mesh */
    1043:     dmmoab->eghost->clear();

    1045:     /* first decipher the leading dimension */
    1046:     for (i = 3; i > 0; i--) {
    1047:       dmmoab->elocal->clear();
    1048:       merr = dmmoab->mbiface->get_entities_by_dimension(dmmoab->fileset, i, *dmmoab->elocal, false); MBERRNM(merr);

    1050:       /* store the current mesh dimension */
    1051:       if (dmmoab->elocal->size()) {
    1052:         dmmoab->dim = i;
    1053:         break;
    1054:       }
    1055:     }

    1057:     DMSetDimension(dm, dmmoab->dim);

    1059: #ifdef MOAB_HAVE_MPI
    1060:     /* filter the ghosted and owned element list */
    1061:     *dmmoab->eghost = *dmmoab->elocal;
    1062:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->elocal, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1063:     *dmmoab->eghost = moab::subtract(*dmmoab->eghost, *dmmoab->elocal);
    1064: #endif

    1066:     dmmoab->neleloc = dmmoab->elocal->size();
    1067:     dmmoab->neleghost = dmmoab->eghost->size();

    1069: #ifdef MOAB_HAVE_MPI
    1070:     MPIU_Allreduce(&dmmoab->neleloc, &dmmoab->nele, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
    1071:     PetscInfo3(NULL, "%d-dim elements: owned - %D, ghosted - %D.\n", dmmoab->dim, dmmoab->neleloc, dmmoab->neleghost);
    1072: #else
    1073:     dmmoab->nele = dmmoab->neleloc;
    1074: #endif
    1075:   }

    1077:   bs = dmmoab->bs;
    1078:   if (!dmmoab->ltog_tag) {
    1079:     /* Get the global ID tag. The global ID tag is applied to each
    1080:        vertex. It acts as an global identifier which MOAB uses to
    1081:        assemble the individual pieces of the mesh */
    1082:     merr = dmmoab->mbiface->tag_get_handle(GLOBAL_ID_TAG_NAME, dmmoab->ltog_tag); MBERRNM(merr);
    1083:   }

    1085:   totsize = dmmoab->vlocal->size();
    1086:   if (totsize != dmmoab->nloc + dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Mismatch between local and owned+ghost vertices. %D != %D.", totsize, dmmoab->nloc + dmmoab->nghost);
    1087:   PetscCalloc1(totsize, &dmmoab->gsindices);
    1088:   {
    1089:     /* first get the local indices */
    1090:     merr = dmmoab->mbiface->tag_get_data(dmmoab->ltog_tag, *dmmoab->vowned, &dmmoab->gsindices[0]); MBERRNM(merr);
    1091:     if (dmmoab->nghost) {  /* next get the ghosted indices */
    1092:       merr = dmmoab->mbiface->tag_get_data(dmmoab->ltog_tag, *dmmoab->vghost, &dmmoab->gsindices[dmmoab->nloc]); MBERRNM(merr);
    1093:     }

    1095:     /* find out the local and global minima of GLOBAL_ID */
    1096:     dmmoab->lminmax[0] = dmmoab->lminmax[1] = dmmoab->gsindices[0];
    1097:     for (i = 0; i < totsize; ++i) {
    1098:       if (dmmoab->lminmax[0] > dmmoab->gsindices[i]) dmmoab->lminmax[0] = dmmoab->gsindices[i];
    1099:       if (dmmoab->lminmax[1] < dmmoab->gsindices[i]) dmmoab->lminmax[1] = dmmoab->gsindices[i];
    1100:     }

    1102:     MPIU_Allreduce(&dmmoab->lminmax[0], &dmmoab->gminmax[0], 1, MPI_INT, MPI_MIN, ((PetscObject)dm)->comm);
    1103:     MPIU_Allreduce(&dmmoab->lminmax[1], &dmmoab->gminmax[1], 1, MPI_INT, MPI_MAX, ((PetscObject)dm)->comm);

    1105:     /* set the GID map */
    1106:     for (i = 0; i < totsize; ++i) {
    1107:       dmmoab->gsindices[i] -= dmmoab->gminmax[0]; /* zero based index needed for IS */

    1109:     }
    1110:     dmmoab->lminmax[0] -= dmmoab->gminmax[0];
    1111:     dmmoab->lminmax[1] -= dmmoab->gminmax[0];

    1113:     PetscInfo4(NULL, "GLOBAL_ID: Local [min, max] - [%D, %D], Global [min, max] - [%D, %D]\n", dmmoab->lminmax[0], dmmoab->lminmax[1], dmmoab->gminmax[0], dmmoab->gminmax[1]);
    1114:   }
    1115:   if (!(dmmoab->bs == dmmoab->numFields || dmmoab->bs == 1)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between block size and number of component fields. %D != 1 OR %D != %D.", dmmoab->bs, dmmoab->bs, dmmoab->numFields);

    1117:   {
    1118:     dmmoab->seqstart = dmmoab->mbiface->id_from_handle(dmmoab->vlocal->front());
    1119:     dmmoab->seqend = dmmoab->mbiface->id_from_handle(dmmoab->vlocal->back());
    1120:     PetscInfo2(NULL, "SEQUENCE: Local [min, max] - [%D, %D]\n", dmmoab->seqstart, dmmoab->seqend);

    1122:     PetscMalloc2(dmmoab->seqend - dmmoab->seqstart + 1, &dmmoab->gidmap, dmmoab->seqend - dmmoab->seqstart + 1, &dmmoab->lidmap);
    1123:     PetscMalloc1(totsize * dmmoab->numFields, &lgmap);

    1125:     i = j = 0;
    1126:     /* set the owned vertex data first */
    1127:     for (moab::Range::iterator iter = dmmoab->vowned->begin(); iter != dmmoab->vowned->end(); iter++, i++) {
    1128:       vent = dmmoab->mbiface->id_from_handle(*iter) - dmmoab->seqstart;
    1129:       dmmoab->gidmap[vent] = dmmoab->gsindices[i];
    1130:       dmmoab->lidmap[vent] = i;
    1131:       for (f = 0; f < dmmoab->numFields; f++, j++) {
    1132:         lgmap[j] = (bs > 1 ? dmmoab->gsindices[i] * dmmoab->numFields + f : totsize * f + dmmoab->gsindices[i]);
    1133:       }
    1134:     }
    1135:     /* next arrange all the ghosted data information */
    1136:     for (moab::Range::iterator iter = dmmoab->vghost->begin(); iter != dmmoab->vghost->end(); iter++, i++) {
    1137:       vent = dmmoab->mbiface->id_from_handle(*iter) - dmmoab->seqstart;
    1138:       dmmoab->gidmap[vent] = dmmoab->gsindices[i];
    1139:       dmmoab->lidmap[vent] = i;
    1140:       for (f = 0; f < dmmoab->numFields; f++, j++) {
    1141:         lgmap[j] = (bs > 1 ? dmmoab->gsindices[i] * dmmoab->numFields + f : totsize * f + dmmoab->gsindices[i]);
    1142:       }
    1143:     }

    1145:     /* We need to create the Global to Local Vector Scatter Contexts
    1146:        1) First create a local and global vector
    1147:        2) Create a local and global IS
    1148:        3) Create VecScatter and LtoGMapping objects
    1149:        4) Cleanup the IS and Vec objects
    1150:     */
    1151:     DMCreateGlobalVector(dm, &global);
    1152:     DMCreateLocalVector(dm, &local);

    1154:     VecGetOwnershipRange(global, &dmmoab->vstart, &dmmoab->vend);

    1156:     /* global to local must retrieve ghost points */
    1157:     ISCreateStride(((PetscObject)dm)->comm, dmmoab->nloc * dmmoab->numFields, dmmoab->vstart, 1, &from);
    1158:     ISSetBlockSize(from, bs);

    1160:     ISCreateGeneral(((PetscObject)dm)->comm, dmmoab->nloc * dmmoab->numFields, &lgmap[0], PETSC_COPY_VALUES, &to);
    1161:     ISSetBlockSize(to, bs);

    1163:     if (!dmmoab->ltog_map) {
    1164:       /* create to the local to global mapping for vectors in order to use VecSetValuesLocal */
    1165:       ISLocalToGlobalMappingCreate(((PetscObject)dm)->comm, dmmoab->bs, totsize * dmmoab->numFields, lgmap,
    1166:                                           PETSC_COPY_VALUES, &dmmoab->ltog_map);
    1167:     }

    1169:     /* now create the scatter object from local to global vector */
    1170:     VecScatterCreate(local, from, global, to, &dmmoab->ltog_sendrecv);

    1172:     /* clean up IS, Vec */
    1173:     PetscFree(lgmap);
    1174:     ISDestroy(&from);
    1175:     ISDestroy(&to);
    1176:     VecDestroy(&local);
    1177:     VecDestroy(&global);
    1178:   }

    1180:   dmmoab->bndyvtx = new moab::Range();
    1181:   dmmoab->bndyfaces = new moab::Range();
    1182:   dmmoab->bndyelems = new moab::Range();
    1183:   /* skin the boundary and store nodes */
    1184:   if (!dmmoab->hlevel) {
    1185:     /* get the skin vertices of boundary faces for the current partition and then filter
    1186:        the local, boundary faces, vertices and elements alone via PSTATUS flags;
    1187:        this should not give us any ghosted boundary, but if user needs such a functionality
    1188:        it would be easy to add it based on the find_skin query below */
    1189:     moab::Skinner skinner(dmmoab->mbiface);

    1191:     /* get the entities on the skin - only the faces */
    1192:     merr = skinner.find_skin(dmmoab->fileset, *dmmoab->elocal, false, *dmmoab->bndyfaces, NULL, true, true, false); MBERRNM(merr); // 'false' param indicates we want faces back, not vertices

    1194: #ifdef MOAB_HAVE_MPI
    1195:     /* filter all the non-owned and shared entities out of the list */
    1196:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyfaces, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1197:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyfaces, PSTATUS_INTERFACE, PSTATUS_NOT); MBERRNM(merr);
    1198: #endif

    1200:     /* get all the nodes via connectivity and the parent elements via adjacency information */
    1201:     merr = dmmoab->mbiface->get_connectivity(*dmmoab->bndyfaces, *dmmoab->bndyvtx, false); MBERRNM(ierr);
    1202:     merr = dmmoab->mbiface->get_adjacencies(*dmmoab->bndyvtx, dmmoab->dim, false, *dmmoab->bndyelems, moab::Interface::UNION); MBERRNM(ierr);
    1203:   }
    1204:   else {
    1205:     /* Let us query the hierarchy manager and get the results directly for this level */
    1206:     for (moab::Range::iterator iter = dmmoab->elocal->begin(); iter != dmmoab->elocal->end(); iter++) {
    1207:       moab::EntityHandle elemHandle = *iter;
    1208:       if (dmmoab->hierarchy->is_entity_on_boundary(elemHandle)) {
    1209:         dmmoab->bndyelems->insert(elemHandle);
    1210:         /* For this boundary element, query the vertices and add them to the list */
    1211:         std::vector<moab::EntityHandle> connect;
    1212:         merr = dmmoab->hierarchy->get_connectivity(elemHandle, dmmoab->hlevel, connect); MBERRNM(ierr);
    1213:         for (unsigned iv=0; iv < connect.size(); ++iv)
    1214:           if (dmmoab->hierarchy->is_entity_on_boundary(connect[iv]))
    1215:             dmmoab->bndyvtx->insert(connect[iv]);
    1216:         /* Next, let us query the boundary faces and add them also to the list */
    1217:         std::vector<moab::EntityHandle> faces;
    1218:         merr = dmmoab->hierarchy->get_adjacencies(elemHandle, dmmoab->dim-1, faces); MBERRNM(ierr);
    1219:         for (unsigned ifa=0; ifa < faces.size(); ++ifa)
    1220:           if (dmmoab->hierarchy->is_entity_on_boundary(faces[ifa]))
    1221:             dmmoab->bndyfaces->insert(faces[ifa]);
    1222:       }
    1223:     }
    1224: #ifdef MOAB_HAVE_MPI
    1225:     /* filter all the non-owned and shared entities out of the list */
    1226:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyvtx,   PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1227:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyfaces, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1228:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyelems, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1229: #endif

    1231:   }
    1232:   PetscInfo3(NULL, "Found %D boundary vertices, %D boundary faces and %D boundary elements.\n", dmmoab->bndyvtx->size(), dmmoab->bndyfaces->size(), dmmoab->bndyelems->size());

    1234:   /* Get the material sets and populate the data for all locally owned elements */
    1235:   {
    1236:     PetscCalloc1(dmmoab->elocal->size(), &dmmoab->materials);
    1237:     /* Get the count of entities of particular type from dmmoab->elocal
    1238:        -- Then, for each non-zero type, loop through and query the fileset to get the material tag data */
    1239:     moab::Range msets;
    1240:     merr = dmmoab->mbiface->get_entities_by_type_and_tag(dmmoab->fileset, moab::MBENTITYSET, &dmmoab->material_tag, NULL, 1, msets, moab::Interface::UNION);MB_CHK_ERR(merr);
    1241:     if (msets.size() == 0) {
    1242:       PetscInfo(NULL, "No material sets found in the fileset.");
    1243:     }

    1245:     for (unsigned i=0; i < msets.size(); ++i) {
    1246:       moab::Range msetelems;
    1247:       merr = dmmoab->mbiface->get_entities_by_dimension(msets[i], dmmoab->dim, msetelems, true);MB_CHK_ERR(merr);
    1248: #ifdef MOAB_HAVE_MPI
    1249:       /* filter all the non-owned and shared entities out of the list */
    1250:       merr = dmmoab->pcomm->filter_pstatus(msetelems, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1251: #endif

    1253:       int partID;
    1254:       moab::EntityHandle mset=msets[i];
    1255:       merr = dmmoab->mbiface->tag_get_data(dmmoab->material_tag, &mset, 1, &partID);MB_CHK_ERR(merr);

    1257:       for (unsigned j=0; j < msetelems.size(); ++j)
    1258:         dmmoab->materials[dmmoab->elocal->index(msetelems[j])]=partID;
    1259:     }
    1260:   }

    1262:   return(0);
    1263: }

    1266: /*@C
    1267:   DMMoabCreateVertices - Creates and adds several vertices to the primary set represented by the DM.

    1269:   Collective

    1271:   Input Parameters:
    1272: + dm - The DM object
    1273: . type - The type of element to create and add (Edge/Tri/Quad/Tet/Hex/Prism/Pyramid/Polygon/Polyhedra)
    1274: . conn - The connectivity of the element
    1275: - nverts - The number of vertices that form the element

    1277:   Output Parameter:
    1278: . overts  - The list of vertices that were created (can be NULL)

    1280:   Level: beginner

    1282: .seealso: DMMoabCreateSubmesh(), DMMoabCreateElement()
    1283: @*/
    1284: PetscErrorCode DMMoabCreateVertices(DM dm, const PetscReal* coords, PetscInt nverts, moab::Range* overts)
    1285: {
    1286:   moab::ErrorCode     merr;
    1287:   DM_Moab            *dmmoab;
    1288:   moab::Range         verts;

    1294:   dmmoab = (DM_Moab*) dm->data;

    1296:   /* Insert new points */
    1297:   merr = dmmoab->mbiface->create_vertices(&coords[0], nverts, verts); MBERRNM(merr);
    1298:   merr = dmmoab->mbiface->add_entities(dmmoab->fileset, verts); MBERRNM(merr);

    1300:   if (overts) *overts = verts;
    1301:   return(0);
    1302: }

    1305: /*@C
    1306:   DMMoabCreateElement - Adds an element of specified type to the primary set represented by the DM.

    1308:   Collective

    1310:   Input Parameters:
    1311: + dm - The DM object
    1312: . type - The type of element to create and add (Edge/Tri/Quad/Tet/Hex/Prism/Pyramid/Polygon/Polyhedra)
    1313: . conn - The connectivity of the element
    1314: - nverts - The number of vertices that form the element

    1316:   Output Parameter:
    1317: . oelem  - The handle to the element created and added to the DM object

    1319:   Level: beginner

    1321: .seealso: DMMoabCreateSubmesh(), DMMoabCreateVertices()
    1322: @*/
    1323: PetscErrorCode DMMoabCreateElement(DM dm, const moab::EntityType type, const moab::EntityHandle* conn, PetscInt nverts, moab::EntityHandle* oelem)
    1324: {
    1325:   moab::ErrorCode     merr;
    1326:   DM_Moab            *dmmoab;
    1327:   moab::EntityHandle  elem;

    1333:   dmmoab = (DM_Moab*) dm->data;

    1335:   /* Insert new element */
    1336:   merr = dmmoab->mbiface->create_element(type, conn, nverts, elem); MBERRNM(merr);
    1337:   merr = dmmoab->mbiface->add_entities(dmmoab->fileset, &elem, 1); MBERRNM(merr);

    1339:   if (oelem) *oelem = elem;
    1340:   return(0);
    1341: }

    1344: /*@C
    1345:   DMMoabCreateSubmesh - Creates a sub-DM object with a set that contains all vertices/elements of the parent
    1346:   in addition to providing support for dynamic mesh modifications. This is useful for AMR calculations to
    1347:   create a DM object on a refined level.

    1349:   Collective

    1351:   Input Parameters:
    1352: . dm - The DM object

    1354:   Output Parameter:
    1355: . newdm  - The sub DM object with updated set information

    1357:   Level: advanced

    1359: .seealso: DMCreate(), DMMoabCreateVertices(), DMMoabCreateElement()
    1360: @*/
    1361: PetscErrorCode DMMoabCreateSubmesh(DM dm, DM *newdm)
    1362: {
    1363:   DM_Moab            *dmmoab;
    1364:   DM_Moab            *ndmmoab;
    1365:   moab::ErrorCode    merr;
    1366:   PetscErrorCode     ierr;

    1371:   dmmoab = (DM_Moab*) dm->data;

    1373:   /* Create the basic DMMoab object and keep the default parameters created by DM impls */
    1374:   DMMoabCreateMoab(((PetscObject)dm)->comm, dmmoab->mbiface, &dmmoab->ltog_tag, PETSC_NULL, newdm);

    1376:   /* get all the necessary handles from the private DM object */
    1377:   ndmmoab = (DM_Moab*) (*newdm)->data;

    1379:   /* set the sub-mesh's parent DM reference */
    1380:   ndmmoab->parent = &dm;

    1382:   /* create a file set to associate all entities in current mesh */
    1383:   merr = ndmmoab->mbiface->create_meshset(moab::MESHSET_SET, ndmmoab->fileset); MBERR("Creating file set failed", merr);

    1385:   /* create a meshset and then add old fileset as child */
    1386:   merr = ndmmoab->mbiface->add_entities(ndmmoab->fileset, *dmmoab->vlocal); MBERR("Adding child vertices to parent failed", merr);
    1387:   merr = ndmmoab->mbiface->add_entities(ndmmoab->fileset, *dmmoab->elocal); MBERR("Adding child elements to parent failed", merr);

    1389:   /* preserve the field association between the parent and sub-mesh objects */
    1390:   DMMoabSetFieldNames(*newdm, dmmoab->numFields, dmmoab->fieldNames);
    1391:   return(0);
    1392: }

    1395: PETSC_EXTERN PetscErrorCode DMMoabView_Ascii(DM dm, PetscViewer viewer)
    1396: {
    1397:   DM_Moab          *dmmoab = (DM_Moab*)(dm)->data;
    1398:   const char       *name;
    1399:   MPI_Comm          comm;
    1400:   PetscMPIInt       size;
    1401:   PetscErrorCode    ierr;

    1404:   PetscObjectGetComm((PetscObject)dm, &comm);
    1405:   MPI_Comm_size(comm, &size);
    1406:   PetscObjectGetName((PetscObject) dm, &name);
    1407:   PetscViewerASCIIPushTab(viewer);
    1408:   if (name) {PetscViewerASCIIPrintf(viewer, "%s in %D dimensions:\n", name, dmmoab->dim);}
    1409:   else      {PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dmmoab->dim);}
    1410:   /* print details about the global mesh */
    1411:   {
    1412:     PetscViewerASCIIPushTab(viewer);
    1413:     PetscViewerASCIIPrintf(viewer, "Sizes: cells=%D, vertices=%D, blocks=%D\n", dmmoab->nele, dmmoab->n, dmmoab->bs);
    1414:     /* print boundary data */
    1415:     PetscViewerASCIIPrintf(viewer, "Boundary trace:\n", dmmoab->bndyelems->size(), dmmoab->bndyfaces->size(), dmmoab->bndyvtx->size());
    1416:     {
    1417:       PetscViewerASCIIPushTab(viewer);
    1418:       PetscViewerASCIIPrintf(viewer, "cells=%D, faces=%D, vertices=%D\n", dmmoab->bndyelems->size(), dmmoab->bndyfaces->size(), dmmoab->bndyvtx->size());
    1419:       PetscViewerASCIIPopTab(viewer);
    1420:     }
    1421:     /* print field data */
    1422:     PetscViewerASCIIPrintf(viewer, "Fields: %D components\n", dmmoab->numFields);
    1423:     {
    1424:       PetscViewerASCIIPushTab(viewer);
    1425:       for (int i = 0; i < dmmoab->numFields; ++i) {
    1426:         PetscViewerASCIIPrintf(viewer, "[%D] - %s\n", i, dmmoab->fieldNames[i]);
    1427:       }
    1428:       PetscViewerASCIIPopTab(viewer);
    1429:     }
    1430:     PetscViewerASCIIPopTab(viewer);
    1431:   }
    1432:   PetscViewerASCIIPopTab(viewer);
    1433:   PetscViewerFlush(viewer);
    1434:   return(0);
    1435: }

    1437: PETSC_EXTERN PetscErrorCode DMMoabView_VTK(DM dm, PetscViewer v)
    1438: {
    1439:   return(0);
    1440: }

    1442: PETSC_EXTERN PetscErrorCode DMMoabView_HDF5(DM dm, PetscViewer v)
    1443: {
    1444:   return(0);
    1445: }

    1447: PETSC_EXTERN PetscErrorCode DMView_Moab(DM dm, PetscViewer viewer)
    1448: {
    1449:   PetscBool      iascii, ishdf5, isvtk;

    1455:   PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);
    1456:   PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);
    1457:   PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);
    1458:   if (iascii) {
    1459:     DMMoabView_Ascii(dm, viewer);
    1460:   } else if (ishdf5) {
    1461: #if defined(PETSC_HAVE_HDF5) && defined(MOAB_HAVE_HDF5)
    1462:     PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);
    1463:     DMMoabView_HDF5(dm, viewer);
    1464:     PetscViewerPopFormat(viewer);
    1465: #else
    1466:     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
    1467: #endif
    1468:   }
    1469:   else if (isvtk) {
    1470:     DMMoabView_VTK(dm, viewer);
    1471:   }
    1472:   return(0);
    1473: }

    1476: PETSC_EXTERN PetscErrorCode DMInitialize_Moab(DM dm)
    1477: {
    1479:   dm->ops->view                            = DMView_Moab;
    1480:   dm->ops->load                            = NULL /* DMLoad_Moab */;
    1481:   dm->ops->setfromoptions                  = DMSetFromOptions_Moab;
    1482:   dm->ops->clone                           = DMClone_Moab;
    1483:   dm->ops->setup                           = DMSetUp_Moab;
    1484:   dm->ops->createlocalsection            = NULL;
    1485:   dm->ops->createdefaultconstraints        = NULL;
    1486:   dm->ops->createglobalvector              = DMCreateGlobalVector_Moab;
    1487:   dm->ops->createlocalvector               = DMCreateLocalVector_Moab;
    1488:   dm->ops->getlocaltoglobalmapping         = NULL;
    1489:   dm->ops->createfieldis                   = NULL;
    1490:   dm->ops->createcoordinatedm              = NULL /* DMCreateCoordinateDM_Moab */;
    1491:   dm->ops->getcoloring                     = NULL;
    1492:   dm->ops->creatematrix                    = DMCreateMatrix_Moab;
    1493:   dm->ops->createinterpolation             = DMCreateInterpolation_Moab;
    1494:   dm->ops->createinjection                 = NULL /* DMCreateInjection_Moab */;
    1495:   dm->ops->refine                          = DMRefine_Moab;
    1496:   dm->ops->coarsen                         = DMCoarsen_Moab;
    1497:   dm->ops->refinehierarchy                 = DMRefineHierarchy_Moab;
    1498:   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Moab;
    1499:   dm->ops->globaltolocalbegin              = DMGlobalToLocalBegin_Moab;
    1500:   dm->ops->globaltolocalend                = DMGlobalToLocalEnd_Moab;
    1501:   dm->ops->localtoglobalbegin              = DMLocalToGlobalBegin_Moab;
    1502:   dm->ops->localtoglobalend                = DMLocalToGlobalEnd_Moab;
    1503:   dm->ops->destroy                         = DMDestroy_Moab;
    1504:   dm->ops->createsubdm                     = NULL /* DMCreateSubDM_Moab */;
    1505:   dm->ops->getdimpoints                    = NULL /* DMGetDimPoints_Moab */;
    1506:   dm->ops->locatepoints                    = NULL /* DMLocatePoints_Moab */;
    1507:   return(0);
    1508: }

    1511: PETSC_EXTERN PetscErrorCode DMClone_Moab(DM dm, DM *newdm)
    1512: {
    1513:   PetscErrorCode     ierr;

    1516:   /* get all the necessary handles from the private DM object */
    1517:   (*newdm)->data = (DM_Moab*) dm->data;
    1518:   ((DM_Moab*)dm->data)->refct++;

    1520:   PetscObjectChangeTypeName((PetscObject) *newdm, DMMOAB);
    1521:   DMInitialize_Moab(*newdm);
    1522:   return(0);
    1523: }

    1526: PETSC_EXTERN PetscErrorCode DMCreate_Moab(DM dm)
    1527: {

    1532:   PetscNewLog(dm, (DM_Moab**)&dm->data);

    1534:   ((DM_Moab*)dm->data)->bs = 1;
    1535:   ((DM_Moab*)dm->data)->numFields = 1;
    1536:   ((DM_Moab*)dm->data)->n = 0;
    1537:   ((DM_Moab*)dm->data)->nloc = 0;
    1538:   ((DM_Moab*)dm->data)->nghost = 0;
    1539:   ((DM_Moab*)dm->data)->nele = 0;
    1540:   ((DM_Moab*)dm->data)->neleloc = 0;
    1541:   ((DM_Moab*)dm->data)->neleghost = 0;
    1542:   ((DM_Moab*)dm->data)->ltog_map = NULL;
    1543:   ((DM_Moab*)dm->data)->ltog_sendrecv = NULL;

    1545:   ((DM_Moab*)dm->data)->refct = 1;
    1546:   ((DM_Moab*)dm->data)->parent = NULL;
    1547:   ((DM_Moab*)dm->data)->vlocal = new moab::Range();
    1548:   ((DM_Moab*)dm->data)->vowned = new moab::Range();
    1549:   ((DM_Moab*)dm->data)->vghost = new moab::Range();
    1550:   ((DM_Moab*)dm->data)->elocal = new moab::Range();
    1551:   ((DM_Moab*)dm->data)->eghost = new moab::Range();

    1553:   DMInitialize_Moab(dm);
    1554:   return(0);
    1555: }