Date: 2021-03-03T22:26:48+00:00

Actual source code: dadd.c
petsc-3.14.5 2021-03-03
Report Typos and Errors

      1: #include <petsc/private/dmdaimpl.h>

      3: /*@
      4:   DMDACreatePatchIS - Creates an index set corresponding to a patch of the DA.

      6:   Not Collective

      8:   Input Parameters:
      9: +  da - the DMDA
     10: .  lower - a matstencil with i, j and k corresponding to the lower corner of the patch
     11: -  upper - a matstencil with i, j and k corresponding to the upper corner of the patch

     13:   Output Parameters:
     14: .  is - the IS corresponding to the patch

     16:   Level: developer

     18: .seealso: DMDACreateDomainDecomposition(), DMDACreateDomainDecompositionScatters()
     19: @*/
     20: PetscErrorCode DMDACreatePatchIS(DM da,MatStencil *lower,MatStencil *upper,IS *is)
     21: {
     22:   PetscInt       ms=0,ns=0,ps=0;
     23:   PetscInt       me=1,ne=1,pe=1;
     24:   PetscInt       mr=0,nr=0,pr=0;
     25:   PetscInt       ii,jj,kk;
     26:   PetscInt       si,sj,sk;
     27:   PetscInt       i,j,k,l,idx;
     28:   PetscInt       base;
     29:   PetscInt       xm=1,ym=1,zm=1;
     30:   const PetscInt *lx,*ly,*lz;
     31:   PetscInt       ox,oy,oz;
     32:   PetscInt       m,n,p,M,N,P,dof;
     33:   PetscInt       nindices;
     34:   PetscInt       *indices;
     35:   DM_DA          *dd = (DM_DA*)da->data;

     39:   /* need to get the sizes of the actual DM rather than the "global" space of a subdomain DM */
     40:   M = dd->M;N = dd->N;P=dd->P;
     41:   m = dd->m;n = dd->n;p=dd->p;
     42:   dof = dd->w;
     43:   DMDAGetOffset(da,&ox,&oy,&oz,NULL,NULL,NULL);
     44:   DMDAGetOwnershipRanges(da,&lx,&ly,&lz);
     45:   nindices = (upper->i - lower->i)*(upper->j - lower->j)*(upper->k - lower->k)*dof;
     46:   PetscMalloc1(nindices,&indices);
     47:   /* start at index 0 on processor 0 */
     48:   mr = 0;
     49:   nr = 0;
     50:   pr = 0;
     51:   ms = 0;
     52:   ns = 0;
     53:   ps = 0;
     54:   if (lx) me = lx[0];
     55:   if (ly) ne = ly[0];
     56:   if (lz) pe = lz[0];
     57:   idx = 0;
     58:   for (k=lower->k-oz;k<upper->k-oz;k++) {
     59:     for (j=lower->j-oy;j < upper->j-oy;j++) {
     60:       for (i=lower->i-ox;i < upper->i-ox;i++) {
     61:         /* "actual" indices rather than ones outside of the domain */
     62:         ii = i;
     63:         jj = j;
     64:         kk = k;
     65:         if (ii < 0) ii = M + ii;
     66:         if (jj < 0) jj = N + jj;
     67:         if (kk < 0) kk = P + kk;
     68:         if (ii > M-1) ii = ii - M;
     69:         if (jj > N-1) jj = jj - N;
     70:         if (kk > P-1) kk = kk - P;
     71:         /* gone out of processor range on x axis */
     72:         while (ii > me-1 || ii < ms) {
     73:           if (mr == m-1) {
     74:             ms = 0;
     75:             me = lx[0];
     76:             mr = 0;
     77:           } else {
     78:             mr++;
     79:             ms = me;
     80:             me += lx[mr];
     81:           }
     82:         }
     83:         /* gone out of processor range on y axis */
     84:         while (jj > ne-1 || jj < ns) {
     85:           if (nr == n-1) {
     86:             ns = 0;
     87:             ne = ly[0];
     88:             nr = 0;
     89:           } else {
     90:             nr++;
     91:             ns = ne;
     92:             ne += ly[nr];
     93:           }
     94:         }
     95:         /* gone out of processor range on z axis */
     96:         while (kk > pe-1 || kk < ps) {
     97:           if (pr == p-1) {
     98:             ps = 0;
     99:             pe = lz[0];
    100:             pr = 0;
    101:           } else {
    102:             pr++;
    103:             ps = pe;
    104:             pe += lz[pr];
    105:           }
    106:         }
    107:         /* compute the vector base on owning processor */
    108:         xm = me - ms;
    109:         ym = ne - ns;
    110:         zm = pe - ps;
    111:         base = ms*ym*zm + ns*M*zm + ps*M*N;
    112:         /* compute the local coordinates on owning processor */
    113:         si = ii - ms;
    114:         sj = jj - ns;
    115:         sk = kk - ps;
    116:         for (l=0;l<dof;l++) {
    117:           indices[idx] = l + dof*(base + si + xm*sj + xm*ym*sk);
    118:           idx++;
    119:         }
    120:       }
    121:     }
    122:   }
    123:   ISCreateGeneral(PETSC_COMM_SELF,idx,indices,PETSC_OWN_POINTER,is);
    124:   return(0);
    125: }

    127: PetscErrorCode DMDASubDomainDA_Private(DM dm, PetscInt *nlocal, DM **sdm)
    128: {
    129:   DM             *da;
    130:   PetscInt       dim,size,i,j,k,idx;
    132:   DMDALocalInfo  info;
    133:   PetscInt       xsize,ysize,zsize;
    134:   PetscInt       xo,yo,zo;
    135:   PetscInt       xs,ys,zs;
    136:   PetscInt       xm=1,ym=1,zm=1;
    137:   PetscInt       xol,yol,zol;
    138:   PetscInt       m=1,n=1,p=1;
    139:   PetscInt       M,N,P;
    140:   PetscInt       pm,mtmp;

    143:   DMDAGetLocalInfo(dm,&info);
    144:   DMDAGetOverlap(dm,&xol,&yol,&zol);
    145:   DMDAGetNumLocalSubDomains(dm,&size);
    146:   PetscMalloc1(size,&da);

    148:   if (nlocal) *nlocal = size;

    150:   dim = info.dim;

    152:   M = info.xm;
    153:   N = info.ym;
    154:   P = info.zm;

    156:   if (dim == 1) {
    157:     m = size;
    158:   } else if (dim == 2) {
    159:     m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)N)));
    160:     while (m > 0) {
    161:       n = size/m;
    162:       if (m*n*p == size) break;
    163:       m--;
    164:     }
    165:   } else if (dim == 3) {
    166:     n = (PetscInt)(0.5 + PetscPowReal(((PetscReal)N*N)*((PetscReal)size)/((PetscReal)P*M),(PetscReal)(1./3.)));    if (!n) n = 1;
    167:     while (n > 0) {
    168:       pm = size/n;
    169:       if (n*pm == size) break;
    170:       n--;
    171:     }
    172:     if (!n) n = 1;
    173:     m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n)));
    174:     if (!m) m = 1;
    175:     while (m > 0) {
    176:       p = size/(m*n);
    177:       if (m*n*p == size) break;
    178:       m--;
    179:     }
    180:     if (M > P && m < p) {mtmp = m; m = p; p = mtmp;}
    181:   }

    183:   zs = info.zs;
    184:   idx = 0;
    185:   for (k = 0; k < p; k++) {
    186:     ys = info.ys;
    187:     for (j = 0; j < n; j++) {
    188:       xs = info.xs;
    189:       for (i = 0; i < m; i++) {
    190:         if (dim == 1) {
    191:           xm = M/m + ((M % m) > i);
    192:         } else if (dim == 2) {
    193:           xm = M/m + ((M % m) > i);
    194:           ym = N/n + ((N % n) > j);
    195:         } else if (dim == 3) {
    196:           xm = M/m + ((M % m) > i);
    197:           ym = N/n + ((N % n) > j);
    198:           zm = P/p + ((P % p) > k);
    199:         }

    201:         xsize = xm;
    202:         ysize = ym;
    203:         zsize = zm;
    204:         xo = xs;
    205:         yo = ys;
    206:         zo = zs;

    208:         DMDACreate(PETSC_COMM_SELF,&(da[idx]));
    209:         DMSetOptionsPrefix(da[idx],"sub_");
    210:         DMSetDimension(da[idx], info.dim);
    211:         DMDASetDof(da[idx], info.dof);

    213:         DMDASetStencilType(da[idx],info.st);
    214:         DMDASetStencilWidth(da[idx],info.sw);

    216:         if (info.bx == DM_BOUNDARY_PERIODIC || (xs != 0)) {
    217:           xsize += xol;
    218:           xo    -= xol;
    219:         }
    220:         if (info.by == DM_BOUNDARY_PERIODIC || (ys != 0)) {
    221:           ysize += yol;
    222:           yo    -= yol;
    223:         }
    224:         if (info.bz == DM_BOUNDARY_PERIODIC || (zs != 0)) {
    225:           zsize += zol;
    226:           zo    -= zol;
    227:         }

    229:         if (info.bx == DM_BOUNDARY_PERIODIC || (xs+xm != info.mx)) xsize += xol;
    230:         if (info.by == DM_BOUNDARY_PERIODIC || (ys+ym != info.my)) ysize += yol;
    231:         if (info.bz == DM_BOUNDARY_PERIODIC || (zs+zm != info.mz)) zsize += zol;

    233:         if (info.bx != DM_BOUNDARY_PERIODIC) {
    234:           if (xo < 0) {
    235:             xsize += xo;
    236:             xo = 0;
    237:           }
    238:           if (xo+xsize > info.mx-1) {
    239:             xsize -= xo+xsize - info.mx;
    240:           }
    241:         }
    242:         if (info.by != DM_BOUNDARY_PERIODIC) {
    243:           if (yo < 0) {
    244:             ysize += yo;
    245:             yo = 0;
    246:           }
    247:           if (yo+ysize > info.my-1) {
    248:             ysize -= yo+ysize - info.my;
    249:           }
    250:         }
    251:         if (info.bz != DM_BOUNDARY_PERIODIC) {
    252:           if (zo < 0) {
    253:             zsize += zo;
    254:             zo = 0;
    255:           }
    256:           if (zo+zsize > info.mz-1) {
    257:             zsize -= zo+zsize - info.mz;
    258:           }
    259:         }

    261:         DMDASetSizes(da[idx], xsize, ysize, zsize);
    262:         DMDASetNumProcs(da[idx], 1, 1, 1);
    263:         DMDASetBoundaryType(da[idx], DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);

    265:         /* set up as a block instead */
    266:         DMSetUp(da[idx]);

    268:         /* nonoverlapping region */
    269:         DMDASetNonOverlappingRegion(da[idx],xs,ys,zs,xm,ym,zm);

    271:         /* this alters the behavior of DMDAGetInfo, DMDAGetLocalInfo, DMDAGetCorners, and DMDAGetGhostedCorners and should be used with care */
    272:         DMDASetOffset(da[idx],xo,yo,zo,info.mx,info.my,info.mz);
    273:         xs += xm;
    274:         idx++;
    275:       }
    276:       ys += ym;
    277:     }
    278:     zs += zm;
    279:   }
    280:   *sdm = da;
    281:   return(0);
    282: }

    284: /*
    285:  Fills the local vector problem on the subdomain from the global problem.

    287:  Right now this assumes one subdomain per processor.

    289:  */
    290: PetscErrorCode DMCreateDomainDecompositionScatters_DA(DM dm,PetscInt nsubdms,DM *subdms,VecScatter **iscat,VecScatter **oscat, VecScatter **lscat)
    291: {
    293:   DMDALocalInfo  info,subinfo;
    294:   DM             subdm;
    295:   MatStencil     upper,lower;
    296:   IS             idis,isis,odis,osis,gdis;
    297:   Vec            svec,dvec,slvec;
    298:   PetscInt       xm,ym,zm,xs,ys,zs;
    299:   PetscInt       i;

    303:   /* allocate the arrays of scatters */
    304:   if (iscat) {PetscMalloc1(nsubdms,iscat);}
    305:   if (oscat) {PetscMalloc1(nsubdms,oscat);}
    306:   if (lscat) {PetscMalloc1(nsubdms,lscat);}

    308:   DMDAGetLocalInfo(dm,&info);
    309:   for (i = 0; i < nsubdms; i++) {
    310:     subdm = subdms[i];
    311:     DMDAGetLocalInfo(subdm,&subinfo);
    312:     DMDAGetNonOverlappingRegion(subdm,&xs,&ys,&zs,&xm,&ym,&zm);

    314:     /* create the global and subdomain index sets for the inner domain */
    315:     lower.i = xs;
    316:     lower.j = ys;
    317:     lower.k = zs;
    318:     upper.i = xs+xm;
    319:     upper.j = ys+ym;
    320:     upper.k = zs+zm;
    321:     DMDACreatePatchIS(dm,&lower,&upper,&idis);
    322:     DMDACreatePatchIS(subdm,&lower,&upper,&isis);

    324:     /* create the global and subdomain index sets for the outer subdomain */
    325:     lower.i = subinfo.xs;
    326:     lower.j = subinfo.ys;
    327:     lower.k = subinfo.zs;
    328:     upper.i = subinfo.xs+subinfo.xm;
    329:     upper.j = subinfo.ys+subinfo.ym;
    330:     upper.k = subinfo.zs+subinfo.zm;
    331:     DMDACreatePatchIS(dm,&lower,&upper,&odis);
    332:     DMDACreatePatchIS(subdm,&lower,&upper,&osis);

    334:     /* global and subdomain ISes for the local indices of the subdomain */
    335:     /* todo - make this not loop over at nonperiodic boundaries, which will be more involved */
    336:     lower.i = subinfo.gxs;
    337:     lower.j = subinfo.gys;
    338:     lower.k = subinfo.gzs;
    339:     upper.i = subinfo.gxs+subinfo.gxm;
    340:     upper.j = subinfo.gys+subinfo.gym;
    341:     upper.k = subinfo.gzs+subinfo.gzm;

    343:     DMDACreatePatchIS(dm,&lower,&upper,&gdis);

    345:     /* form the scatter */
    346:     DMGetGlobalVector(dm,&dvec);
    347:     DMGetGlobalVector(subdm,&svec);
    348:     DMGetLocalVector(subdm,&slvec);

    350:     if (iscat) {VecScatterCreate(dvec,idis,svec,isis,&(*iscat)[i]);}
    351:     if (oscat) {VecScatterCreate(dvec,odis,svec,osis,&(*oscat)[i]);}
    352:     if (lscat) {VecScatterCreate(dvec,gdis,slvec,NULL,&(*lscat)[i]);}

    354:     DMRestoreGlobalVector(dm,&dvec);
    355:     DMRestoreGlobalVector(subdm,&svec);
    356:     DMRestoreLocalVector(subdm,&slvec);

    358:     ISDestroy(&idis);
    359:     ISDestroy(&isis);

    361:     ISDestroy(&odis);
    362:     ISDestroy(&osis);

    364:     ISDestroy(&gdis);
    365:   }
    366:   return(0);
    367: }

    369: PetscErrorCode DMDASubDomainIS_Private(DM dm,PetscInt n,DM *subdm,IS **iis,IS **ois)
    370: {
    372:   PetscInt       i;
    373:   DMDALocalInfo  info,subinfo;
    374:   MatStencil     lower,upper;

    377:   DMDAGetLocalInfo(dm,&info);
    378:   if (iis) {PetscMalloc1(n,iis);}
    379:   if (ois) {PetscMalloc1(n,ois);}

    381:   for (i = 0;i < n; i++) {
    382:     DMDAGetLocalInfo(subdm[i],&subinfo);
    383:     if (iis) {
    384:       /* create the inner IS */
    385:       lower.i = info.xs;
    386:       lower.j = info.ys;
    387:       lower.k = info.zs;
    388:       upper.i = info.xs+info.xm;
    389:       upper.j = info.ys+info.ym;
    390:       upper.k = info.zs+info.zm;
    391:       DMDACreatePatchIS(dm,&lower,&upper,&(*iis)[i]);
    392:     }

    394:     if (ois) {
    395:       /* create the outer IS */
    396:       lower.i = subinfo.xs;
    397:       lower.j = subinfo.ys;
    398:       lower.k = subinfo.zs;
    399:       upper.i = subinfo.xs+subinfo.xm;
    400:       upper.j = subinfo.ys+subinfo.ym;
    401:       upper.k = subinfo.zs+subinfo.zm;
    402:       DMDACreatePatchIS(dm,&lower,&upper,&(*ois)[i]);
    403:     }
    404:   }
    405:   return(0);
    406: }

    408: PetscErrorCode DMCreateDomainDecomposition_DA(DM dm,PetscInt *len,char ***names,IS **iis,IS **ois,DM **subdm)
    409: {
    411:   DM             *sdm;
    412:   PetscInt       n,i;

    415:   DMDASubDomainDA_Private(dm,&n,&sdm);
    416:   if (names) {
    417:     PetscMalloc1(n,names);
    418:     for (i=0;i<n;i++) (*names)[i] = NULL;
    419:   }
    420:   DMDASubDomainIS_Private(dm,n,sdm,iis,ois);
    421:   if (subdm) *subdm = sdm;
    422:   else {
    423:     for (i=0;i<n;i++) {
    424:       DMDestroy(&sdm[i]);
    425:     }
    426:   }
    427:   if (len) *len = n;
    428:   return(0);
    429: }