Date: 2026-07-30T21:14:32+00:00

Actual source code: iscoloring.c

      1: #include <petsc/private/isimpl.h>
      2: #include <petscviewer.h>
      3: #include <petscsf.h>

      5: const char *const ISColoringTypes[] = {"global", "ghosted", "ISColoringType", "IS_COLORING_", NULL};

      7: /*@
      8:   ISColoringReference - Increases the reference count of an `ISColoring` object by one

     10:   Logically collective

     12:   Input Parameter:
     13: . coloring - the `ISColoring` object

     15:   Level: developer

     17:   Note:
     18:   The reference count is decreased by a matching call to `ISColoringDestroy()`.

     20: .seealso: `ISColoring`, `ISColoringCreate()`, `ISColoringDestroy()`
     21: @*/
     22: PetscErrorCode ISColoringReference(ISColoring coloring)
     23: {
     24:   PetscFunctionBegin;
     25:   coloring->refct++;
     26:   PetscFunctionReturn(PETSC_SUCCESS);
     27: }

     29: /*@
     30:   ISColoringSetType - indicates if the coloring is for the local representation (including ghost points) or the global representation of a `Mat`

     32:   Collective

     34:   Input Parameters:
     35: + coloring - the coloring object
     36: - type     - either `IS_COLORING_LOCAL` or `IS_COLORING_GLOBAL`

     38:   Level: intermediate

     40:   Notes:
     41:   `IS_COLORING_LOCAL` can lead to faster computations since parallel ghost point updates are not needed for each color

     43:   With `IS_COLORING_LOCAL` the coloring is in the numbering of the local vector, for `IS_COLORING_GLOBAL` it is in the numbering of the global vector

     45: .seealso: `MatFDColoringCreate()`, `ISColoring`, `ISColoringType`, `ISColoringCreate()`, `IS_COLORING_LOCAL`, `IS_COLORING_GLOBAL`, `ISColoringGetType()`
     46: @*/
     47: PetscErrorCode ISColoringSetType(ISColoring coloring, ISColoringType type)
     48: {
     49:   PetscFunctionBegin;
     50:   coloring->ctype = type;
     51:   PetscFunctionReturn(PETSC_SUCCESS);
     52: }

     54: /*@
     55:   ISColoringGetType - gets if the coloring is for the local representation (including ghost points) or the global representation

     57:   Collective

     59:   Input Parameter:
     60: . coloring - the coloring object

     62:   Output Parameter:
     63: . type - either `IS_COLORING_LOCAL` or `IS_COLORING_GLOBAL`

     65:   Level: intermediate

     67: .seealso: `MatFDColoringCreate()`, `ISColoring`, `ISColoringType`, `ISColoringCreate()`, `IS_COLORING_LOCAL`, `IS_COLORING_GLOBAL`, `ISColoringSetType()`
     68: @*/
     69: PetscErrorCode ISColoringGetType(ISColoring coloring, ISColoringType *type)
     70: {
     71:   PetscFunctionBegin;
     72:   *type = coloring->ctype;
     73:   PetscFunctionReturn(PETSC_SUCCESS);
     74: }

     76: /*@
     77:   ISColoringDestroy - Destroys an `ISColoring` coloring context.

     79:   Collective

     81:   Input Parameter:
     82: . iscoloring - the coloring context

     84:   Level: advanced

     86: .seealso: `ISColoring`, `ISColoringView()`, `MatColoring`
     87: @*/
     88: PetscErrorCode ISColoringDestroy(ISColoring *iscoloring)
     89: {
     90:   PetscInt i;

     92:   PetscFunctionBegin;
     93:   if (!*iscoloring) PetscFunctionReturn(PETSC_SUCCESS);
     94:   PetscAssertPointer(*iscoloring, 1);
     95:   if (--(*iscoloring)->refct > 0) {
     96:     *iscoloring = NULL;
     97:     PetscFunctionReturn(PETSC_SUCCESS);
     98:   }

    100:   if ((*iscoloring)->is) {
    101:     for (i = 0; i < (*iscoloring)->n; i++) PetscCall(ISDestroy(&(*iscoloring)->is[i]));
    102:     PetscCall(PetscFree((*iscoloring)->is));
    103:   }
    104:   if ((*iscoloring)->allocated) PetscCall(PetscFree((*iscoloring)->colors));
    105:   PetscCall(PetscCommDestroy(&(*iscoloring)->comm));
    106:   PetscCall(PetscFree(*iscoloring));
    107:   PetscFunctionReturn(PETSC_SUCCESS);
    108: }

    110: /*@
    111:   ISColoringViewFromOptions - Processes command line options to determine if/how an `ISColoring` object is to be viewed.

    113:   Collective

    115:   Input Parameters:
    116: + obj  - the `ISColoring` object
    117: . bobj - prefix to use for viewing, or `NULL` to use prefix of `mat`
    118: - name - option to activate viewing

    120:   Options Database Key:
    121: . -name [viewertype][:...] - option name and values. See `PetscObjectViewFromOptions()` for the possible arguments

    123:   Level: intermediate

    125:   Developer Note:
    126:   This cannot use `PetscObjectViewFromOptions()` because `ISColoring` is not a `PetscObject`

    128: .seealso: `ISColoring`, `ISColoringView()`, `PetscObjectViewFromOptions()`
    129: @*/
    130: PetscErrorCode ISColoringViewFromOptions(ISColoring obj, PetscObject bobj, const char name[])
    131: {
    132:   PetscViewer       viewer;
    133:   PetscBool         flg;
    134:   PetscViewerFormat format;
    135:   char             *prefix;

    137:   PetscFunctionBegin;
    138:   prefix = bobj ? bobj->prefix : NULL;
    139:   PetscCall(PetscOptionsCreateViewer(obj->comm, NULL, prefix, name, &viewer, &format, &flg));
    140:   if (flg) {
    141:     PetscCall(PetscViewerPushFormat(viewer, format));
    142:     PetscCall(ISColoringView(obj, viewer));
    143:     PetscCall(PetscViewerPopFormat(viewer));
    144:     PetscCall(PetscViewerDestroy(&viewer));
    145:   }
    146:   PetscFunctionReturn(PETSC_SUCCESS);
    147: }

    149: /*@
    150:   ISColoringView - Views an `ISColoring` coloring context.

    152:   Collective

    154:   Input Parameters:
    155: + iscoloring - the coloring context
    156: - viewer     - the viewer

    158:   Level: advanced

    160: .seealso: `ISColoring()`, `ISColoringViewFromOptions()`, `ISColoringDestroy()`, `ISColoringGetIS()`, `MatColoring`
    161: @*/
    162: PetscErrorCode ISColoringView(ISColoring iscoloring, PetscViewer viewer)
    163: {
    164:   PetscInt  i;
    165:   PetscBool isascii;
    166:   IS       *is;

    168:   PetscFunctionBegin;
    169:   PetscAssertPointer(iscoloring, 1);
    170:   if (!viewer) PetscCall(PetscViewerASCIIGetStdout(iscoloring->comm, &viewer));

    173:   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERASCII, &isascii));
    174:   if (isascii) {
    175:     MPI_Comm    comm;
    176:     PetscMPIInt size, rank;

    178:     PetscCall(PetscObjectGetComm((PetscObject)viewer, &comm));
    179:     PetscCallMPI(MPI_Comm_size(comm, &size));
    180:     PetscCallMPI(MPI_Comm_rank(comm, &rank));
    181:     PetscCall(PetscViewerASCIIPrintf(viewer, "ISColoring Object: %d MPI processes\n", size));
    182:     PetscCall(PetscViewerASCIIPrintf(viewer, "ISColoringType: %s\n", ISColoringTypes[iscoloring->ctype]));
    183:     PetscCall(PetscViewerASCIIPushSynchronized(viewer));
    184:     PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Number of colors %" PetscInt_FMT "\n", rank, iscoloring->n));
    185:     PetscCall(PetscViewerFlush(viewer));
    186:     PetscCall(PetscViewerASCIIPopSynchronized(viewer));
    187:   }

    189:   PetscCall(ISColoringGetIS(iscoloring, PETSC_USE_POINTER, PETSC_IGNORE, &is));
    190:   for (i = 0; i < iscoloring->n; i++) PetscCall(ISView(iscoloring->is[i], viewer));
    191:   PetscCall(ISColoringRestoreIS(iscoloring, PETSC_USE_POINTER, &is));
    192:   PetscFunctionReturn(PETSC_SUCCESS);
    193: }

    195: /*@C
    196:   ISColoringGetColors - Returns an array with the color for each local node

    198:   Not Collective

    200:   Input Parameter:
    201: . iscoloring - the coloring context

    203:   Output Parameters:
    204: + n      - number of nodes
    205: . nc     - number of colors
    206: - colors - color for each node

    208:   Level: advanced

    210:   Notes:
    211:   Do not free the `colors` array.

    213:   The `colors` array will only be valid for the lifetime of the `ISColoring`

    215: .seealso: `ISColoring`, `ISColoringValue`, `ISColoringRestoreIS()`, `ISColoringView()`, `ISColoringGetIS()`
    216: @*/
    217: PetscErrorCode ISColoringGetColors(ISColoring iscoloring, PetscInt *n, PetscInt *nc, const ISColoringValue **colors)
    218: {
    219:   PetscFunctionBegin;
    220:   PetscAssertPointer(iscoloring, 1);

    222:   if (n) *n = iscoloring->N;
    223:   if (nc) *nc = iscoloring->n;
    224:   if (colors) *colors = iscoloring->colors;
    225:   PetscFunctionReturn(PETSC_SUCCESS);
    226: }

    228: /*@C
    229:   ISColoringGetIS - Extracts index sets from the coloring context. Each is contains the nodes of one color

    231:   Collective

    233:   Input Parameters:
    234: + iscoloring - the coloring context
    235: - mode       - if this value is `PETSC_OWN_POINTER` then the caller owns the pointer and must free the array of `IS` and each `IS` in the array

    237:   Output Parameters:
    238: + nn   - number of index sets in the coloring context
    239: - isis - array of index sets

    241:   Level: advanced

    243:   Note:
    244:   If mode is `PETSC_USE_POINTER` then `ISColoringRestoreIS()` must be called when the `IS` are no longer needed

    246: .seealso: `ISColoring`, `IS`, `ISColoringRestoreIS()`, `ISColoringView()`, `ISColoringGetColoring()`, `ISColoringGetColors()`
    247: @*/
    248: PetscErrorCode ISColoringGetIS(ISColoring iscoloring, PetscCopyMode mode, PetscInt *nn, IS *isis[])
    249: {
    250:   PetscFunctionBegin;
    251:   PetscAssertPointer(iscoloring, 1);

    253:   if (nn) *nn = iscoloring->n;
    254:   if (isis) {
    255:     if (!iscoloring->is) {
    256:       PetscInt        *mcolors, **ii, nc = iscoloring->n, i, base, n = iscoloring->N;
    257:       ISColoringValue *colors = iscoloring->colors;
    258:       IS              *is;

    260:       if (PetscDefined(USE_DEBUG)) {
    261:         for (i = 0; i < n; i++) PetscCheck(((PetscInt)colors[i]) < nc, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Coloring is our of range index %" PetscInt_FMT "value %d number colors %" PetscInt_FMT, i, (int)colors[i], nc);
    262:       }

    264:       /* generate the lists of nodes for each color */
    265:       PetscCall(PetscCalloc1(nc, &mcolors));
    266:       for (i = 0; i < n; i++) mcolors[colors[i]]++;

    268:       PetscCall(PetscMalloc1(nc, &ii));
    269:       PetscCall(PetscMalloc1(n, &ii[0]));
    270:       for (i = 1; i < nc; i++) ii[i] = ii[i - 1] + mcolors[i - 1];
    271:       PetscCall(PetscArrayzero(mcolors, nc));

    273:       if (iscoloring->ctype == IS_COLORING_GLOBAL) {
    274:         PetscCallMPI(MPI_Scan(&iscoloring->N, &base, 1, MPIU_INT, MPI_SUM, iscoloring->comm));
    275:         base -= iscoloring->N;
    276:         for (i = 0; i < n; i++) ii[colors[i]][mcolors[colors[i]]++] = i + base; /* global idx */
    277:       } else if (iscoloring->ctype == IS_COLORING_LOCAL) {
    278:         for (i = 0; i < n; i++) ii[colors[i]][mcolors[colors[i]]++] = i; /* local idx */
    279:       } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Not provided for this ISColoringType type");

    281:       PetscCall(PetscMalloc1(nc, &is));
    282:       for (i = 0; i < nc; i++) PetscCall(ISCreateGeneral(iscoloring->comm, mcolors[i], ii[i], PETSC_COPY_VALUES, is + i));

    284:       if (mode != PETSC_OWN_POINTER) iscoloring->is = is;
    285:       *isis = is;
    286:       PetscCall(PetscFree(ii[0]));
    287:       PetscCall(PetscFree(ii));
    288:       PetscCall(PetscFree(mcolors));
    289:     } else {
    290:       *isis = iscoloring->is;
    291:       if (mode == PETSC_OWN_POINTER) iscoloring->is = NULL;
    292:     }
    293:   }
    294:   PetscFunctionReturn(PETSC_SUCCESS);
    295: }

    297: /*@C
    298:   ISColoringRestoreIS - Restores the index sets extracted from the coloring context with `ISColoringGetIS()` using `PETSC_USE_POINTER`

    300:   Collective

    302:   Input Parameters:
    303: + iscoloring - the coloring context
    304: . mode       - who retains ownership of the is
    305: - is         - array of index sets

    307:   Level: advanced

    309: .seealso: `ISColoring()`, `IS`, `ISColoringGetIS()`, `ISColoringView()`, `PetscCopyMode`
    310: @*/
    311: PetscErrorCode ISColoringRestoreIS(ISColoring iscoloring, PetscCopyMode mode, IS *is[])
    312: {
    313:   PetscFunctionBegin;
    314:   PetscAssertPointer(iscoloring, 1);

    316:   /* currently nothing is done here */
    317:   PetscFunctionReturn(PETSC_SUCCESS);
    318: }

    320: /*@
    321:   ISColoringCreate - Generates an `ISColoring` context from lists (provided by each MPI process) of colors for each node.

    323:   Collective

    325:   Input Parameters:
    326: + comm    - communicator for the processors creating the coloring
    327: . ncolors - max color value
    328: . n       - number of nodes on this processor
    329: . colors  - array containing the colors for this MPI rank, color numbers begin at 0, for each local node
    330: - mode    - see `PetscCopyMode` for meaning of this flag.

    332:   Output Parameter:
    333: . iscoloring - the resulting coloring data structure

    335:   Options Database Key:
    336: . -is_coloring_view - Activates `ISColoringView()`

    338:   Level: advanced

    340:   Notes:
    341:   By default sets coloring type to  `IS_COLORING_GLOBAL`

    343: .seealso: `ISColoring`, `ISColoringValue`, `MatColoringCreate()`, `ISColoringView()`, `ISColoringDestroy()`, `ISColoringSetType()`
    344: @*/
    345: PetscErrorCode ISColoringCreate(MPI_Comm comm, PetscInt ncolors, PetscInt n, const ISColoringValue colors[], PetscCopyMode mode, ISColoring *iscoloring)
    346: {
    347:   PetscMPIInt size, rank, tag;
    348:   PetscInt    base, top, i;
    349:   PetscInt    nc, ncwork;
    350:   MPI_Status  status;

    352:   PetscFunctionBegin;
    353:   if (ncolors != PETSC_DECIDE && ncolors > IS_COLORING_MAX) {
    354:     PetscCheck(ncolors <= PETSC_UINT16_MAX, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Max color value exceeds %d limit. This number is unrealistic. Perhaps a bug in code? Current max: %d user requested: %" PetscInt_FMT, PETSC_UINT16_MAX, PETSC_IS_COLORING_MAX, ncolors);
    355:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Max color value exceeds limit. Perhaps reconfigure PETSc with --with-is-color-value-type=short? Current max: %d user requested: %" PetscInt_FMT, PETSC_IS_COLORING_MAX, ncolors);
    356:   }
    357:   PetscCall(PetscNew(iscoloring));
    358:   PetscCall(PetscCommDuplicate(comm, &(*iscoloring)->comm, &tag));
    359:   comm = (*iscoloring)->comm;

    361:   /* compute the number of the first node on my processor */
    362:   PetscCallMPI(MPI_Comm_size(comm, &size));

    364:   /* should use MPI_Scan() */
    365:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
    366:   if (rank == 0) {
    367:     base = 0;
    368:     top  = n;
    369:   } else {
    370:     PetscCallMPI(MPI_Recv(&base, 1, MPIU_INT, rank - 1, tag, comm, &status));
    371:     top = base + n;
    372:   }
    373:   if (rank < size - 1) PetscCallMPI(MPI_Send(&top, 1, MPIU_INT, rank + 1, tag, comm));

    375:   /* compute the total number of colors */
    376:   ncwork = 0;
    377:   for (i = 0; i < n; i++) {
    378:     if (ncwork < colors[i]) ncwork = colors[i];
    379:   }
    380:   ncwork++;
    381:   PetscCallMPI(MPIU_Allreduce(&ncwork, &nc, 1, MPIU_INT, MPI_MAX, comm));
    382:   PetscCheck(nc <= ncolors, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Number of colors passed in %" PetscInt_FMT " is less than the actual number of colors in array %" PetscInt_FMT, ncolors, nc);
    383:   (*iscoloring)->n     = nc;
    384:   (*iscoloring)->is    = NULL;
    385:   (*iscoloring)->N     = n;
    386:   (*iscoloring)->refct = 1;
    387:   (*iscoloring)->ctype = IS_COLORING_GLOBAL;
    388:   if (mode == PETSC_COPY_VALUES) {
    389:     PetscCall(PetscMalloc1(n, &(*iscoloring)->colors));
    390:     PetscCall(PetscArraycpy((*iscoloring)->colors, colors, n));
    391:     (*iscoloring)->allocated = PETSC_TRUE;
    392:   } else if (mode == PETSC_OWN_POINTER) {
    393:     (*iscoloring)->colors    = (ISColoringValue *)colors;
    394:     (*iscoloring)->allocated = PETSC_TRUE;
    395:   } else {
    396:     (*iscoloring)->colors    = (ISColoringValue *)colors;
    397:     (*iscoloring)->allocated = PETSC_FALSE;
    398:   }
    399:   PetscCall(ISColoringViewFromOptions(*iscoloring, NULL, "-is_coloring_view"));
    400:   PetscCall(PetscInfo(0, "Number of colors %" PetscInt_FMT "\n", nc));
    401:   PetscFunctionReturn(PETSC_SUCCESS);
    402: }

    404: /*@
    405:   ISBuildTwoSided - Takes an `IS` that describes where each element will be mapped globally over all ranks.
    406:   Generates an `IS` that contains new numbers from remote or local on the `IS`.

    408:   Collective

    410:   Input Parameters:
    411: + ito    - an `IS` describes to which rank each entry will be mapped. Negative target rank will be ignored
    412: - toindx - an `IS` describes what indices should send. `NULL` means sending natural numbering

    414:   Output Parameter:
    415: . rows - contains new numbers from remote or local

    417:   Level: advanced

    419:   Developer Note:
    420:   This manual page is incomprehensible and still needs to be fixed

    422: .seealso: [](sec_scatter), `IS`, `MatPartitioningCreate()`, `ISPartitioningToNumbering()`, `ISPartitioningCount()`
    423: @*/
    424: PetscErrorCode ISBuildTwoSided(IS ito, IS toindx, IS *rows)
    425: {
    426:   const PetscInt *ito_indices, *toindx_indices;
    427:   PetscInt       *send_indices, rstart, *recv_indices, nrecvs, nsends;
    428:   PetscInt       *tosizes, *fromsizes, j, *tosizes_tmp, *tooffsets_tmp, ito_ln;
    429:   PetscMPIInt    *toranks, *fromranks, size, target_rank, *fromperm_newtoold, nto, nfrom;
    430:   PetscLayout     isrmap;
    431:   MPI_Comm        comm;
    432:   PetscSF         sf;
    433:   PetscSFNode    *iremote;

    435:   PetscFunctionBegin;
    436:   PetscCall(PetscObjectGetComm((PetscObject)ito, &comm));
    437:   PetscCallMPI(MPI_Comm_size(comm, &size));
    438:   PetscCall(ISGetLocalSize(ito, &ito_ln));
    439:   PetscCall(ISGetLayout(ito, &isrmap));
    440:   PetscCall(PetscLayoutGetRange(isrmap, &rstart, NULL));
    441:   PetscCall(ISGetIndices(ito, &ito_indices));
    442:   PetscCall(PetscCalloc2(size, &tosizes_tmp, size + 1, &tooffsets_tmp));
    443:   for (PetscInt i = 0; i < ito_ln; i++) {
    444:     if (ito_indices[i] < 0) continue;
    445:     else PetscCheck(ito_indices[i] < size, comm, PETSC_ERR_ARG_OUTOFRANGE, "target rank %" PetscInt_FMT " is larger than communicator size %d ", ito_indices[i], size);
    446:     tosizes_tmp[ito_indices[i]]++;
    447:   }
    448:   nto = 0;
    449:   for (PetscMPIInt i = 0; i < size; i++) {
    450:     tooffsets_tmp[i + 1] = tooffsets_tmp[i] + tosizes_tmp[i];
    451:     if (tosizes_tmp[i] > 0) nto++;
    452:   }
    453:   PetscCall(PetscCalloc2(nto, &toranks, 2 * nto, &tosizes));
    454:   nto = 0;
    455:   for (PetscMPIInt i = 0; i < size; i++) {
    456:     if (tosizes_tmp[i] > 0) {
    457:       toranks[nto]         = i;
    458:       tosizes[2 * nto]     = tosizes_tmp[i];   /* size */
    459:       tosizes[2 * nto + 1] = tooffsets_tmp[i]; /* offset */
    460:       nto++;
    461:     }
    462:   }
    463:   nsends = tooffsets_tmp[size];
    464:   PetscCall(PetscCalloc1(nsends, &send_indices));
    465:   if (toindx) PetscCall(ISGetIndices(toindx, &toindx_indices));
    466:   for (PetscInt i = 0; i < ito_ln; i++) {
    467:     if (ito_indices[i] < 0) continue;
    468:     PetscCall(PetscMPIIntCast(ito_indices[i], &target_rank));
    469:     send_indices[tooffsets_tmp[target_rank]] = toindx ? toindx_indices[i] : (i + rstart);
    470:     tooffsets_tmp[target_rank]++;
    471:   }
    472:   if (toindx) PetscCall(ISRestoreIndices(toindx, &toindx_indices));
    473:   PetscCall(ISRestoreIndices(ito, &ito_indices));
    474:   PetscCall(PetscFree2(tosizes_tmp, tooffsets_tmp));
    475:   PetscCall(PetscCommBuildTwoSided(comm, 2, MPIU_INT, nto, toranks, tosizes, &nfrom, &fromranks, &fromsizes));
    476:   PetscCall(PetscFree2(toranks, tosizes));
    477:   PetscCall(PetscMalloc1(nfrom, &fromperm_newtoold));
    478:   for (PetscMPIInt i = 0; i < nfrom; i++) fromperm_newtoold[i] = i;
    479:   PetscCall(PetscSortMPIIntWithArray(nfrom, fromranks, fromperm_newtoold));
    480:   nrecvs = 0;
    481:   for (PetscMPIInt i = 0; i < nfrom; i++) nrecvs += fromsizes[i * 2];
    482:   PetscCall(PetscCalloc1(nrecvs, &recv_indices));
    483:   PetscCall(PetscMalloc1(nrecvs, &iremote));
    484:   nrecvs = 0;
    485:   for (PetscMPIInt i = 0; i < nfrom; i++) {
    486:     for (j = 0; j < fromsizes[2 * fromperm_newtoold[i]]; j++) {
    487:       iremote[nrecvs].rank    = fromranks[i];
    488:       iremote[nrecvs++].index = fromsizes[2 * fromperm_newtoold[i] + 1] + j;
    489:     }
    490:   }
    491:   PetscCall(PetscSFCreate(comm, &sf));
    492:   PetscCall(PetscSFSetGraph(sf, nsends, nrecvs, NULL, PETSC_OWN_POINTER, iremote, PETSC_OWN_POINTER));
    493:   PetscCall(PetscSFSetType(sf, PETSCSFBASIC));
    494:   /* how to put a prefix ? */
    495:   PetscCall(PetscSFSetFromOptions(sf));
    496:   PetscCall(PetscSFBcastBegin(sf, MPIU_INT, send_indices, recv_indices, MPI_REPLACE));
    497:   PetscCall(PetscSFBcastEnd(sf, MPIU_INT, send_indices, recv_indices, MPI_REPLACE));
    498:   PetscCall(PetscSFDestroy(&sf));
    499:   PetscCall(PetscFree(fromranks));
    500:   PetscCall(PetscFree(fromsizes));
    501:   PetscCall(PetscFree(fromperm_newtoold));
    502:   PetscCall(PetscFree(send_indices));
    503:   if (rows) {
    504:     PetscCall(PetscSortInt(nrecvs, recv_indices));
    505:     PetscCall(ISCreateGeneral(comm, nrecvs, recv_indices, PETSC_OWN_POINTER, rows));
    506:   } else {
    507:     PetscCall(PetscFree(recv_indices));
    508:   }
    509:   PetscFunctionReturn(PETSC_SUCCESS);
    510: }

    512: /*@
    513:   ISPartitioningToNumbering - Takes an `IS' that represents a partitioning (the MPI rank that each local entry belongs to) and on each MPI process
    514:   generates an `IS` that contains a new global node number in the new ordering for each entry

    516:   Collective

    518:   Input Parameter:
    519: . part - a partitioning as generated by `MatPartitioningApply()` or `MatPartitioningApplyND()`

    521:   Output Parameter:
    522: . is - on each processor the index set that defines the global numbers
    523:          (in the new numbering) for all the nodes currently (before the partitioning)
    524:          on that processor

    526:   Level: advanced

    528:   Note:
    529:   The resulting `IS` tells where each local entry is mapped to in a new global ordering

    531: .seealso: [](sec_scatter), `IS`, `MatPartitioningCreate()`, `AOCreateBasic()`, `ISPartitioningCount()`
    532: @*/
    533: PetscErrorCode ISPartitioningToNumbering(IS part, IS *is)
    534: {
    535:   MPI_Comm        comm;
    536:   IS              ndorder;
    537:   PetscInt        n, *starts = NULL, *sums = NULL, *lsizes = NULL, *newi = NULL;
    538:   const PetscInt *indices = NULL;
    539:   PetscMPIInt     np, npt;

    541:   PetscFunctionBegin;
    543:   PetscAssertPointer(is, 2);
    544:   /* see if the partitioning comes from nested dissection */
    545:   PetscCall(PetscObjectQuery((PetscObject)part, "_petsc_matpartitioning_ndorder", (PetscObject *)&ndorder));
    546:   if (ndorder) {
    547:     PetscCall(PetscObjectReference((PetscObject)ndorder));
    548:     *is = ndorder;
    549:     PetscFunctionReturn(PETSC_SUCCESS);
    550:   }

    552:   PetscCall(PetscObjectGetComm((PetscObject)part, &comm));
    553:   /* count the number of partitions, i.e., virtual processors */
    554:   PetscCall(ISGetLocalSize(part, &n));
    555:   PetscCall(ISGetIndices(part, &indices));
    556:   np = 0;
    557:   for (PetscInt i = 0; i < n; i++) PetscCall(PetscMPIIntCast(PetscMax(np, indices[i]), &np));
    558:   PetscCallMPI(MPIU_Allreduce(&np, &npt, 1, MPI_INT, MPI_MAX, comm));
    559:   np = npt + 1; /* so that it looks like a MPI_Comm_size output */

    561:   /*
    562:      lsizes - number of elements of each partition on this particular processor
    563:      sums - total number of "previous" nodes for any particular partition
    564:      starts - global number of first element in each partition on this processor
    565:   */
    566:   PetscCall(PetscMalloc3(np, &lsizes, np, &starts, np, &sums));
    567:   PetscCall(PetscArrayzero(lsizes, np));
    568:   for (PetscInt i = 0; i < n; i++) lsizes[indices[i]]++;
    569:   PetscCallMPI(MPIU_Allreduce(lsizes, sums, np, MPIU_INT, MPI_SUM, comm));
    570:   PetscCallMPI(MPI_Scan(lsizes, starts, np, MPIU_INT, MPI_SUM, comm));
    571:   for (PetscMPIInt i = 0; i < np; i++) starts[i] -= lsizes[i];
    572:   for (PetscMPIInt i = 1; i < np; i++) {
    573:     sums[i] += sums[i - 1];
    574:     starts[i] += sums[i - 1];
    575:   }

    577:   /*
    578:       For each local index give it the new global number
    579:   */
    580:   PetscCall(PetscMalloc1(n, &newi));
    581:   for (PetscInt i = 0; i < n; i++) newi[i] = starts[indices[i]]++;
    582:   PetscCall(PetscFree3(lsizes, starts, sums));

    584:   PetscCall(ISRestoreIndices(part, &indices));
    585:   PetscCall(ISCreateGeneral(comm, n, newi, PETSC_OWN_POINTER, is));
    586:   PetscCall(ISSetPermutation(*is));
    587:   PetscFunctionReturn(PETSC_SUCCESS);
    588: }

    590: /*@
    591:   ISPartitioningCount - Takes a `IS` that represents a partitioning (the MPI rank that each local entry belongs to) and determines the number of
    592:   resulting elements on each (partition) rank

    594:   Collective

    596:   Input Parameters:
    597: + part - a partitioning as generated by `MatPartitioningApply()` or `MatPartitioningApplyND()`
    598: - len  - length of the array count, this is the total number of partitions

    600:   Output Parameter:
    601: . count - array of length size, to contain the number of elements assigned
    602:         to each partition, where size is the number of partitions generated
    603:          (see notes below).

    605:   Level: advanced

    607:   Notes:
    608:   By default the number of partitions generated (and thus the length
    609:   of count) is the size of the communicator associated with `IS`,
    610:   but it can be set by `MatPartitioningSetNParts()`.

    612:   The resulting array of lengths can for instance serve as input of `PCBJacobiSetTotalBlocks()`.

    614:   If the partitioning has been obtained by `MatPartitioningApplyND()`, the returned count does not include the separators.

    616: .seealso: [](sec_scatter), `IS`, `MatPartitioningCreate()`, `AOCreateBasic()`, `ISPartitioningToNumbering()`,
    617:           `MatPartitioningSetNParts()`, `MatPartitioningApply()`, `MatPartitioningApplyND()`
    618: @*/
    619: PetscErrorCode ISPartitioningCount(IS part, PetscInt len, PetscInt count[])
    620: {
    621:   MPI_Comm        comm;
    622:   PetscInt        i, n, *lsizes;
    623:   const PetscInt *indices;

    625:   PetscFunctionBegin;
    626:   PetscCall(PetscObjectGetComm((PetscObject)part, &comm));
    627:   if (len == PETSC_DEFAULT) {
    628:     PetscMPIInt size;

    630:     PetscCallMPI(MPI_Comm_size(comm, &size));
    631:     len = size;
    632:   }

    634:   /* count the number of partitions */
    635:   PetscCall(ISGetLocalSize(part, &n));
    636:   PetscCall(ISGetIndices(part, &indices));
    637:   if (PetscDefined(USE_DEBUG)) {
    638:     PetscInt np = 0, npt;
    639:     for (i = 0; i < n; i++) np = PetscMax(np, indices[i]);
    640:     PetscCallMPI(MPIU_Allreduce(&np, &npt, 1, MPIU_INT, MPI_MAX, comm));
    641:     np = npt + 1; /* so that it looks like a MPI_Comm_size output */
    642:     PetscCheck(np <= len, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Length of count array %" PetscInt_FMT " is less than number of partitions %" PetscInt_FMT, len, np);
    643:   }

    645:   /*
    646:         lsizes - number of elements of each partition on this particular processor
    647:         sums - total number of "previous" nodes for any particular partition
    648:         starts - global number of first element in each partition on this processor
    649:   */
    650:   PetscCall(PetscCalloc1(len, &lsizes));
    651:   for (i = 0; i < n; i++) {
    652:     if (indices[i] > -1) lsizes[indices[i]]++;
    653:   }
    654:   PetscCall(ISRestoreIndices(part, &indices));
    655:   PetscCallMPI(MPIU_Allreduce(lsizes, count, len, MPIU_INT, MPI_SUM, comm));
    656:   PetscCall(PetscFree(lsizes));
    657:   PetscFunctionReturn(PETSC_SUCCESS);
    658: }

    660: /*@
    661:   ISAllGather - Given an index set `IS` on each processor, generates a large
    662:   index set (same on each processor) by concatenating together each
    663:   processors index set.

    665:   Collective

    667:   Input Parameter:
    668: . is - the distributed index set

    670:   Output Parameter:
    671: . isout - the concatenated index set (same on all processors)

    673:   Level: intermediate

    675:   Notes:
    676:   `ISAllGather()` is clearly not scalable for large index sets.

    678:   The `IS` created on each processor must be created with a common
    679:   communicator (e.g., `PETSC_COMM_WORLD`). If the index sets were created
    680:   with `PETSC_COMM_SELF`, this routine will not work as expected, since
    681:   each process will generate its own new `IS` that consists only of
    682:   itself.

    684:   The communicator for this new `IS` is `PETSC_COMM_SELF`

    686: .seealso: [](sec_scatter), `IS`, `ISCreateGeneral()`, `ISCreateStride()`, `ISCreateBlock()`
    687: @*/
    688: PetscErrorCode ISAllGather(IS is, IS *isout)
    689: {
    690:   PetscInt       *indices, n, i, N, step, first;
    691:   const PetscInt *lindices;
    692:   MPI_Comm        comm;
    693:   PetscMPIInt     size, *sizes = NULL, *offsets = NULL, nn;
    694:   PetscBool       stride;

    696:   PetscFunctionBegin;
    698:   PetscAssertPointer(isout, 2);

    700:   PetscCall(PetscObjectGetComm((PetscObject)is, &comm));
    701:   PetscCallMPI(MPI_Comm_size(comm, &size));
    702:   PetscCall(ISGetLocalSize(is, &n));
    703:   PetscCall(PetscObjectTypeCompare((PetscObject)is, ISSTRIDE, &stride));
    704:   if (size == 1 && stride) { /* should handle parallel ISStride also */
    705:     PetscCall(ISStrideGetInfo(is, &first, &step));
    706:     PetscCall(ISCreateStride(PETSC_COMM_SELF, n, first, step, isout));
    707:   } else {
    708:     PetscCall(PetscMalloc2(size, &sizes, size, &offsets));

    710:     PetscCall(PetscMPIIntCast(n, &nn));
    711:     PetscCallMPI(MPI_Allgather(&nn, 1, MPI_INT, sizes, 1, MPI_INT, comm));
    712:     offsets[0] = 0;
    713:     for (i = 1; i < size; i++) {
    714:       PetscInt s = offsets[i - 1] + sizes[i - 1];
    715:       PetscCall(PetscMPIIntCast(s, &offsets[i]));
    716:     }
    717:     N = offsets[size - 1] + sizes[size - 1];

    719:     PetscCall(PetscMalloc1(N, &indices));
    720:     PetscCall(ISGetIndices(is, &lindices));
    721:     PetscCallMPI(MPI_Allgatherv((void *)lindices, nn, MPIU_INT, indices, sizes, offsets, MPIU_INT, comm));
    722:     PetscCall(ISRestoreIndices(is, &lindices));
    723:     PetscCall(PetscFree2(sizes, offsets));

    725:     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, N, indices, PETSC_OWN_POINTER, isout));
    726:   }
    727:   PetscFunctionReturn(PETSC_SUCCESS);
    728: }

    730: /*@C
    731:   ISAllGatherColors - Given a set of colors on each processor, generates a large
    732:   set (same on each processor) by concatenating together each processors colors

    734:   Collective

    736:   Input Parameters:
    737: + comm     - communicator to share the indices
    738: . n        - local size of set
    739: - lindices - local colors

    741:   Output Parameters:
    742: + outN       - total number of indices
    743: - outindices - all of the colors

    745:   Level: intermediate

    747:   Note:
    748:   `ISAllGatherColors()` is clearly not scalable for large index sets.

    750: .seealso: `ISColoringValue`, `ISColoring()`, `ISCreateGeneral()`, `ISCreateStride()`, `ISCreateBlock()`, `ISAllGather()`
    751: @*/
    752: PetscErrorCode ISAllGatherColors(MPI_Comm comm, PetscInt n, ISColoringValue lindices[], PetscInt *outN, ISColoringValue *outindices[])
    753: {
    754:   ISColoringValue *indices;
    755:   PetscInt         N;
    756:   PetscMPIInt      size, *offsets = NULL, *sizes = NULL, nn;

    758:   PetscFunctionBegin;
    759:   PetscCall(PetscMPIIntCast(n, &nn));
    760:   PetscCallMPI(MPI_Comm_size(comm, &size));
    761:   PetscCall(PetscMalloc2(size, &sizes, size, &offsets));

    763:   PetscCallMPI(MPI_Allgather(&nn, 1, MPI_INT, sizes, 1, MPI_INT, comm));
    764:   offsets[0] = 0;
    765:   for (PetscMPIInt i = 1; i < size; i++) offsets[i] = offsets[i - 1] + sizes[i - 1];
    766:   N = offsets[size - 1] + sizes[size - 1];
    767:   PetscCall(PetscFree2(sizes, offsets));

    769:   PetscCall(PetscMalloc1(N + 1, &indices));
    770:   PetscCallMPI(MPI_Allgatherv(lindices, nn, MPIU_COLORING_VALUE, indices, sizes, offsets, MPIU_COLORING_VALUE, comm));

    772:   *outindices = indices;
    773:   if (outN) *outN = N;
    774:   PetscFunctionReturn(PETSC_SUCCESS);
    775: }

    777: /*@
    778:   ISComplement - Given an index set `IS` generates the complement index set. That is
    779:   all indices that are NOT in the given set.

    781:   Collective

    783:   Input Parameters:
    784: + is   - the index set
    785: . nmin - the first index desired in the local part of the complement
    786: - nmax - the largest index desired in the local part of the complement (note that all indices in `is` must be greater or equal to `nmin` and less than `nmax`)

    788:   Output Parameter:
    789: . isout - the complement

    791:   Level: intermediate

    793:   Notes:
    794:   The communicator for `isout` is the same as for the input `is`

    796:   For a parallel `is`, this will generate the local part of the complement on each process

    798:   To generate the entire complement (on each process) of a parallel `is`, first call `ISAllGather()` and then
    799:   call this routine.

    801: .seealso: [](sec_scatter), `IS`, `ISCreateGeneral()`, `ISCreateStride()`, `ISCreateBlock()`, `ISAllGather()`
    802: @*/
    803: PetscErrorCode ISComplement(IS is, PetscInt nmin, PetscInt nmax, IS *isout)
    804: {
    805:   const PetscInt *indices;
    806:   PetscInt        n, i, j, unique, cnt, *nindices;
    807:   PetscBool       sorted;

    809:   PetscFunctionBegin;
    811:   PetscAssertPointer(isout, 4);
    812:   PetscCheck(nmin >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "nmin %" PetscInt_FMT " cannot be negative", nmin);
    813:   PetscCheck(nmin <= nmax, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "nmin %" PetscInt_FMT " cannot be greater than nmax %" PetscInt_FMT, nmin, nmax);
    814:   PetscCall(ISSorted(is, &sorted));
    815:   PetscCheck(sorted, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Index set must be sorted");

    817:   PetscCall(ISGetLocalSize(is, &n));
    818:   PetscCall(ISGetIndices(is, &indices));
    819:   if (PetscDefined(USE_DEBUG)) {
    820:     for (i = 0; i < n; i++) {
    821:       PetscCheck(indices[i] >= nmin, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Index %" PetscInt_FMT "'s value %" PetscInt_FMT " is smaller than minimum given %" PetscInt_FMT, i, indices[i], nmin);
    822:       PetscCheck(indices[i] < nmax, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Index %" PetscInt_FMT "'s value %" PetscInt_FMT " is larger than maximum given %" PetscInt_FMT, i, indices[i], nmax);
    823:     }
    824:   }
    825:   /* Count number of unique entries */
    826:   unique = (n > 0);
    827:   for (i = 0; i < n - 1; i++) {
    828:     if (indices[i + 1] != indices[i]) unique++;
    829:   }
    830:   PetscCall(PetscMalloc1(nmax - nmin - unique, &nindices));
    831:   cnt = 0;
    832:   for (i = nmin, j = 0; i < nmax; i++) {
    833:     if (j < n && i == indices[j]) do {
    834:         j++;
    835:       } while (j < n && i == indices[j]);
    836:     else nindices[cnt++] = i;
    837:   }
    838:   PetscCheck(cnt == nmax - nmin - unique, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of entries found in complement %" PetscInt_FMT " does not match expected %" PetscInt_FMT, cnt, nmax - nmin - unique);
    839:   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)is), cnt, nindices, PETSC_OWN_POINTER, isout));
    840:   PetscCall(ISSetInfo(*isout, IS_SORTED, IS_GLOBAL, PETSC_FALSE, PETSC_TRUE));
    841:   PetscCall(ISRestoreIndices(is, &indices));
    842:   PetscFunctionReturn(PETSC_SUCCESS);
    843: }