/*
 *  Copyright (C) 2026, Northwestern University
 *  See COPYRIGHT notice in top-level directory.
 */

#ifdef HAVE_CONFIG_H
# include <config.h>
#endif

#include <stdio.h>
#include <string.h> /* memcpy(), strerror() */
#include <unistd.h> /* lseek() */
#include <limits.h> /* INT_MAX */

#include <errno.h>  /* errno */
#include <assert.h>

#include "gioi.h"

static int use_alltoallw;

typedef struct {
    MPI_Offset  num; /* number of elements in off[] and len[] */
    MPI_Offset *off; /* list of write offsets by this rank in round m */
    MPI_Offset *len; /* list of write lengths by this rank in round m */
} off_len_list;

/*----< LUSTRE_Calc_aggregator() >-------------------------------------------*/
static int
LUSTRE_Calc_aggregator(GIO_File    fh,
                       MPI_Offset  off,
                       MPI_Offset *len)
{
    MPI_Offset stripe_id, avail_bytes;

    stripe_id = off / fh->hints->striping_unit;

    avail_bytes = (stripe_id + 1) * fh->hints->striping_unit - off;
    if (avail_bytes < *len) {
        /* The request [off, off+len) has only [off, off+avail_bytes) part
         * falling into aggregator's file domain */
        *len = avail_bytes;
    }
    /* return the index to aggr_ranks[] */
    return (stripe_id % fh->hints->cb_nodes);
}

#define CACHE_REQ(list, nelems, buf) {   \
    MPI_Aint buf_addr;                   \
    list.len[list.num] = nelems;         \
    MPI_Get_address(buf, &buf_addr);     \
    list.off[list.num] = buf_addr;       \
    list.num++;                          \
}

/*----< LUSTRE_Fill_send_buffer() >------------------------------------------*/
/* This subroutine is called only when bview is not contiguous */
static int
LUSTRE_Fill_send_buffer(GIO_File           fh,
                        const void        *buf,
                        char             **send_buf,
                        size_t             send_total_size,
                        const MPI_Offset  *send_size,
                        char             **self_buf,
                        off_len_list      *send_list)
{
    char *user_buf_ptr=NULL, *send_buf_ptr=NULL, *same_buf_ptr=NULL;
    int aggr, first_aggr, isUserBuf;
    MPI_Offset send_size_rem=0, size, copy_size=0;
    MPI_Offset len, rem_len, user_buf_idx, rem_off;

#if GIO_DEBUG_MODE == 1
    char *orig_ptr;
#endif

#if GIO_PROFILING_MODE == 1
    int num_memcpy=0;
#endif

    *self_buf = NULL;

    /* user_buf_idx is the index offset to buf, indicating the starting
     * location to be copied.
     *
     * bview stores the flattened offset-length pairs of the user buffer.
     * bview.npairs: the number of offset-length pairs
     * bview.off[i]: the ith pair's byte offset to buf. Note bview.off[] may
     *      not be sorted in an increasing order, unlike fileview which is
     *      required by MPI standard to be sorted in a monotonically
     *      non-decreasing order.
     * bview.len[i]: length of the ith pair
     * bview.idx: index to the offset-length pair currently being processed,
     *      incremented each round, ranging from 0 to (bview.npairs-1).
     * bview.rem: amount of data in the pair that has not been copied over,
     *      changed each round, only relevant to the pair pointed by bview.idx.
     */
    user_buf_idx = fh->bview.off[fh->bview.idx]
                 + fh->bview.len[fh->bview.idx]
                 - fh->bview.rem;
                 /* in case data left to be copied from previous round */

#if GIO_DEBUG_MODE == 1
    /* If this request is not zero-sized, fh->fview.npairs has been adjusted
     * to be a positive number at the call to GIO_write_at_all().
     */
    assert(fh->fview.npairs > 0);

    orig_ptr = (char*)buf + user_buf_idx;
#endif

    /* fh->fview.npairs: the number of file offset-length pairs this rank
     *     writes to.
     * fh->fview.idx: the index to the pair of fh->fview.offs[], and
     *     fh->fview.len[] that have been processed in the previous round.
     * The while loop below packs write data into send buffers, send_buf[],
     * based on this rank's off-len pairs in its file view,
     */
    rem_off = fh->fview.off[fh->fview.idx]
            + fh->fview.len[fh->fview.idx]
            - fh->fview.rem;
    rem_len = fh->fview.rem;

    first_aggr = -1;
    isUserBuf = 0;
    while (send_total_size > 0) {
        /* this off-len request may span to more than one I/O aggregator */
        while (rem_len != 0) {
            len = rem_len;
            aggr = LUSTRE_Calc_aggregator(fh, rem_off, &len);
            /* NOTE: len may be modified by LUSTRE_Calc_aggregator() to be no
             * more than a file stripe unit size that aggregator "aggr" is
             * responsible for. Note aggr is not the MPI rank ID in fh->comm,
             * It is the index to array fh->hints->aggr_ranks[].
             *
             * Now len is the amount of data in ith off-len pair that should be
             * sent to aggregator aggr. Note aggr can also be self. In this
             * case, data is also packed into send_buf[aggr] or pointed to a
             * segment of buf when the data to be packed is contiguous, and
             * then data in send_buf[aggr] will later be copied over to write
             * buffer in MEMCPY_UNPACK, instead of calling MPI_Issend to send.
             *
             * send_size[aggr] is the data amount of this rank needs to send to
             * aggregator aggr in this round.
             *
             * len and send_size[aggr] are both always <= striping_unit
             */

            if (first_aggr != aggr) {
                assert(send_size_rem == 0);
                first_aggr = aggr;
                isUserBuf = 1;
                send_size_rem = send_size[aggr];
                copy_size = 0;
                same_buf_ptr = (char*)buf + user_buf_idx; /* no increment */

                /* user_buf_ptr and send_buf_ptr increment after each memcpy */
                user_buf_ptr = same_buf_ptr;
                if (send_buf != NULL)
                    send_buf_ptr = send_buf[aggr];
#if GIO_DEBUG_MODE == 1
                /* orig_ptr is for checking whether user_buf_ptr is reused */
                orig_ptr = user_buf_ptr;
#endif
            }

            /* copy len amount of data from buf to send_buf[aggr] */
            size = len;

            while (size > 0) {
                MPI_Offset size_in_buf = MIN(size, fh->bview.rem);
                copy_size     += size_in_buf;
                user_buf_idx  += size_in_buf;
                send_size_rem -= size_in_buf;
                fh->bview.rem -= size_in_buf;
                if (fh->bview.rem == 0) { /* move on to next off-len pair */
                    if (fh->bview.npairs > 1) {
                        /* user buffer type is not contiguous */
                        if (send_size_rem) {
                            /* after this round of memcpy, send_buf[aggr] is
                             * still not full
                             */
                            isUserBuf = 0;
                            memcpy(send_buf_ptr, user_buf_ptr, copy_size);
#if GIO_PROFILING_MODE == 1
                            num_memcpy++;
#endif
#if GIO_DEBUG_MODE == 1
                            assert(orig_ptr == user_buf_ptr);
                            user_buf_ptr += copy_size;
#endif
                            send_buf_ptr += copy_size;
                            copy_size = 0;
                        } else if (isUserBuf == 0) {
                            /* send_buf[aggr] is full and not using user buf to
                             * send, copy over the remaining delayed data
                             */
                            memcpy(send_buf_ptr, user_buf_ptr, copy_size);
#if GIO_PROFILING_MODE == 1
                            num_memcpy++;
#endif
#if GIO_DEBUG_MODE == 1
                            assert(orig_ptr == user_buf_ptr);
                            user_buf_ptr += copy_size;
#endif
                        }
                    }

                    /* update bview.idx, bview.rem, and user_buf_idx */
                    fh->bview.idx++;
                    if (fh->bview.idx == fh->bview.npairs) {
                        /* Done with all the copying */
#if GIO_DEBUG_MODE == 1
                        assert(size == size_in_buf);
                        assert(rem_len == len);
                        assert(send_total_size == len);
#endif
                        break;
                    }
                    user_buf_idx  = fh->bview.off[fh->bview.idx];
                    fh->bview.rem = fh->bview.len[fh->bview.idx];
                    user_buf_ptr = (char*) buf + user_buf_idx;
#if GIO_DEBUG_MODE == 1
                    /* reset orig_ptr for checking if user_buf_ptr is reused */
                    orig_ptr = user_buf_ptr;
#endif
                }
                else if (send_size_rem == 0 && isUserBuf == 0) {
                    /* bview.rem > 0, send_buf[aggr] is full, and not using
                     * user buf to send, copy over the remaining delayed data
                     */
                    memcpy(send_buf_ptr, user_buf_ptr, copy_size);
#if GIO_PROFILING_MODE == 1
                    num_memcpy++;
#endif
#if GIO_DEBUG_MODE == 1
                    assert(orig_ptr == user_buf_ptr);
                    user_buf_ptr += copy_size;
#endif
                }
                size -= size_in_buf;
            }

            if (send_size_rem == 0) { /* data for aggr is fully packed */
                first_aggr = -1;

                if (aggr != fh->my_cb_nodes_index) {
                    /* send only if not self rank */
                    if (isUserBuf)
                        CACHE_REQ(send_list[aggr], send_size[aggr],
                                  same_buf_ptr)
                    else
                        CACHE_REQ(send_list[aggr], send_size[aggr],
                                  send_buf[aggr])
                }
                else if (isUserBuf) {
                    /* send buffer is also (part of) user's buf. Return the
                     * buffer pointer, so data for self can be directly
                     * unpacked from user buf to write buffer.
                     */
                    *self_buf = same_buf_ptr;
                }
            }
            /* len is the amount of data copied */
            rem_off           += len;
            rem_len           -= len;
            fh->fview.rem -= len;
            send_total_size   -= len;
            if (send_total_size == 0) break;
        }
        if (send_total_size == 0) break;

        /* done with this off-len pair, move on to the next */
        if (fh->fview.rem == 0) {
            fh->fview.idx++;
            fh->fview.rem = fh->fview.len[fh->fview.idx];
        }
        rem_off = fh->fview.off[fh->fview.idx];
        rem_len = fh->fview.rem;
    }

#if GIO_PROFILING_MODE == 1
    gio_wr_count[8] = MAX(num_memcpy, gio_wr_count[8]);
#endif

    return GIO_NOERR;
}


#define MEMCPY_UNPACK(x, inbuf, start, count, outbuf) {          \
    int _k;                                                      \
    char *_ptr = (inbuf);                                        \
    MPI_Offset *mem_ptrs = others_req[x].ptr + (start);          \
    MPI_Offset *mem_lens = others_req[x].len + (start);          \
    for (_k=0; _k<count; _k++) {                                 \
        memcpy((outbuf) + mem_ptrs[_k], _ptr, mem_lens[_k]);     \
        _ptr += mem_lens[_k];                                    \
    }                                                            \
}

/*----< Exch_data_send() >---------------------------------------------------*/
static int
Exch_data_send(GIO_File  fh,
    const void          *buf,          /* user buffer */
          char          *write_buf,    /* OUT: internal buffer used to write
                                        * to file, only matter when send to
                                        * self */
          char         **send_buf_ptr, /* OUT: [cb_nodes] point to internal
                                        * send buffer */
    const MPI_Offset    *send_size,    /* [cb_nodes] send_size[i] is amount of
                                        * this rank sent to aggregator i */
          MPI_Offset     self_count,   /* No. offset-length pairs sent to self
                                        * rank */
          MPI_Offset     start_pos,    /* others_req[myrank].cur */
    const GIOI_Access   *others_req,   /* [nprocs] only used when send to self,
                                        * others_req[myrank] */
    const MPI_Offset    *buf_idx,      /* [cb_nodes] indices to user buffer
                                        * for sending this rank's write data
                                        * to aggregator i */
          off_len_list  *send_list)    /* OUT: displacement-length pairs of
                                        * send buffer */
{
    int i, err, myrank, cb_nodes;

    *send_buf_ptr = NULL;

    MPI_Comm_rank(fh->comm, &myrank);

    cb_nodes = fh->hints->cb_nodes;
    if (fh->bview.npairs <= 1) {
        /* If buftype is contiguous, data can be directly sent from user buf
         * at location given by buf_idx.
         */
        for (i = 0; i < cb_nodes; i++) {
            if (send_size[i] && i != fh->my_cb_nodes_index)
                CACHE_REQ(send_list[i], send_size[i], (char*)buf + buf_idx[i]);
        }
    } else {
        char **send_buf, *self_buf;

        /* total send size of this round */
        size_t send_total_size = 0;
        for (i = 0; i < cb_nodes; i++)
            send_total_size += send_size[i];

        if (send_total_size == 0) return GIO_NOERR;

        /* The user buffer to be used to send in this round is not contiguous,
         * allocate send_buf[], a contiguous space, copy data to send_buf,
         * including ones to be sent to self, and then use send_buf to send.
         */
        send_buf = (char **) GIOI_Malloc(sizeof(char*) * cb_nodes);
        send_buf[0] = (char *) GIOI_Malloc(send_total_size);
        for (i = 1; i < cb_nodes; i++)
            send_buf[i] = send_buf[i - 1] + send_size[i - 1];

        err = LUSTRE_Fill_send_buffer(fh, buf, send_buf, send_total_size,
                                      send_size, &self_buf, send_list);
        if (err != GIO_NOERR) return err;

        /* Send buffers must not be touched before MPI_Waitall() is completed,
         * and thus send_buf will be freed in LUSTRE_Exch_and_write()
         */

        if (fh->my_cb_nodes_index >= 0 && send_size[fh->my_cb_nodes_index] > 0) {
            /* contents of user buf that must be sent to self has been copied
             * into send_buf[fh->my_cb_nodes_index]. Now unpack it into
             * write_buf.
             */
            if (self_buf == NULL) self_buf = send_buf[fh->my_cb_nodes_index];
            MEMCPY_UNPACK(myrank, self_buf, start_pos, self_count, write_buf);
        }

        *send_buf_ptr = send_buf[0];
        GIOI_Free(send_buf);
    }

    return GIO_NOERR;
}

/*----< heap_merge() >-------------------------------------------------------*/
/* This heap-merge sort also coalesces sorted offset-length pairs whenever
 * possible.
 *
 * Heapify(a, i, heapsize); Algorithm from Cormen et al. pg. 143 modified for a
 * heap with smallest element at root. The recursion has been removed so that
 * there are no function calls. Function calls are too expensive.
 */
static void
heap_merge(const GIOI_Access *others_req,
           const MPI_Offset  *count,
           MPI_Offset        *srt_off,
           MPI_Offset        *srt_len,
           const MPI_Offset  *start_pos,
           int                nprocs,
           int                nprocs_recv,
           MPI_Offset        *total_elements)
{
    typedef struct {
        MPI_Offset *off_list;
        MPI_Offset *len_list;
        MPI_Offset nelem;
    } heap_struct;

    heap_struct *a, tmp;
    int i, j, heapsize, l, r, k, smallest;

    a = (heap_struct *) GIOI_Malloc(sizeof(heap_struct) * (nprocs_recv + 1));

    j = 0;
    for (i = 0; i < nprocs; i++) {
        if (count[i]) {
            a[j].off_list = others_req[i].off + start_pos[i];
            a[j].len_list = others_req[i].len + start_pos[i];
            a[j].nelem = count[i];
            j++;
        }
    }

#define SWAP(x, y, tmp) { tmp = x ; x = y ; y = tmp ; }

    heapsize = nprocs_recv;

    /* Build a heap out of the first element from each list, with the smallest
     * element of the heap at the root. The first for loop is to find and move
     * the smallest a[*].off_list[0] to a[0].
     */
    for (i = heapsize / 2 - 1; i >= 0; i--) {
        k = i;
        for (;;) {
            r = 2 * (k + 1);
            l = r - 1;
            if ((l < heapsize) && (*(a[l].off_list) < *(a[k].off_list)))
                smallest = l;
            else
                smallest = k;

            if ((r < heapsize) && (*(a[r].off_list) < *(a[smallest].off_list)))
                smallest = r;

            if (smallest != k) {
                SWAP(a[k], a[smallest], tmp);
                k = smallest;
            } else
                break;
        }
    }

    /* The heap keeps the smallest element in its first element, i.e.
     * a[0].off_list[0].
     */
    j = 0;
    for (i = 0; i < *total_elements; i++) {
        /* extract smallest element from heap, i.e. the root */
        if (j == 0 || srt_off[j - 1] + srt_len[j - 1] < *(a[0].off_list)) {
            srt_off[j] = *(a[0].off_list);
            srt_len[j] = *(a[0].len_list);
            j++;
        } else {
            /* this offset-length pair can be coalesced into the previous one */
            srt_len[j - 1] = *(a[0].off_list) + *(a[0].len_list) - srt_off[j - 1];
        }
        (a[0].nelem)--;

        if (a[0].nelem) {
            (a[0].off_list)++;
            (a[0].len_list)++;
        } else {
            a[0] = a[heapsize - 1];
            heapsize--;
        }

        /* Heapify(a, 0, heapsize); */
        k = 0;
        for (;;) {
            r = 2 * (k + 1);
            l = r - 1;
            if ((l < heapsize) && (*(a[l].off_list) < *(a[k].off_list)))
                smallest = l;
            else
                smallest = k;

            if ((r < heapsize) && (*(a[r].off_list) < *(a[smallest].off_list)))
                smallest = r;

            if (smallest != k) {
                SWAP(a[k], a[smallest], tmp);
                k = smallest;
            } else
                break;
        }
    }
    GIOI_Free(a);
    *total_elements = j;
}

/*----< Exch_data_recv() >---------------------------------------------------*/
static int
Exch_data_recv(GIO_File  fh,
    const void          *buf,         /* user buffer */
          char          *write_buf,   /* OUT: internal buffer used to write
                                       * to file */
          char         **recv_buf,    /* OUT: [nbufs] internal buffer used to
                                       * receive from other processes */
    const MPI_Offset    *recv_size,   /* [nprocs] recv_size[i] is amount of
                                       * this aggregator recv from rank i */
          MPI_Offset     range_off,   /* starting file offset of this
                                       * aggregator's write region */
          MPI_Offset     range_size,  /* amount of this aggregator's write
                                       * region */
    const MPI_Offset    *recv_count,  /* [nprocs] recv_count[i] is the number
                                       * of offset-length pairs received from
                                       * rank i */
    const MPI_Offset    *start_pos,   /* [nprocs] start_pos[i] starting value
                                       * of others_req[i].cur */
    const GIOI_Access   *others_req,  /* [nprocs] others_req[i] is rank i's
                                       * write requests fall into this
                                       * aggregator's file domain */
    const MPI_Offset    *buf_idx,     /* [cb_nodes] indices to user buffer
                                       * offsets for sending this rank's
                                       * write data to aggregator i */
          off_len_list  *srt_off_len, /* OUT: list of write offset-length
                                       * pairs of this aggregator */
          off_len_list  *recv_list)   /* OUT: displacement-length pairs of
                                       * recv buffer */
{
    char *buf_ptr, *contig_buf;
    size_t alloc_sz;
    int i, j, nprocs, myrank, nprocs_recv, hole, build_srt_off_len;
    MPI_Offset sum_recv;

    MPI_Comm_size(fh->comm, &nprocs);
    MPI_Comm_rank(fh->comm, &myrank);

    /* srt_off_len contains the file offset-length pairs to be written by this
     * aggregator at this round. The file region starts from range_off with
     * size of range_size.
     */

    srt_off_len->num = 0;
    srt_off_len->off = NULL;
    sum_recv = 0;
    nprocs_recv = 0;

    /* calculate receive metadata */
    j = -1;
    for (i = 0; i < nprocs; i++) {
        srt_off_len->num += recv_count[i];
        if (j == -1 && recv_count[i] > 0) j = i;
        sum_recv += recv_size[i];
        if (recv_size[i])
            nprocs_recv++;
    }

    if (nprocs_recv == 0) return GIO_NOERR;

    /* determine whether checking holes is necessary */
    if (srt_off_len->num == 0) {
        /* this process has nothing to receive and hence no hole */
        build_srt_off_len = 0;
        hole = 0;
    } else if (srt_off_len->num == 1) {
        build_srt_off_len = 0;
        hole = 0;
        alloc_sz = sizeof(MPI_Offset) + sizeof(MPI_Offset);
        srt_off_len->off = (MPI_Offset*) GIOI_Malloc(alloc_sz);
        srt_off_len->len = (MPI_Offset*) (srt_off_len->off + 1);
        srt_off_len->off[0] = others_req[j].off[start_pos[j]];
        srt_off_len->len[0] = others_req[j].len[start_pos[j]];
    } else if (fh->hints->ds_write == GIOI_HINT_ENABLE) {
        /* skip building of srt_off_len and proceed to read-modify-write */
        build_srt_off_len = 0;
        /* assuming there are holes */
        hole = 1;
    } else if (fh->hints->ds_write == GIOI_HINT_AUTO) {
        if (DO_HEAP_MERGE(nprocs_recv, srt_off_len->num)) {
            /* When the number of sorted offset-length lists or the total
             * number of offset-length pairs are too large, the heap-merge sort
             * below for building srt_off_len can become very expensive. Such
             * sorting is also used to check holes to determine whether
             * read-modify-write is necessary.
             */
            build_srt_off_len = 0;
            /* assuming there are holes */
            hole = 1;
        }
        else /* heap-merge is less expensive, proceed to build srt_off_len */
            build_srt_off_len = 1;

#if GIO_PROFILING_MODE == 1
        if (build_srt_off_len) {
            gio_wr_count[1]++;
            gio_wr_count[2] = MAX(gio_wr_count[2], srt_off_len->num);
            gio_wr_count[3] = MAX(gio_wr_count[3], nprocs_recv);
        } else {
            gio_wr_count[4]++;
            gio_wr_count[5] = MAX(gio_wr_count[5], srt_off_len->num);
            gio_wr_count[6] = MAX(gio_wr_count[6], nprocs_recv);
        }
#endif
    } else { /* if (fh->hints->ds_write == GIOI_HINT_DISABLE) */
        /* User explicitly disable data sieving to skip read-modify-write.
         * Whether or not there is a hole is not important. However,
         * srt_off_len must be constructed to merge all others_req[] into a
         * single sorted list. This step is necessary because after this
         * subroutine returns, write data from all non-aggregators will be
         * packed into the write_buf, with a possibility of overlaps, and
         * as srt_off_len stores the coalesced offset-length pairs of
         * individual non-contiguous write requests, it is used to write them
         * to the file.
         */
        build_srt_off_len = 1;
    }

    if (build_srt_off_len) {
        /* merge all the offset-length pairs from others_req[] (already sorted
         * individually) into a single list of offset-length pairs.
         */
        alloc_sz = sizeof(MPI_Offset) + sizeof(MPI_Offset);
        srt_off_len->off = (MPI_Offset*) GIOI_Malloc(alloc_sz * srt_off_len->num);
        srt_off_len->len = (MPI_Offset*) (srt_off_len->off + srt_off_len->num);

#if GIO_PROFILING_MODE == 1
        double curT = MPI_Wtime();
#endif
        heap_merge(others_req, recv_count, srt_off_len->off, srt_off_len->len,
                   start_pos, nprocs, nprocs_recv, &srt_off_len->num);

        /* Now, (srt_off_len->off and srt_off_len->len) are in an increasing
         * order of file offsets. In addition, they are coalesced.
         */
#if GIO_PROFILING_MODE == 1
        gio_wr_time[5] += MPI_Wtime() - curT;
#endif
        /* whether or not there are holes */
        hole = (srt_off_len->num > 1);
    }

    /* data sieving */
    if (fh->hints->ds_write != GIOI_HINT_DISABLE && hole) {
        if (fh->skip_read)
            memset(write_buf, 0, range_size);
        else {
            MPI_Offset r_len;
            r_len = GIOI_UFS_read_contig(fh, write_buf, range_size, range_off);
            if (r_len < 0) return (int)r_len;
        }

        /* Once read, holes have been filled and thus the number of
         * offset-length pairs, srt_off_len->num, becomes one.
         */
        srt_off_len->num = 1;
        if (srt_off_len->off == NULL) { /* if has not been malloc-ed yet */
            alloc_sz = sizeof(MPI_Offset) + sizeof(MPI_Offset);
            srt_off_len->off = (MPI_Offset*) GIOI_Malloc(alloc_sz);
            srt_off_len->len = (MPI_Offset*) (srt_off_len->off + 1);
        }
        srt_off_len->off[0] = range_off;
        srt_off_len->len[0] = range_size;
    }

    /* It is possible sum_recv (sum of message sizes to be received) is larger
     * than the size of collective buffer, write_buf, if writes from multiple
     * remote processes overlap. Receiving messages into overlapped regions of
     * the same write_buffer may cause a problem. To avoid it, we allocate a
     * temporary buffer big enough to receive all messages into disjointed
     * regions. Earlier in LUSTRE_Exch_and_write(), write_buf is already
     * allocated with twice amount of the file stripe size, with the second
     * half to be used to receive messages. If sum_recv is smaller than file
     * stripe size, we can reuse that space. But if sum_recv is bigger (an
     * overlap case, which is rare), we allocate a separate buffer of size
     * sum_recv.
     */
    sum_recv -= recv_size[myrank];
    if (sum_recv > fh->hints->striping_unit)
        *recv_buf = (char *) GIOI_Realloc(*recv_buf, sum_recv);
    contig_buf = *recv_buf;

    /* cache displacement-length pairs of receive buffer */
    buf_ptr = contig_buf;
    for (i = 0; i < nprocs; i++) {
        if (recv_size[i] == 0)
            continue;
        if (i != myrank) {
            if (recv_count[i] > 1) {
                CACHE_REQ(recv_list[i], recv_size[i], buf_ptr)
                buf_ptr += recv_size[i];
            } else {
                /* recv_count[i] is the number of noncontiguous offset-length
                 * pairs describing the write requests of rank i that fall
                 * into this aggregator's file domain. When recv_count[i] is 1,
                 * there is only one such pair, meaning the receive message is
                 * to be stored contiguously. Such message can be received
                 * directly into write_buf.
                 */
                CACHE_REQ(recv_list[i], recv_size[i],
                          write_buf + others_req[i].ptr[start_pos[i]])
            }
        } else if (fh->bview.npairs <= 1 && recv_count[i] > 0) {
            /* send/recv to/from self uses memcpy(). The case when buftype is
             * not contiguous will be handled later in Exch_data_send().
             */
            char *fromBuf = (char *) buf + buf_idx[fh->my_cb_nodes_index];
            MEMCPY_UNPACK(i, fromBuf, start_pos[i], recv_count[i], write_buf);
        }
    }
    return GIO_NOERR;
}

/*----< LUSTRE_Calc_my_req() >-----------------------------------------------*/
/* calculates what portions of the read/write requests of this process fall
 * into the file domains of all I/O aggregators.
 *   IN: fh->fview: this rank's flattened write requests
 *       fh->fview.npairs: number of noncontiguous offset-length file requests
 *       fh->fview.off[fh->fview.npairs] file offsets of individual
 *       noncontiguous requests.
 *       fh->fview.len[fh->fview.npairs] lengths of individual noncontiguous
 *       requests.
 *   IN: buf_is_contig: whether the write buffer is contiguous or not
 *   OUT: my_req_ptr[cb_nodes] offset-length pairs of this process's requests
 *        fall into the file domain of each aggregator.
 *   OUT: buf_idx_ptr[cb_nodes] index pointing to the starting location in
 *        user_buf for data to be sent to each aggregator.
 */
static void
LUSTRE_Calc_my_req(GIO_File      fh,
                   int           buf_is_contig,
                   GIOI_Access **my_req_ptr,
                   MPI_Offset  **buf_idx)
{
    int aggr, *aggr_ranks, cb_nodes;
    size_t nelems, alloc_sz;
    MPI_Offset i, l, rem_len, avail_len, *avail_lens, curr_idx, off;
    GIOI_Access *my_req;

    cb_nodes = fh->hints->cb_nodes;

    /* my_req[i].num gives the number of contiguous requests of this process
     * that fall in aggregator i's file domain (not process MPI rank i).
     */
    my_req = (GIOI_Access *) GIOI_Calloc(cb_nodes, sizeof(GIOI_Access));
    *my_req_ptr = my_req;
    if (buf_is_contig) buf_idx[0] = NULL;

    if (fh->fview.size == 0) /* zero-sized request */
        return;

    /* For non-zero sized requests, fh->fview.npairs has been checked and
     * adjusted to a possitive number at the beginning of
     * GIOI_Lustre_write_coll().
     */
    assert(fh->fview.npairs > 0);

    /* First pass is just to calculate how much space is needed to allocate
     * my_req.
     */
    alloc_sz = sizeof(int) + sizeof(MPI_Offset);
    aggr_ranks = (int*) GIOI_Malloc(alloc_sz * fh->fview.npairs);
    avail_lens = (MPI_Offset*) (aggr_ranks + fh->fview.npairs);

    /* Note that MPI standard (MPI 3.1 Chapter 13.1.1 and MPI 4.0 Chapter
     * 14.1.1) requires that the typemap displacements of etype and
     * filetype are non-negative and monotonically non-decreasing. This
     * makes fh->fview.off[] to be monotonically non-decreasing.
     */

/*
Alternative: especially for when fh->fview.npairs is large
1 This rank's aggregate file access region is from st_off to end_off.
2 start with the 1st aggregator ID and keep assign aggregator until next stripe.
  This can avoid too many calls to LUSTRE_Calc_aggregator()
*/

    /* nelems will be the number of offset-length pairs for my_req[] */
    nelems = 0;
    for (i = 0; i < fh->fview.npairs; i++) {
        /* short circuit offset/len processing if zero-byte read/write. */
        if (fh->fview.len[i] == 0)
            continue;

        off = fh->fview.off[i];
        avail_len = fh->fview.len[i];
        /* LUSTRE_Calc_aggregator() modifies the value of 'avail_len' to the
         * amount that is only covered by the aggr's file domain. The remaining
         * (tail) will continue to be processed to determine to whose file
         * domain it belongs. As LUSTRE_Calc_aggregator() can be expensive for
         * large value of fh->fview.npairs, we keep a copy of the returned
         * values of 'aggr' and 'avail_len' in aggr_ranks[] and avail_lens[] to
         * be used in the next for loop (not next iteration).
         *
         * Note the returned value in 'aggr' is the index to aggr_ranks[], i.e.
         * the 'aggr'th element of array aggr_ranks[], rather than the
         * aggregator's MPI rank ID in fh->comm.
         */
        aggr = LUSTRE_Calc_aggregator(fh, off, &avail_len);
        aggr_ranks[i] = aggr;      /* first aggregator ID of this request */
        avail_lens[i] = avail_len; /* length covered, may be < fh->fview.len[i] */
        assert(aggr >= 0 && aggr <= cb_nodes);
        my_req[aggr].num++; /* increment for aggregator aggr */
        nelems++;           /* true number of noncontiguous requests in terms
                             * of file domains */

        /* rem_len is the amount of ith offset-length pair that is not covered
         * by aggregator aggr's file domain.
         */
        rem_len = fh->fview.len[i] - avail_len;
        assert(rem_len >= 0);

        while (rem_len > 0) {
            off += avail_len;    /* move forward to first remaining byte */
            avail_len = rem_len; /* save remaining size, pass to calc */
            aggr = LUSTRE_Calc_aggregator(fh, off, &avail_len);
            my_req[aggr].num++;
            nelems++;
            rem_len -= avail_len;/* reduce remaining length by amount from fd */
        }
    }

    /* allocate space for buf_idx.
     * buf_idx is relevant only if buftype is contiguous. buf_idx[i] gives the
     * starting index in user_buf where data will be sent to aggregator 'i'.
     * This allows sends to be done without extra buffer.
     */
    if (buf_idx != NULL && buf_is_contig) {
        buf_idx[0] = (MPI_Offset *) GIOI_Malloc(sizeof(MPI_Offset) * nelems);
        for (i = 1; i < cb_nodes; i++)
            buf_idx[i] = buf_idx[i - 1] + my_req[i - 1].num;
    }

    /* allocate space for my_req and its members offsets and lens */
    alloc_sz = sizeof(MPI_Offset) * 2;
    my_req[0].off = (MPI_Offset*) GIOI_Malloc(alloc_sz * nelems);
    my_req[0].len = my_req[0].off + my_req[0].num;
    for (i=1; i<cb_nodes; i++) {
        my_req[i].off = my_req[i-1].off + my_req[i-1].num * 2;
        my_req[i].len = my_req[i].off + my_req[i].num;
        my_req[i-1].num = 0; /* reset, will increase where needed later */
    }
    my_req[cb_nodes-1].num = 0;

    for (i=0; i<cb_nodes; i++)
        my_req[i].num = 0; /* reset, will be incremented where needed later */

    /* now fill in my_req */
    curr_idx = 0;
    for (i = 0; i < fh->fview.npairs; i++) {
        /* short circuit offset/len processing if zero-byte read/write. */
        if (fh->fview.len[i] == 0)
            continue;

        off = fh->fview.off[i];
        aggr = aggr_ranks[i];
        assert(aggr >= 0 && aggr <= cb_nodes);
        avail_len = avail_lens[i];

        l = my_req[aggr].num;
        if (buf_idx != NULL && buf_is_contig) {
            buf_idx[aggr][l] = curr_idx;
            curr_idx += avail_len;
        }
        rem_len = fh->fview.len[i] - avail_len;

        /* Each my_req[i] contains the number of this process's noncontiguous
         * requests that fall into aggregator aggr's file domain.
         * my_req[aggr].off[] and my_req[aggr].len store the offsets and
         * lengths of the requests.
         */
        my_req[aggr].off[l] = off;
        my_req[aggr].len[l] = avail_len;
        my_req[aggr].num++;

        while (rem_len != 0) {
            off += avail_len;
            avail_len = rem_len;
            aggr = LUSTRE_Calc_aggregator(fh, off, &avail_len);
            assert(aggr >= 0 && aggr <= cb_nodes);
            l = my_req[aggr].num;
            if (buf_idx != NULL && buf_is_contig) {
                buf_idx[aggr][l] = curr_idx;
                curr_idx += avail_len;
            }
            rem_len -= avail_len;

            my_req[aggr].off[l] = off;
            my_req[aggr].len[l] = avail_len;
            my_req[aggr].num++;
        }
    }
    GIOI_Free(aggr_ranks);
}

/*----< LUSTRE_Calc_others_req() >-------------------------------------------*/
/* LUSTRE_Calc_others_req() calculates what requests from each of other
 * processes fall in this aggregator's file domain.
 *   IN: my_req[cb_nodes]: offset-length pairs of this rank's requests fall
 *       into each of aggregators
 *   OUT: count_others_req_per_proc[i]: number of noncontiguous requests of
 *        rank i that falls in this aggregator's file domain.
 *   OUT: others_req_ptr[nprocs]: requests of each of other ranks fall into
 *        this aggregator's file domain.
 */
static int
LUSTRE_Calc_others_req(GIO_File            fh,
                       const GIOI_Access  *my_req,
                       GIOI_Access       **others_req_ptr)
{
    size_t npairs, alloc_sz, pair_sz;
    int i, err, myrank, nprocs, do_alltoallv, nreqs;
    MPI_Offset *count_my_req_per_proc, *count_others_req_per_proc;
    GIOI_Access *others_req;
    MPI_Request *reqs;

    /* first find out how much to send/recv and from/to whom */

    MPI_Comm_size(fh->comm, &nprocs);
    MPI_Comm_rank(fh->comm, &myrank);

    others_req = (GIOI_Access *) GIOI_Malloc(sizeof(GIOI_Access) * nprocs);
    *others_req_ptr = others_req;

    /* Use my_req[i].num (the number of noncontiguous requests fall in
     * aggregator i's file domain) to set count_others_req_per_proc[j] (the
     * number of noncontiguous requests from process j fall into this
     * aggregator's file domain).
     *
     * The below MPI_Alltoall() is actually an all-to-many, i,e, all ranks
     * send to aggregators only.
     */
    count_my_req_per_proc = (MPI_Offset *) GIOI_Calloc(nprocs * 2, sizeof(MPI_Offset));
    count_others_req_per_proc = count_my_req_per_proc + nprocs;
    for (i=0; i<fh->hints->cb_nodes; i++)
        count_my_req_per_proc[fh->hints->aggr_ranks[i]] = my_req[i].num;

#if 1
    reqs = GIOI_Malloc(sizeof(MPI_Request) * (nprocs + fh->hints->cb_nodes));
    nreqs = 0;
    if (fh->is_agg) {
        for (i=0; i<nprocs; i++)
            MPI_Irecv(count_others_req_per_proc+i, 1, MPI_OFFSET, i, 0,
                      fh->comm, &reqs[nreqs++]);
    }
    for (i=0; i<fh->hints->cb_nodes; i++) {
        int dest = fh->hints->aggr_ranks[i];
        MPI_Issend(&my_req[i].num, 1, MPI_OFFSET, dest, 0, fh->comm,
                   &reqs[nreqs++]);
    }
    if (nreqs) {
#ifdef HAVE_MPI_STATUSES_IGNORE
        MPI_Waitall(nreqs, reqs, MPI_STATUSES_IGNORE);
#else
        MPI_Status *sts;
        sts = (MPI_Status*) GIOI_Malloc(sizeof(MPI_Status) * nreqs);
        MPI_Waitall(nreqs, reqs, sts);
        GIOI_Free(sts);
#endif
    }
    GIOI_Free(reqs);
#else
    MPI_Alltoall(count_my_req_per_proc, 1, MPI_OFFSET,
                 count_others_req_per_proc, 1, MPI_OFFSET, fh->comm);
#endif

    /* calculate total number of offset-length pairs to be handled by this
     * aggregator, only aggregators will have non-zero number of pairs.
     */
    npairs = 0;
    for (i=0; i<nprocs; i++) {
        npairs += count_others_req_per_proc[i];
        others_req[i].num = count_others_req_per_proc[i];
        others_req[i].cur = 0;
    }
    GIOI_Free(count_my_req_per_proc);

    /* The best communication approach for aggregators to collect offset-length
     * pairs from the non-aggregators is to allocate a single contiguous memory
     * space for my_req[] to store all its pairs of offsets and lens. The same
     * for others_req[].
     */
    pair_sz = sizeof(MPI_Offset) * 2;
    alloc_sz = pair_sz + sizeof(MPI_Offset);
    others_req[0].off = (MPI_Offset*) GIOI_Malloc(alloc_sz * npairs);
    others_req[0].len = others_req[0].off + others_req[0].num;
    others_req[0].ptr = (MPI_Offset*) (others_req[0].off + npairs * 2);
    for (i=1; i<nprocs; i++) {
        others_req[i].off = others_req[i-1].off + others_req[i-1].num * 2;
        others_req[i].len = others_req[i].off + others_req[i].num;
        others_req[i].ptr = others_req[i-1].ptr + others_req[i-1].num;
    }

    /* now send the calculated offsets and lengths to respective processes */

#ifdef CONSIDER_ALLTOALLV
    /* On Perlmutter at NERSC, when the number of processes per compute node is
     * large, using MPI_Alltoallv() instead of MPI_Isend/Irecv may avoid
     * possible hanging.  When hanging occurs, the error messages are
     * 1. RXC (0x11291:0) PtlTE 397:[Fatal] OVERFLOW buffer list exhausted
     * 2. MPICH WARNING: OFI is failing to make progress on posting a receive.
     *    MPICH suspects a hang due to completion queue exhaustion. Setting
     *    environment variable FI_CXI_DEFAULT_CQ_SIZE to a higher number might
     *    circumvent this scenario. OFI retry continuing...
     *
     * Below use a threshold of 48, number of processes per compute node.
     */
    do_alltoallv = (fh->num_NUMAs > 0) ? (nprocs / fh->num_NUMAs > 48) : 0;
#else
    do_alltoallv=0;
#endif

    if (do_alltoallv) {
        MPI_Offset *r_off_buf=NULL, *s_off_buf=NULL;
#ifdef HAVE_MPI_LARGE_COUNT
        MPI_Count *sendCounts, *recvCounts;
        MPI_Aint  *sdispls, *rdispls;
        sendCounts = (MPI_Count*) GIOI_Calloc(nprocs * 2, sizeof(MPI_Count));
        recvCounts = sendCounts + nprocs;
        sdispls    = (MPI_Aint*) GIOI_Calloc(nprocs * 2, sizeof(MPI_Aint));
        rdispls    = sdispls + nprocs;
#else
        int *sendCounts, *recvCounts, *sdispls, *rdispls;
        sendCounts = (int*) GIOI_Calloc(nprocs * 2, sizeof(int));
        recvCounts = sendCounts + nprocs;
        sdispls    = (int*) GIOI_Calloc(nprocs * 2, sizeof(int));
        rdispls    = sdispls + nprocs;
#endif

        /* prepare receive side */
        r_off_buf = others_req[0].off;
        for (i=0; i<nprocs; i++) {
#ifdef HAVE_MPI_LARGE_COUNT
            /* Note others_req[*].off are allocated in a single malloc(). */
            recvCounts[i] = others_req[i].num * pair_sz;
            rdispls[i] = (char*)others_req[i].off - (char*)r_off_buf;
#else
            MPI_Offset disp = (char*)others_req[i].off - (char*)r_off_buf;

            if (others_req[i].num * pair_sz > INT_MAX || disp > INT_MAX)
                return GIO_EINTOVERFLOW;

            /* Note others_req[*].off are allocated in a single malloc(). */
            recvCounts[i] = (int)(others_req[i].num * pair_sz);
            rdispls[i] = (int)disp;
#endif
        }

        /* prepare send side */
        s_off_buf = my_req[0].off;
        for (i=0; i<fh->hints->cb_nodes; i++) {
            int dest = fh->hints->aggr_ranks[i];
#ifdef HAVE_MPI_LARGE_COUNT
            /* Note all my_req[*].off are allocated in a single malloc(). */
            sendCounts[dest] = my_req[i].num * pair_sz;
            sdispls[dest] = (char*)my_req[i].off - (char*)s_off_buf;
#else
            MPI_Offset disp = (char*)my_req[i].off - (char*)s_off_buf;

            if (my_req[i].num * pair_sz > INT_MAX || disp > INT_MAX)
                return GIO_EINTOVERFLOW;

            /* Note all my_req[*].off are allocated in a single malloc(). */
            sendCounts[dest] = (int)(my_req[i].num * pair_sz);
            sdispls[dest] = (int)disp;
#endif
        }

#ifdef HAVE_MPI_LARGE_COUNT
        MPI_Alltoallv_c(s_off_buf, sendCounts, sdispls, MPI_BYTE,
                        r_off_buf, recvCounts, rdispls, MPI_BYTE, fh->comm);
#else
        MPI_Alltoallv(s_off_buf, sendCounts, sdispls, MPI_BYTE,
                      r_off_buf, recvCounts, rdispls, MPI_BYTE, fh->comm);
#endif

        GIOI_Free(sdispls);
        GIOI_Free(sendCounts);
    }
    else { /* instead of using alltoall, use MPI_Issend and MPI_Irecv */
        reqs = (MPI_Request *)
            GIOI_Malloc(sizeof(MPI_Request) * (nprocs + fh->hints->cb_nodes));

        nreqs = 0;
        for (i = 0; i < nprocs; i++) {
            if (others_req[i].num == 0) /* nothing to receive from rank i */
                continue;

            /* Note the memory address of others_req[i].len is right after
             * others_req[i].off. This allows the following recv call to
             * receive both offsets and lens in a single call.
             */
            if (i == myrank) {
                /* send to self uses memcpy(), here
                 * others_req[i].num == my_req[fh->my_cb_nodes_index].num
                 */
                memcpy(others_req[i].off,
                       my_req[fh->my_cb_nodes_index].off,
                       my_req[fh->my_cb_nodes_index].num * pair_sz);
            }
            else {
                MPI_Count r_amnt = others_req[i].num * pair_sz;

                if (r_amnt > INT_MAX) {
#ifdef HAVE_MPI_LARGE_COUNT
                    err = MPI_Irecv_c(others_req[i].off, r_amnt, MPI_BYTE,
                                      i, 0, fh->comm, &reqs[nreqs++]);
                    err = GIOI_error_mpi(err, "MPI_Irecv_c");
#else
                    err = GIO_EINTOVERFLOW;
#endif
                }
                else {
                    int nelems = (int)r_amnt;
                    err = MPI_Irecv(others_req[i].off, nelems, MPI_BYTE,
                                    i, 0, fh->comm, &reqs[nreqs++]);
                    err = GIOI_error_mpi(err, "MPI_Irecv");
                }
                if (err != GIO_NOERR) return err;
            }
        }

#if GIO_DEBUG_MODE == 1
/* WRF hangs below when calling MPI_Waitall(), at running 16 nodes, 128 ranks
 * per node on Perlmutter, when these 3 env variables are set:
 *    FI_UNIVERSE_SIZE        = 2048
 *    FI_CXI_DEFAULT_CQ_SIZE  = 524288
 *    FI_CXI_RX_MATCH_MODE    = software
 *
 * Using MPI_Alltoallv seems to be able to avoid such hanging problem. (above)
 */
// MPI_Barrier(fh->comm); /* This barrier prevents the MPI_Waitall below from hanging !!! */
#endif

        for (i=0; i<fh->hints->cb_nodes; i++) {
            if (my_req[i].num == 0 || i == fh->my_cb_nodes_index)
                continue; /* nothing to send or send to self */

            /* Note the memory address of my_req[i].len is right after
             * my_req[i].off. This allows the following Issend call to
             * send both offsets and lens in a single call.
             */
            MPI_Count s_amnt = my_req[i].num * pair_sz;

            if (s_amnt > INT_MAX) {
#ifdef HAVE_MPI_LARGE_COUNT
                err = MPI_Issend_c(my_req[i].off, s_amnt, MPI_BYTE,
                                   fh->hints->aggr_ranks[i], 0, fh->comm,
                                   &reqs[nreqs++]);
                err = GIOI_error_mpi(err, "MPI_Issend_c");
#else
                err = GIO_EINTOVERFLOW;
#endif
            }
            else {
                int nelems = (int)s_amnt;
                err = MPI_Issend(my_req[i].off, nelems, MPI_BYTE,
                                 fh->hints->aggr_ranks[i], 0, fh->comm,
                                 &reqs[nreqs++]);
                err = GIOI_error_mpi(err, "MPI_Issend");
            }
            if (err != GIO_NOERR) return err;
        }

        if (nreqs) {
#ifdef HAVE_MPI_STATUSES_IGNORE
            MPI_Waitall(nreqs, reqs, MPI_STATUSES_IGNORE);
#else
            MPI_Status *sts;
            sts = (MPI_Status*) GIOI_Malloc(sizeof(MPI_Status) * nreqs);
            MPI_Waitall(nreqs, reqs, sts);
            GIOI_Free(sts);
#endif
        }
        GIOI_Free(reqs);
    }

    return GIO_NOERR;
}

/*----< comm_phase_alltoallw() >---------------------------------------------*/
static int
comm_phase_alltoallw(GIO_File      fh,
                     off_len_list *send_list,  /* [cb_nodes] */
                     off_len_list *recv_list)  /* [nprocs] */
{
    /* This subroutine performs the sam communication tasks as the below
     * commit_comm_phase(), but using MPI_Alltoallw() instead of MPI_Issend and
     * MPI_Irecv.
     *
     * It creates a datatype combining all displacement-length
     * pairs in each element of send_list[]. The datatype is used when calling
     * MPI_Issend to send write data to the I/O aggregators. Similarly, it
     * creates a datatype combining all displacement-length pairs in each
     * element of recv_list[] and uses it when calling MPI_Irecv or MPI_Recv
     * to receive write data from all processes.
     */
    int i, err, nprocs, rank;
    MPI_Datatype *sendTypes, *recvTypes;

    MPI_Comm_size(fh->comm, &nprocs);
    MPI_Comm_rank(fh->comm, &rank);

    /* calculate send/recv derived types metadata */
    int *sendCounts, *recvCounts, *sdispls, *rdispls;
    sendCounts = (int*) GIOI_Calloc(nprocs * 2, sizeof(int));
    recvCounts = sendCounts + nprocs;
    sdispls    = (int*) GIOI_Calloc(nprocs * 2, sizeof(int));
    rdispls    = sdispls + nprocs;

    /* allocate send/recv derived type arrays */
    sendTypes = (MPI_Datatype*) GIOI_Malloc(sizeof(MPI_Datatype) * nprocs * 2);
    recvTypes = sendTypes + nprocs;

    for (i=0; i<nprocs; i++)
        sendTypes[i] = recvTypes[i] = MPI_BYTE;

    /* prepare receive side: construct recv derived data types */
    if (fh->is_agg && recv_list != NULL) {
        for (i=0; i<nprocs; i++) {
            /* check if nothing to receive or if self */
            if (recv_list[i].num == 0 || i == rank) continue;

            recvCounts[i] = 1;

            /* combine reqs using new datatype */
            err = GIOI_type_create_hindexed(recv_list[i].num, recv_list[i].off,
                                            recv_list[i].len, &recvTypes[i]);
            if (err != GIO_NOERR) return err;
        }
    }

    /* prepare send side: construct send derived data types */
    for (i=0; i<fh->hints->cb_nodes; i++) {
        /* check if nothing to send or if self */
        if (send_list[i].num == 0 || i == fh->my_cb_nodes_index) continue;

        int dest = fh->hints->aggr_ranks[i];
        sendCounts[dest] = 1;

        /* combine reqs using new datatype */
        err = GIOI_type_create_hindexed(send_list[i].num, send_list[i].off,
                                        send_list[i].len, &sendTypes[dest]);
        if (err != GIO_NOERR) return err;
    }

    /* No need to call MPI_Alltoallw_c(), as we are using MPI derived datatypes
     * for send and receive, and thus the count arguments are alwasys 1.
     */
    MPI_Alltoallw(MPI_BOTTOM, sendCounts, sdispls, sendTypes,
                  MPI_BOTTOM, recvCounts, rdispls, recvTypes, fh->comm);

    for (i=0; i<nprocs; i++) {
        if (sendTypes[i] != MPI_BYTE)
            MPI_Type_free(&sendTypes[i]);
        if (recvTypes[i] != MPI_BYTE)
            MPI_Type_free(&recvTypes[i]);
    }
    GIOI_Free(sdispls);
    GIOI_Free(sendCounts);
    GIOI_Free(sendTypes);

    /* clear send_list and recv_list for future reuse */
    for (i = 0; i < fh->hints->cb_nodes; i++)
        send_list[i].num = 0;

    if (recv_list != NULL)
        for (i = 0; i < nprocs; i++)
            recv_list[i].num = 0;

    return GIO_NOERR;
}

/*----< commit_comm_phase() >------------------------------------------------*/
static int
commit_comm_phase(GIO_File      fh,
                  off_len_list *send_list,  /* [cb_nodes] */
                  off_len_list *recv_list)  /* [nprocs] */
{
    /* This subroutine creates a datatype combining all displacement-length
     * pairs in each element of send_list[]. The datatype is used when calling
     * MPI_Issend to send write data to the I/O aggregators. Similarly, it
     * creates a datatype combining all displacement-length pairs in each
     * element of recv_list[] and uses it when calling MPI_Irecv or MPI_Recv
     * to receive write data from all processes.
     */
    int i, err, nprocs, rank, nreqs;
    MPI_Request *reqs;
    MPI_Datatype sendType, recvType;
#if GIO_PROFILING_MODE == 1
    int j;
    double dtype_time=MPI_Wtime();
#endif

    if (use_alltoallw)
        return comm_phase_alltoallw(fh, send_list, recv_list);

    MPI_Comm_size(fh->comm, &nprocs);
    MPI_Comm_rank(fh->comm, &rank);

    nreqs = fh->hints->cb_nodes;
    nreqs += (fh->is_agg) ? nprocs : 0;
    reqs = (MPI_Request *) GIOI_Malloc(sizeof(MPI_Request) * nreqs);
    nreqs = 0;

    /* receiving part */
#if GIO_PROFILING_MODE == 1
    /* recv buffer type profiling */
    // int nrecvs=0;
    MPI_Offset max_r_amnt=0, max_r_count=0;
#endif

    if (fh->is_agg && recv_list != NULL) {
        for (i = 0; i < nprocs; i++) {
            /* check if nothing to receive or if self */
            if (recv_list[i].num == 0 || i == rank) continue;

#if GIO_PROFILING_MODE == 1
            MPI_Offset r_amnt=0;
            for (j=0; j<recv_list[i].num; j++)
                r_amnt += recv_list[i].len[j];
            max_r_amnt = MAX(max_r_amnt, r_amnt);
            max_r_count = MAX(max_r_count, recv_list[i].num);
            // nrecvs++;
#endif

            /* combine reqs using new datatype */
            err = GIOI_type_create_hindexed(recv_list[i].num, recv_list[i].off,
                                            recv_list[i].len, &recvType);
            if (err != GIO_NOERR) return err;

            if (fh->atomicity) { /* Blocking Recv */
                MPI_Status status;
                MPI_Recv(MPI_BOTTOM, 1, recvType, i, 0, fh->comm, &status);
            }
            else
                MPI_Irecv(MPI_BOTTOM, 1, recvType, i, 0, fh->comm,
                          &reqs[nreqs++]);
            MPI_Type_free(&recvType);
        }
    }

    /* send reqs */
#if GIO_PROFILING_MODE == 1
    /* send buffer type profiling */
    // int nsends=0;
    MPI_Offset max_s_amnt=0, max_s_count=0;
#endif

    for (i = 0; i < fh->hints->cb_nodes; i++) {
        /* check if nothing to send or if self */
        if (send_list[i].num == 0 || i == fh->my_cb_nodes_index) continue;

#if GIO_PROFILING_MODE == 1
        MPI_Offset s_amnt=0;
        for (j=0; j<send_list[i].num; j++)
            s_amnt += send_list[i].len[j];
        max_s_amnt = MAX(max_s_amnt, s_amnt);
        max_s_count = MAX(max_s_count, send_list[i].num);
        // nsends++;
#endif

        /* combine reqs using new datatype */
        err = GIOI_type_create_hindexed(send_list[i].num, send_list[i].off,
                                        send_list[i].len, &sendType);
        if (err != GIO_NOERR) return err;

        MPI_Issend(MPI_BOTTOM, 1, sendType, fh->hints->aggr_ranks[i], 0,
                   fh->comm, &reqs[nreqs++]);
        MPI_Type_free(&sendType);
    }

#if GIO_PROFILING_MODE == 1
    gio_wr_time[4] += MPI_Wtime() - dtype_time;

/*
    gio_wr_count[2] = MAX(gio_wr_count[2], nsends);
    gio_wr_count[3] = MAX(gio_wr_count[3], nrecvs);
    gio_wr_count[4] = MAX(gio_wr_count[4], max_r_amnt);
    gio_wr_count[5] = MAX(gio_wr_count[5], max_s_amnt);
    gio_wr_count[6] = MAX(gio_wr_count[6], max_r_count);
    gio_wr_count[7] = MAX(gio_wr_count[7], max_s_count);
*/
#endif

    if (nreqs > 0) {
#ifdef HAVE_MPI_STATUSES_IGNORE
        MPI_Waitall(nreqs, reqs, MPI_STATUSES_IGNORE);
#else
        MPI_Status *sts;
        sts = (MPI_Status*) GIOI_Malloc(nreqs * sizeof(MPI_Status));
        MPI_Waitall(nreqs, reqs, sts);
        GIOI_Free(sts);
#endif
    }

    GIOI_Free(reqs);

    /* clear send_list and recv_list for future reuse */
    for (i = 0; i < fh->hints->cb_nodes; i++)
        send_list[i].num = 0;

    if (recv_list != NULL)
        for (i = 0; i < nprocs; i++)
            recv_list[i].num = 0;

    return GIO_NOERR;
}

/*----< LUSTRE_Exch_and_write() >--------------------------------------------*/
/* Each process sends all its write requests to I/O aggregators based on the
 * file domain assignment to the aggregators. In this implementation, a file is
 * first divided into stripes which are assigned to the aggregators in a
 * round-robin fashion. The "exchange" of write data from non-aggregators to
 * aggregators is carried out in 'ntimes' rounds. Each round covers an
 * aggregate file region of size equal to the file stripe size times the number
 * of I/O aggregators. The file writes are carried out in every 'nbufs'
 * iterations, where 'nbufs' == cb_buffer_size / file stripe size. This approach
 * is different from ROMIO's implementation as in MPICH 4.2.3.
 *
 * Other implementations developers are referring to the paper: Wei-keng Liao,
 * and Alok Choudhary. "Dynamically Adapting File Domain Partitioning Methods
 * for Collective I/O Based on Underlying Parallel File System Locking
 * Protocols", in The Supercomputing Conference, 2008.
 */
static MPI_Offset
LUSTRE_Exch_and_write(GIO_File      fh,
                      const void   *buf,
                      GIOI_Access  *others_req,
                      GIOI_Access  *my_req,
                      MPI_Offset    min_st_off,
                      MPI_Offset    max_end_off,
                      MPI_Offset  **buf_idx)
{
    char **write_buf = NULL, **recv_buf = NULL, **send_buf = NULL;
    size_t alloc_sz;
    int err, nprocs, myrank, nbufs, ibuf, batch_idx=0, cb_nodes, striping_unit;
    MPI_Offset i, j, m, ntimes, w_len=0, total_w_len=0;
    MPI_Offset end_loc, req_off, iter_end_off, step_size;
    MPI_Offset **recv_size=NULL, **recv_count=NULL, **recv_start_pos=NULL;
    MPI_Offset *send_size, *off_list, *this_buf_idx=NULL;
    off_len_list *srt_off_len=NULL, *send_list=NULL, *recv_list=NULL;

    MPI_Comm_size(fh->comm, &nprocs);
    MPI_Comm_rank(fh->comm, &myrank);

    cb_nodes = fh->hints->cb_nodes;
    striping_unit = fh->hints->striping_unit;

    /* The aggregate access region (across all processes) of this collective
     * write starts from min_st_off and ends at max_end_off. The collective
     * write is carried out in 'ntimes' rounds of two-phase I/O. Each round
     * covers an aggregate file region of size 'step_size' written only by
     * cb_nodes number of I/O aggregators. Note non-aggregators must also
     * participate all ntimes rounds to send their requests to I/O aggregators.
     *
     * step_size = the number of I/O aggregators x striping_unit
     *
     * Note the number of write phases = ntimes / nbufs, as writes (and
     * communication) are accumulated for nbufs rounds before flushed.
     */
    step_size = (MPI_Offset)cb_nodes * striping_unit;

    /* align min_st_off downward to the nearest file stripe boundary */
    min_st_off -= min_st_off % (MPI_Offset) striping_unit;

    /* ntimes is the number of rounds of two-phase I/O */
    ntimes = (max_end_off - min_st_off + 1) / step_size;
    if ((max_end_off - min_st_off + 1) % step_size)
        ntimes++;

#if GIO_PROFILING_MODE == 1
    gio_wr_count[0] = MAX(gio_wr_count[0], ntimes);
#endif

    /* collective buffer is divided into 'nbufs' sub-buffers. Each sub-buffer
     * is of size equal to Lustre stripe size. Write data of non-aggregators
     * are sent to aggregators and stored in aggregators' sub-buffers, one for
     * each round. All nbufs sub-buffers are altogether flushed to file every
     * nbufs rounds.
     *
     * fh->hints->cb_buffer_size, collective buffer size, for Lustre must be at
     * least striping_unit. This requirement has been checked at the file
     * open/create time when fh->io_buf is allocated.
     *
     * Note cb_buffer_size and striping_unit may also be adjusted earlier in
     * GIOI_Lustre_write_coll().
     */
    nbufs = fh->hints->cb_buffer_size / striping_unit;
    assert(nbufs > 0); /* must at least 1 */

    /* in case number of rounds is less than nbufs */
    nbufs = (ntimes < nbufs) ? (int)ntimes : nbufs;

    /* off_list[m] is the starting file offset of this aggregator's write
     *     region in iteration m (file domain of iteration m). This offset
     *     may not be aligned with file stripe boundaries.
     * end_loc is the ending file offset of this aggregator's file domain.
     */
    off_list = (MPI_Offset*) GIOI_Malloc(sizeof(MPI_Offset) * ntimes);
    end_loc = -1;
    for (m = 0; m < ntimes; m++)
        off_list[m] = max_end_off;
    for (i = 0; i < nprocs; i++) {
        for (j = 0; j < others_req[i].num; j++) {
            req_off = others_req[i].off[j];
            m = (int) ((req_off - min_st_off) / step_size);
            off_list[m] = MIN(off_list[m], req_off);
            end_loc = MAX(end_loc, (others_req[i].off[j] + others_req[i].len[j] - 1));
        }
    }

    /* Allocate displacement-length pair arrays, describing the send buffer.
     * send_list[i].num: number displacement-length pairs.
     * send_list[i].len: length in bytes.
     * send_list[i].off: displacement (send buffer address).
     */
    send_list = (off_len_list*) GIOI_Malloc(sizeof(off_len_list) * cb_nodes);
    for (i = 0; i < cb_nodes; i++) {
        send_list[i].num = 0;
        alloc_sz = sizeof(MPI_Offset) * 2;
        send_list[i].off = (MPI_Offset*) GIOI_Malloc(alloc_sz * nbufs);
        send_list[i].len = send_list[i].off + nbufs;
    }

    /* end_loc >= 0 indicates this process has something to write to the file.
     * Only I/O aggregators can have end_loc > 0. write_buf is the collective
     * buffer and only matter for I/O aggregators. recv_buf is the buffer used
     * only by aggregators to receive requests from non-aggregators. Its size
     * may be larger then the file stripe size, in case when writes from
     * non-aggregators overlap. In this case, it will be realloc-ed in
     * LUSTRE_W_Exchange_data(). The received data is later copied over to
     * write_buf, whose contents will be written to file.
     */
    if (end_loc >= 0 && nbufs > 0) {
        /* Allocate displacement-length pair arrays, describing the recv buffer.
         * recv_list[i].num: number displacement-length pairs.
         * recv_list[i].len: length in bytes.
         * recv_list[i].off: displacement (recv buffer address).
         */
        assert(fh->is_agg);

        recv_list = (off_len_list*) GIOI_Malloc(sizeof(off_len_list) * nprocs);
        for (i = 0; i < nprocs; i++) {
            recv_list[i].num = 0;
            alloc_sz = sizeof(MPI_Offset) * 2;
            recv_list[i].off = (MPI_Offset*) GIOI_Malloc(alloc_sz * nbufs);
            recv_list[i].len = recv_list[i].off + nbufs;
        }

        /* collective buffer was allocated at file open/create. For Lustre, its
         * size must be at least striping_unit, which has been checked at the
         * time fh->io_buf is allocated.
         */
        assert(fh->io_buf != NULL);

        /* divide collective buffer into nbufs sub-buffers */
        write_buf = (char **) GIOI_Malloc(sizeof(char*) * nbufs);
        write_buf[0] = fh->io_buf;

        /* Similarly, receive buffer consists of nbufs sub-buffers */
        recv_buf = (char **) GIOI_Malloc(sizeof(char*) * nbufs);
        recv_buf[0] = (char *) GIOI_Malloc(striping_unit);

        /* recv_count[j][i] is the number of off-len pairs to be received from
         * each proc i in round j
         */
        recv_count    = (MPI_Offset**) GIOI_Malloc(sizeof(MPI_Offset*) * 3*nbufs);
        recv_count[0] = (MPI_Offset*)  GIOI_Malloc(sizeof(MPI_Offset) * 3*nbufs * nprocs);

        /* recv_size[j][i] is the receive size from proc i in round j */
        recv_size = recv_count + nbufs;
        recv_size[0] = recv_count[0] + nbufs * nprocs;

        /* recv_start_pos[j][i] is the starting index of offset-length arrays
         * pointed by others_req[i].cur for remote rank i in round j
         */
        recv_start_pos = recv_size + nbufs;
        recv_start_pos[0] = recv_size[0] + nbufs * nprocs;

        for (j = 1; j < nbufs; j++) {
            write_buf[j] = write_buf[j-1] + striping_unit;
            /* recv_buf[j] may be realloc in LUSTRE_W_Exchange_data() */
            recv_buf[j]       = (char*) GIOI_Malloc(striping_unit);
            recv_count[j]     = recv_count[j-1]     + nprocs;
            recv_size[j]      = recv_size[j-1]      + nprocs;
            recv_start_pos[j] = recv_start_pos[j-1] + nprocs;
        }

        /* srt_off_len consists of file offset-length pairs sorted in a
         * monotonically non-decreasing order (required by MPI-IO standard)
         * which is used when writing to the file
         */
        srt_off_len = (off_len_list*) GIOI_Malloc(sizeof(off_len_list) * nbufs);
    }

    /* send_buf[] will be allocated in LUSTRE_W_Exchange_data(), when the use
     * buffer is not contiguous.
     */
    send_buf = (char **) GIOI_Malloc(sizeof(char*) * nbufs);

    /* this_buf_idx contains indices to the user write buffer for sending this
     * rank's write data to aggregators, one for each aggregator. It is used
     * only when user buffer is contiguous.
     */
    if (fh->bview.npairs <= 1)
        this_buf_idx = (MPI_Offset *) GIOI_Malloc(sizeof(MPI_Offset) * cb_nodes);

    /* array of data sizes to be sent to each aggregator in a 2-phase round */
    send_size = (MPI_Offset *) GIOI_Calloc(cb_nodes, sizeof(MPI_Offset));

    /* min_st_off is the beginning file offsets of the aggregate access region
     *     of this collective write, and it has been downward aligned to the
     *     nearest file stripe boundary
     * iter_end_off is the ending file offset of aggregate write region of
     *     iteration m, upward aligned to the file stripe boundary.
     */
    iter_end_off = min_st_off + step_size;

    ibuf = 0;
    for (m = 0; m < ntimes; m++) {
        MPI_Offset range_size, range_off;

        /* Note that MPI standard (MPI 3.1 Chapter 13.1.1 and MPI 4.0 Chapter
         * 14.1.1) requires that the typemap displacements of etype and
         * filetype are non-negative and monotonically non-decreasing. This
         * simplifies implementation a bit compared to reads.
         */

        /* Calculate what should be communicated.
         *
         * First, calculate the amount to be sent to each aggregator i, at this
         * round m, by going through all offset-length pairs in my_req[i].
         *
         * iter_end_off - ending file offset of aggregate write region of this
         *                round, and upward aligned to the file stripe
         *                boundary. Note the aggregate write region of this
         *                round starts from (iter_end_off-step_size) to
         *                iter_end_off, aligned with file stripe boundaries.
         * send_size[i] - total size in bytes of this process's write data
         *                fall into aggregator i's FD in this round.
         * recv_size[m][i] - size in bytes of data to be received by this
         *                aggregator from process i in round m.
         * recv_count[m][i] - number of noncontiguous offset-length pairs from
         *                process i fall into this aggregator's write region
         *                in round m.
         */
        for (i = 0; i < cb_nodes; i++) {
            /* reset communication metadata to all 0s for this round */
            send_size[i] = 0;

            if (my_req[i].num == 0) continue;
            /* my_req[i].num is the number of this rank's offset-length pairs
             * to be sent to aggregator i
             */

            if (my_req[i].cur == my_req[i].num)
                continue; /* done with aggregator i */

            if (fh->bview.npairs <= 1)
                /* buf_idx is used only when user buffer is contiguous.
                 * this_buf_idx[i] points to the starting offset of user
                 * buffer, buf, for amount of send_size[i] to be sent to
                 * aggregator i at this round.
                 */
                this_buf_idx[i] = buf_idx[i][my_req[i].cur];

            /* calculate the send amount from this rank to aggregator i */
            for (j = my_req[i].cur; j < my_req[i].num; j++) {
                if (my_req[i].off[j] < iter_end_off)
                    send_size[i] += my_req[i].len[j];
                else
                    break;
            }

            /* update my_req[i].cur to point to the jth offset-length
             * pair of my_req[i], which will be used as the first pair in the
             * next round of iteration.
             */
            my_req[i].cur = j;
        }

        /* range_off is the starting file offset of this aggregator's write
         *     region at this round (may not be aligned to stripe boundary).
         * range_size is the size (in bytes) of this aggregator's write region
         *     for this round (whose size is always <= striping_unit).
         */
        range_off = off_list[m];
        range_size = MIN(striping_unit - range_off % striping_unit,
                         end_loc - range_off + 1);

        /* Calculate the amount to be received from each process i at this
         * round, by going through all offset-length pairs of others_req[i].
         */
        if (recv_count != NULL) {
            for (i=0; i<nprocs; i++) {
                /* reset communication metadata to all 0s for this round */
                recv_count[ibuf][i] = recv_size[ibuf][i] = 0;
                recv_start_pos[ibuf][i] = 0;

                if (others_req[i].num == 0) continue;

                recv_start_pos[ibuf][i] = others_req[i].cur;
                for (j = others_req[i].cur; j < others_req[i].num; j++) {
                    if (others_req[i].off[j] < iter_end_off) {
                        recv_count[ibuf][i]++;
                        others_req[i].ptr[j] = others_req[i].off[j] - range_off;
                        recv_size[ibuf][i] += others_req[i].len[j];
                    } else {
                        break;
                    }
                }
                /* update others_req[i].cur to point to the jth offset-length
                 * pair of others_req[i], which will be used as the first pair
                 * in the next round of iteration.
                 */
                others_req[i].cur = j;
            }
        }
        iter_end_off += step_size;

        /* exchange phase - each process sends it's write data to I/O
         * aggregators and aggregators receive from non-aggregators.
         * Communication are MPI_Issend and MPI_Irecv only. There is no
         * collective communication. Only aggregators have non-NULL write_buf
         * and recv_buf. All processes have non-NULL send_buf.
         */
        char *wbuf = (write_buf == NULL) ? NULL : write_buf[ibuf];

        /* Exch_data_recv() and Exch_data_send() below perform one round of
         * communication phase and there are ntimes rounds.
         */
        if (recv_list != NULL) { /* this aggregator has something to received */
            char *rbuf = (recv_buf  == NULL) ? NULL :  recv_buf[ibuf];

            err = Exch_data_recv(fh,
                       buf,                 /* IN: user buffer */
                       wbuf,                /* OUT: write buffer */
                       &rbuf,               /* OUT: receive buffer */
                       recv_size[ibuf],     /* IN: changed each round */
                       range_off,           /* IN: changed each round */
                       range_size,          /* IN: changed each round */
                       recv_count[ibuf],    /* IN: changed each round */
                       recv_start_pos[ibuf],/* IN: changed each round */
                       others_req,          /* IN: changed each round */
                       this_buf_idx,        /* IN: changed each round */
                       &srt_off_len[ibuf],  /* OUT: write off-len pairs */
                       recv_list);          /* OUT: recv disp-len pairs */
            if (err != GIO_NOERR)
                goto over;

            /* rbuf might be realloc-ed */
            if (recv_buf != NULL) recv_buf[ibuf] = rbuf;
        }

        /* sender part */
        MPI_Offset self_count, self_start_pos;
        if (recv_count == NULL) {
            self_count = 0;
            self_start_pos = 0;
        }
        else {
            self_count     = recv_count[ibuf][myrank];
            self_start_pos = recv_start_pos[ibuf][myrank];
        }
        send_buf[ibuf] = NULL;

        err = Exch_data_send(fh,
                             buf,             /* IN: user buffer */
                             wbuf,            /* OUT: write buffer */
                             &send_buf[ibuf], /* OUT: send buffer */
                             send_size,       /* IN: changed each round */
                             self_count,
                             self_start_pos,
                             others_req,      /* IN: changed each round */
                             this_buf_idx,    /* IN: changed each round */
                             send_list);      /* OUT: send disp-len pairs */
        if (err != GIO_NOERR) goto over;

        if (m % nbufs < nbufs - 1 && m < ntimes - 1) {
            /* continue to the next round */
            ibuf++;
        }
        else {
            /* commit communication and write this batch of numBufs to file */
            int numBufs = ibuf + 1;

            /* reset ibuf to the first element of nbufs */
            ibuf = 0;

#if GIO_PROFILING_MODE == 1
            double curT = MPI_Wtime();
#endif
            /* communication phase */
            commit_comm_phase(fh, send_list, recv_list);
#if GIO_PROFILING_MODE == 1
            if (fh->is_agg) gio_wr_time[3] += MPI_Wtime() - curT;
#endif

            /* free send_buf allocated in LUSTRE_W_Exchange_data() */
            for (j = 0; j < numBufs; j++) {
                if (send_buf[j] != NULL) {
                    GIOI_Free(send_buf[j]);
                    send_buf[j] = NULL;
                }
            }
            if (!fh->is_agg) /* non-aggregators are done for this batch */
                continue;

            if (recv_list == NULL) /*  this aggregator has nothing to write */
                continue;

            /* this aggregator unpacks the data in recv_buf[] into write_buf */
            if (end_loc >= 0) {
                for (j = 0; j < numBufs; j++) {
                    char *buf_ptr = recv_buf[j];
                    for (i = 0; i < nprocs; i++) {
                        if (recv_count[j][i] > 1 && i != myrank) {
                            /* When recv_count[j][i] == 1, this case has
                             * been taken care of earlier by receiving the
                             * message directly into write_buf.
                             */
                            MEMCPY_UNPACK(i, buf_ptr, recv_start_pos[j][i],
                                          recv_count[j][i], write_buf[j]);
                            buf_ptr += recv_size[j][i];
                        }
                    }
                }
            }

            /* this aggregator writes to numBufs number of stripes */
            for (j=0; j<numBufs; j++) {

                /* if there is no data to write in round (batch_idx + j) */
                if (srt_off_len[j].num == 0)
                    continue;

                /* range_off  starting file offset of this aggregator's write
                 *            region for this round (may not be aligned to
                 *            stripe boundary)
                 * range_size size (in bytes) of this rank's write region for
                 *            this round, <= striping_unit
                 */
                range_off = off_list[batch_idx + j];
                range_size = MIN(striping_unit - range_off % striping_unit,
                                 end_loc - range_off + 1);

                /* When srt_off_len[j].num == 1, either there is no hole in the
                 * write buffer or the file domain has been read-modify-written
                 * with the received write data. When srt_off_len[j].num > 1,
                 * data sieving is not performed and holes have been found. In
                 * this case, srt_off_len[] is the list of sorted offset-length
                 * pairs describing noncontiguous writes. Now call writes for
                 * each offset-length pair. Note the offset-length pairs
                 * (represented by srt_off_len[j].off, srt_off_len[j].len, and
                 * srt_off_len[j].num) have been coalesced in
                 * LUSTRE_W_Exchange_data().
                 */
                for (i = 0; i < srt_off_len[j].num; i++) {
                    /* all write requests in this round should fall into file
                     * range of [range_off, range_off+range_size). This below
                     * assertion should never fail.
                     */
                    assert(srt_off_len[j].off[i] < range_off + range_size &&
                           srt_off_len[j].off[i] >= range_off);

                    w_len = GIOI_UFS_write_contig(fh,
                                     write_buf[j] + (srt_off_len[j].off[i] - range_off),
                                     srt_off_len[j].len[i],
                                     srt_off_len[j].off[i], 1);
                    if (w_len < 0) goto over;
                    total_w_len += w_len;
                }
                if (srt_off_len[j].num > 0) {
                    GIOI_Free(srt_off_len[j].off);
                    srt_off_len[j].num = 0;
                }
            }
            batch_idx += numBufs; /* only matters for aggregators */
        }
    }

over:
    if (srt_off_len)
        GIOI_Free(srt_off_len);
    if (write_buf != NULL)
        GIOI_Free(write_buf);
    if (recv_buf != NULL) {
        for (j = 0; j < nbufs; j++)
            GIOI_Free(recv_buf[j]);
        GIOI_Free(recv_buf);
    }
    if (recv_count != NULL) {
        GIOI_Free(recv_count[0]);
        GIOI_Free(recv_count);
    }
    GIOI_Free(send_size);
    GIOI_Free(off_list);
    if (fh->bview.npairs <= 1)
        GIOI_Free(this_buf_idx);
    if (send_buf != NULL)
        GIOI_Free(send_buf);
    if (send_list != NULL) {
        for (i=0; i<cb_nodes; i++)
            GIOI_Free(send_list[i].off);
        GIOI_Free(send_list);
    }
    if (recv_list != NULL) {
        for (i=0; i<nprocs; i++)
            GIOI_Free(recv_list[i].off);
        GIOI_Free(recv_list);
    }

#if GIO_DEBUG_MODE == 1
    /* check any pending messages to be received */
    MPI_Status probe_st;
    int probe_flag;
    MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, fh->comm, &probe_flag, &probe_st);
    if (probe_flag) {
        fprintf(stderr,"ERROR ++++ MPI_Iprobe rank=%4d is_agg=%d: ---- cb_nodes=%d ntimes=%lld nbufs=%d\n",myrank,fh->is_agg,cb_nodes,ntimes,nbufs);
        fflush(stdout);
    }
#endif

    /* When w_len < 0, it is a GIO error code. */
    return (w_len < 0) ? w_len : total_w_len;
}

/*----< offset_compare() >---------------------------------------------------*/
/* This subroutine is used to sort st_end_all[nprocs] */
static int
offset_compare(const void *a, const void *b)
{
    if (*(MPI_Offset*)a > *(MPI_Offset*)b) return (1);
    if (*(MPI_Offset*)a < *(MPI_Offset*)b) return (-1);
    return (0);
}

/*----< GIOI_Lustre_write_coll() >------------------------------------------*/
MPI_Offset
GIOI_Lustre_write_coll(GIO_File    fh,
                       const void *buf)
{
    /* Uses a generalized version of the extended two-phase method described in
     * "An Extended Two-Phase Method for Accessing Sections of Out-of-Core
     * Arrays", Rajeev Thakur and Alok Choudhary, Scientific Programming,
     * (5)4:301--317, Winter 1996.
     * http://www.mcs.anl.gov/home/thakur/ext2ph.ps
     */

    int i, err, nprocs, myrank, do_collect=1, do_ex_wr;
    MPI_Offset st_off, end_off, min_st_off=-1, max_end_off=-1, w_len=0;

#if GIO_PROFILING_MODE == 1
    MPI_Barrier(fh->comm);
    double curT = MPI_Wtime();
#endif

    MPI_Comm_size(fh->comm, &nprocs);
    MPI_Comm_rank(fh->comm, &myrank);

    /* GIO never reuses a fileview across two or more GIO calls. */

    /* fh->fview contains a list of starting file offsets and lengths of
     * write requests made by this rank. Similarly, bview contains a list of
     * offset-length pairs describing the write buffer layout. Note GIO
     * never re-uses a fileview or buffer view.
     *
     * Note that MPI standard (MPI 3.1 Chapter 13.1.1 and MPI 4.0 Chapter
     * 14.1.1) requires that the typemap displacements of etype and filetype
     * set by the user are non-negative and monotonically non-decreasing. This
     * makes fh->fview.off[] to be monotonically non-decreasing.
     *
     * This rank's aggregate file access region is from st_off to end_off.
     * Note end_off points to the last byte-offset to be accessed. E.g., if
     * st_off=0 and end_off=99, then the aggregate file access region is of
     * size 100 bytes. If this rank has no data to write, set both st_off and
     * end_off to -1.
     */
    if (fh->fview.size == 0) /* zero-sized request */
        st_off = end_off = -1;
    else {
        /* Note fview.off[] is always relative to beginning of file */
        st_off  = fh->fview.off[0];
        end_off = fh->fview.off[fh->fview.npairs-1]
                + fh->fview.len[fh->fview.npairs-1] - 1;
    }

    fh->bview.idx = 0;
    fh->bview.rem = fh->bview.size;
    if (fh->bview.size > 0 && fh->bview.npairs > 1)
        fh->bview.rem = fh->bview.len[0];

    if (fh->hints->cb_write == GIOI_HINT_DISABLE) {
        /* collective write is explicitly disabled by user */
        do_collect = 0;
    }
    else {
        /* Calculate the aggregate access region of all ranks and check if
         * write requests are interleaved among all ranks.
         */
        int is_interleaved, large_indv_req = 1;
        MPI_Offset striping_range, *st_end_all = NULL;

        /* Gather starting and ending file offsets of write requests from all
         * ranks into st_end_all[]. Even indices of st_end_all[] are starting
         * offsets, and odd indices are ending offsets.
         */
#ifdef TRY_ALLREDUCE
        st_end_all = (MPI_Offset*) GIOI_Calloc(nprocs * 2, sizeof(MPI_Offset));
        st_end_all[myrank*2]  = st_off;
        st_end_all[myrank*2+1] = end_off;
        MPI_Allreduce(MPI_IN_PLACE, st_end_all, nprocs*2, MPI_OFFSET, MPI_MAX,
                      fh->comm);
#else
        MPI_Offset st_end[2];
        st_end[0] = st_off;
        st_end[1] = end_off;
        st_end_all = (MPI_Offset*) GIOI_Malloc(sizeof(MPI_Offset) * nprocs * 2);
        MPI_Allgather(st_end, 2, MPI_OFFSET, st_end_all, 2, MPI_OFFSET,
                      fh->comm);
#endif

        /* The loop below does the followings.
         * 1. Calculate this rank's aggregate access region.
         * 2. Check whether or not the requests are interleaved among all ranks.
         * 3. Check whether there are LARGE individual requests. Here, "large"
         *    means a write range is > (striping_factor * striping_unit). In
         *    this case, independent write will perform faster than collective.
         */
        striping_range = fh->hints->striping_unit * fh->hints->striping_factor;
        is_interleaved = 0;

        qsort(st_end_all, nprocs, sizeof(MPI_Offset)*2, offset_compare);
        for (i=0; i<2*nprocs; i+=2) { /* find the 1st non-zero sized */
            if (st_end_all[i] >= 0) {
                min_st_off  = st_end_all[i];
                max_end_off = st_end_all[i+1];
                if (st_end_all[i+1] - st_end_all[i] < striping_range)
                    large_indv_req = 0;
                break;
            }
        }
        for (i+=2; i<2*nprocs; i+=2) {
            if (st_end_all[i] == -1) /* zero-sized request */
                continue;
            if (st_end_all[i] <  st_end_all[i-1] &&
                st_end_all[i] <= st_end_all[i+1])
                is_interleaved = 1;
            min_st_off  = MIN(min_st_off,  st_end_all[i]);
            max_end_off = MAX(max_end_off, st_end_all[i+1]);
            if (st_end_all[i+1] - st_end_all[i] < striping_range)
                large_indv_req = 0;
        }
        GIOI_Free(st_end_all);

        if (min_st_off == -1 && max_end_off == -1) {
#if GIO_DEBUG_MODE == 1
            /* Warn a zero-sized collective write */
            if (myrank == 0)
                printf("\n%s at %d: zero--sized collective write!\n",
                       __func__,__LINE__);
            return 0;
#endif
        }

        if (fh->hints->cb_write == GIOI_HINT_ENABLE) {
            /* explicitly enabled by user */
            do_collect = 1;
        }
        else if (fh->hints->cb_write == GIOI_HINT_AUTO) {
            /* Check if collective write is actually necessary, only when
             * cb_write hint is set to GIOI_HINT_AUTO.
             *
             * Two typical access patterns can benefit from collective write.
             *   1) access file regions of all processes are interleaved, and
             *   2) the individual request sizes are not too big, i.e. no
             *      bigger than striping_range. Large individual requests may
             *      result in a high communication cost in order to
             *      redistribute requests from non-aggregators to I/O
             *      aggregators.
             */
            if (nprocs == 1)
                do_collect = 0;
            else if (!is_interleaved && large_indv_req &&
                     fh->hints->cb_nodes <= fh->hints->striping_factor) {
                /* do independent write, if every rank's write range >
                 * striping_range and writes are not interleaved in file
                 * space
                 */
                do_collect = 0;
            }
        }
    }

    if (!do_collect) {
        /* switch to perform independent write */

        if (fh->fview.npairs == 0) /* zero-sized request */
            return 0;

        if (!fh->is_open) {
            /* If file has not been opened (only happen to non-I/O
             * aggregators), open it now and obtain hint striping_unit.
             */
            err = GIOI_Lustre_open_on_demand(fh);
            if (err != GIO_NOERR) return err;
        }

        return GIOI_UFS_write_indep(fh, buf);
    }

    /* Now we are using collective I/O (two-phase I/O strategy) */

#ifdef ADJUST_STRIPING_UNIT
    /* adjust striping_unit when striping_factor is twice or more than the
     * number of compute nodes. Note cb_node is set to at least
     * striping_factor, if nprocs >= striping_factor. Adjustment below is to
     * let each aggregator to write to two or more consecutive OSTs, which can
     * most likely improve the performance. This will still yield an effect of
     * any one OST receiving write requests from aggregators running on only
     * one compute node.
     */
    int orig_striping_unit = fh->hints->striping_unit;

    if (fh->hints->striping_factor >= fh->num_NUMAs * 2) {
        fh->hints->striping_unit *= (fh->hints->striping_factor / fh->num_NUMAs);

        if (fh->hints->cb_buffer_size < fh->hints->striping_unit) {
            char value[MPI_MAX_INFO_VAL + 1];

            fh->hints->cb_buffer_size = fh->hints->striping_unit;
            sprintf(value, "%d", fh->hints->cb_buffer_size);
            MPI_Info_set(fh->info, "cb_buffer_size", value);
            if (fh->is_agg) {
                GIOI_Free(fh->io_buf);
                fh->io_buf = (void*) GIOI_Calloc(1, fh->hints->cb_buffer_size);
            }
        }
#if GIO_DEBUG_MODE == 1
        if (myrank == 0)
            printf("Warning: %s line %d: Change striping_unit from %d to %d\n",
                   __func__, __LINE__, orig_striping_unit, fh->hints->striping_unit);
#endif
    }
#endif

    /* my_req[cb_nodes] is an array of access info, one for each I/O aggregator
     * whose file domain has this rank's request.
     */
    GIOI_Access *my_req;

    /* others_req[nprocs] is an array of access info, one for each ranks, both
     * aggregators and non-aggregators, whose write requests fall into this
     * aggregator's file domain. others_req[] matters only for aggregators.
     */
    GIOI_Access *others_req;
    MPI_Offset **buf_idx = NULL;

    if (fh->bview.npairs <= 1)
        buf_idx = (MPI_Offset**) GIOI_Malloc(sizeof(MPI_Offset*) *
                                             fh->hints->cb_nodes);

    /* Calculate the portions of this rank's write requests that fall into the
     * file domains of each I/O aggregator. No inter-process communication is
     * performed in LUSTRE_Calc_my_req().
     */
    LUSTRE_Calc_my_req(fh, (fh->bview.npairs <= 1), &my_req, buf_idx);

    if (fh->hints->ds_write != GIOI_HINT_DISABLE) {
        /* When data sieving is considered, below check the current file size
         * first. If the aggregate access region of this collective write is
         * beyond the current file size, then we can safely skip the read of
         * the read-modify-write of data sieving.
         */
        if (fh->is_agg) {
            /* Obtain the current file size. Note an MPI_Allgather() has been
             * called above to calculate the aggregate access region. Thus all
             * prior independent I/O should have completed by now, so it is
             * safe to call lseek() to query the file size.
             */
            MPI_Offset cur_off, fsize;

            cur_off = lseek(fh->fd_sys, 0, SEEK_CUR);
            fsize   = lseek(fh->fd_sys, 0, SEEK_END);
            /* Ignore the error, and proceed as if file size is very large. */
#if GIO_DEBUG_MODE == 1
            if (fsize == -1)
                fprintf(stderr, "%s at %d: lseek SEEK_END failed on file %s (%s)\n",
                        __func__,__LINE__, fh->filename, strerror(errno));
#endif
            fh->skip_read = (fsize >=0 && min_st_off >= fsize);

            /* restore file pointer */
            lseek(fh->fd_sys, cur_off, SEEK_SET);
        }
    }
    else
        fh->skip_read = 1;

    /* For aggregators, calculate the portions of all other ranks' requests
     * fall into this aggregator's file domain (note only I/O aggregators are
     * assigned file domains).
     *
     * Inter-process communication is required to construct others_req[],
     * including MPI_Alltoall, MPI_Issend, MPI_Irecv, and MPI_Waitall.
     */
    err = LUSTRE_Calc_others_req(fh, my_req, &others_req);
    if (err != GIO_NOERR) return err;

    /* Two-phase I/O: first communication phase to exchange write data from all
     * ranks to the I/O aggregators, followed by the write phase where only I/O
     * aggregators write to the file.
     *
     * Unless MPI_Alltoallw() is used (when use_alltoallw is set to 1), there
     * is no collective MPI communication beyond this point, as
     * LUSTRE_Exch_and_write() calls only MPI_Issend, MPI_Irecv, and
     * MPI_Waitall. Thus it is safe for those non-aggregators making zero-sized
     * request to skip the call.
     */

    /* if this rank has data to write, then participate exchange-and-write */
    do_ex_wr = (fh->bview.size == 0) ? 0 : 1;
    use_alltoallw = 0;

#ifdef USE_MPI_ALLTOALLW
    {
        /* When num_NUMAs < striping_factor, using MPI_Alltoallw in
         * commit_comm_phase() is faster than MPI_Issend/MPI_Irecv ... ?
         */
        char *env_str;
        if ((env_str = getenv("GIO_USE_ALLTOALLW")) != NULL)
            use_alltoallw = (strcasecmp(env_str, "true") == 0) ? 1: 0;
    }
#endif

#if GIO_PROFILING_MODE == 1
    if (fh->is_agg) gio_wr_time[1] += MPI_Wtime() - curT;
#endif

    if (do_ex_wr || fh->is_agg)
        /* This rank participates exchange and write only when it has non-zero
         * data to write or is an I/O aggregator
         */
        w_len = LUSTRE_Exch_and_write(fh, buf, others_req, my_req,
                                      min_st_off, max_end_off, buf_idx);

    /* free all memory allocated */
    GIOI_Free(others_req[0].off);
    GIOI_Free(others_req);

    if (buf_idx != NULL) {
        GIOI_Free(buf_idx[0]);
        GIOI_Free(buf_idx);
    }
    GIOI_Free(my_req[0].off);
    GIOI_Free(my_req);

#ifdef ADJUST_STRIPING_UNIT
    /* restore the original striping_unit */
    fh->hints->striping_unit = orig_striping_unit;
#endif

    /* If this collective write is followed by an independent write, it's
     * possible to have those subsequent writes on other processes race ahead
     * and sneak in before the read-modify-write completes.  We carry out a
     * collective communication at the end here so no one can start independent
     * I/O before collective I/O completes.
     *
     * need to do some gymnastics with the error codes so that if something
     * went wrong, all processes report error, but if a process has a more
     * specific error code, we can still have that process report the
     * additional information
     */
    /* optimization: if only one process performing I/O, we can perform
     * a less-expensive Bcast. */
    if (fh->hints->cb_nodes == 1)
        MPI_Bcast(&w_len, 1, MPI_OFFSET, fh->hints->aggr_ranks[0], fh->comm);
    else
        MPI_Allreduce(MPI_IN_PLACE, &w_len, 1, MPI_OFFSET, MPI_MIN, fh->comm);

#if GIO_PROFILING_MODE == 1
    if (fh->is_agg) gio_wr_time[0] += MPI_Wtime() - curT;
#endif

    /* w_len may not be the same as bview.size, because data sieving may
     * write more than requested. When w_len < 0, it is a GIO error code.
     */
    return (w_len < 0) ? w_len : fh->bview.size;
}

