/*
 *  Copyright (C) 2026, Northwestern University
 *  See COPYRIGHT notice in top-level directory.
 */

#ifdef HAVE_CONFIG_H
# include <config.h>
#endif

#include <stdio.h>
#include <stdlib.h>
#include <string.h>     /* strerror() */
#include <fcntl.h>      /* open() */
#include <sys/types.h>  /* open(), umask() */
#include <sys/stat.h>   /* umask() */

#include <assert.h>
#include <sys/errno.h>
#include <unistd.h>     /* pwrite() */

#include "gioi.h"

/*----< GIOI_UFS_write_contig() >--------------------------------------------*/
MPI_Offset
GIOI_UFS_write_contig(GIO_File    fh,
                      const void *buf,
                      MPI_Offset  w_size,
                      MPI_Offset  offset,
                      int         is_coll)
{
    char *p;
    ssize_t err = 0;
    size_t w_count;
    MPI_Offset bytes_xfered = 0;

    if (w_size == 0) return GIO_NOERR;

#if GIO_DEBUG_MODE == 1
    assert(fh->is_open == 1);
    assert(fh->fd_sys >= 0);
#endif

#if GIO_DEBUG_MODE == 1 && (defined(HAVE_LUSTRE) || defined(MIMIC_LUSTRE))
    if (is_coll && fh->hints->lustre_num_osts > 0) {
        /* This call arrived from GIOI_Lustre_write_coll() */
        int rank;
        static int striping_factor=-1, cb_nodes=-1;
        static MPI_Offset first_ost_id = -1;
        MPI_Offset ost_id;

        MPI_Comm_rank(fh->comm, &rank);

        /* reset first_ost_id when striping_factor or cb_nodes changes */
        if (striping_factor != fh->hints->striping_factor ||
            cb_nodes != fh->hints->cb_nodes)
            first_ost_id = -1;

        ost_id = (offset / fh->hints->striping_unit)
               % fh->hints->lustre_num_osts;

        if (first_ost_id == -1)
            first_ost_id = ost_id;
        else if (ost_id != fh->hints->NUMA_ID) {
            /* Processes running on the same NUMA node should alwasy write to
             * the same OST.
             */
            printf("Warning %s on rank %d: pwrite offset=%lld w_size=%lld ost_id=%lld != NUMA_ID %d\n",__func__,rank,offset,w_size,ost_id,fh->hints->NUMA_ID);
        }
    }
#endif

#if GIO_PROFILING_MODE == 1
    double timing = MPI_Wtime();
#endif
    p = (char *) buf;
    while (bytes_xfered < w_size) {
        w_count = w_size - bytes_xfered;
        err = GIOI_pwrite(fh->fd_sys, p, w_count, offset + bytes_xfered);
        if (err == -1)
            goto err_out;
        if (err == 0)
            break;
        bytes_xfered += err;
        p += err;
    }
#if GIO_PROFILING_MODE == 1
    gio_wr_time[2] += MPI_Wtime() - timing;
#endif

err_out:
    if (err == -1)
        bytes_xfered = GIOI_error_posix("pwrite");

    return bytes_xfered;
}

/*----< GIOI_UFS_write_indep() >---------------------------------------------*/
/* This subroutine implements independent file write. It consists of two major
 * code segments. The first one is for when data sieving is disabled and the
 * second one enabled.
 *
 * Note in GIO, the file view and buffer view are never used for more than
 * one round, which greatly simplifies the implementation of this subroutine.
 */
MPI_Offset
GIOI_UFS_write_indep(GIO_File    fh,
                     const void *buf)
{
    char *ptr, *cpy_ptr, *tmp_buf=NULL;
    MPI_Offset i, j, k, ntimes;
    MPI_Offset file_off, lock_off, lock_len, len, total_len=0;
    MPI_Offset tmp_buf_size, file_rem, buf_rem;

#if GIO_DEBUG_MODE == 1
    /* User I/O hints should have been checked already. */
    assert(fh->hints->ind_wr_buffer_size > 0);

    /* zero-sized requests should have returned in GIO_read_indep() */
    assert(fh->bview.npairs > 0);
    assert(fh->bview.size > 0);

    /* File should have been opened already, even for non-aggregators */
    assert(fh->is_open == 1);
#endif

    if (fh->fview.npairs <= 1 && fh->bview.npairs <= 1) {
        /* Both buffer and file views are contiguous. */
        return GIOI_UFS_write_contig(fh, (char*)buf + fh->bview.off[0],
                                     fh->bview.size, fh->fview.off[0], 0);
    }

    lock_off = fh->fview.off[0];
    if (fh->fview.npairs > 1)
        lock_len = fh->fview.off[fh->fview.npairs-1]
                 + fh->fview.len[fh->fview.npairs-1]
                 - lock_off;
    else
        lock_len = fh->fview.size;

    /* if atomicity is true, lock (exclusive) the whole region */
    if (fh->atomicity)
        GIOI_WRITE_LOCK(fh, lock_off, SEEK_SET, lock_len);

    /* When data sieving write is disabled or fview is contiguous, write is
     * carried out in multiple rounds. In each round, write data is first
     * aggregated into a contiguous buffer, tmp_buf, and then write it to file
     * at the end of each round. This can improve performance when the write
     * buffer is non-contiguous and fview is contiguous, i.e. by reducing
     * the number of file writes.
     */
    if (fh->hints->ds_write == GIOI_HINT_DISABLE || fh->fview.npairs <= 1) {

        if (fh->bview.npairs <= 1) { /* use buf directly to write */
            tmp_buf = (char*)buf + fh->bview.off[0];
            buf_rem = fh->bview.size;
            ntimes = 1;
            tmp_buf_size = fh->bview.size;
        }
        else { /* buf is noncontiguous */
            tmp_buf_size = MIN(fh->bview.size, fh->hints->ind_wr_buffer_size);
            tmp_buf = (char*) GIOI_Malloc_align(tmp_buf_size, fh->hints->striping_unit);
            buf_rem = fh->bview.len[0];
            ntimes = fh->bview.size / tmp_buf_size;
            if (fh->bview.size % tmp_buf_size)
                ntimes++;
        }

        file_off = fh->fview.off[0];
        file_rem = fh->fview.len[0];

        /* pointer to buf, starting location to copy over to tmp_buf */
        cpy_ptr = (char*)buf + fh->bview.off[0];

        k = (fh->bview.npairs <= 1) ? 1 : 0; /* whether to skip while loop k */
        j = 0;
        for (i=0; i<ntimes; i++) { /* perform write in ntimes rounds */
            MPI_Offset req_len, tmp_buf_rem;

            /* copy data from buf to tmp_buf */
            tmp_buf_rem = tmp_buf_size;
            ptr = tmp_buf;
            while (k < fh->bview.npairs) {
                req_len = MIN(tmp_buf_rem, buf_rem);
                memcpy(ptr, cpy_ptr, req_len);

                ptr += req_len;
                tmp_buf_rem -= req_len;

                if (buf_rem == req_len) { /* done with pair k */
                    k++;
                    if (k < fh->bview.npairs) {
                        cpy_ptr = (char*)buf + fh->bview.off[k];
                        buf_rem = fh->bview.len[k];
                    }
                }
                else { /* there is still data remained in pair k */
                    cpy_ptr += req_len;
                    buf_rem -= req_len;
                }

                if (tmp_buf_rem == 0) break;
            }

            /* using tmp_buf to write to the file */
            tmp_buf_rem = tmp_buf_size;
            ptr = tmp_buf;
            while (j < fh->fview.npairs) {
                req_len = MIN(tmp_buf_rem, file_rem);
                /* write from offset file_off of length req_len */
                len = GIOI_UFS_write_contig(fh, ptr, req_len, file_off, 0);
                if (len < 0) return len;
                total_len += len;

                ptr += req_len;
                tmp_buf_rem -= req_len;

                if (file_rem == req_len) { /* done with pair j */
                    j++;
                    if (j < fh->fview.npairs) {
                        file_off = fh->fview.off[j];
                        file_rem = fh->fview.len[j];
                    }
                }
                else { /* there is still data remained in pair j */
                    file_off += req_len;
                    file_rem -= req_len;
                }

                if (tmp_buf_rem == 0) break;
            }
        }

        /* free tmp_buf if allocated */
        if (tmp_buf != buf) GIOI_Free(tmp_buf);
    }
    else {
        /* fview is noncontiguous and data sieving is not disabled */
        MPI_Offset disp, first_stripe, last_stripe, lock_rem, cpy_len;

        /* allocate read-modify-write buffer */
        tmp_buf_size = MIN(lock_len, fh->hints->striping_unit);
        tmp_buf = (char*) GIOI_Malloc_align(tmp_buf_size, fh->hints->striping_unit);

        /* lock_rem is the amount remained to be locked for the entire
         * read-modify-write region.
         */
        lock_rem = lock_len;

        /* perform ntimes rounds of read-modify-write */
        first_stripe = lock_off / fh->hints->striping_unit;
        last_stripe = (lock_off + lock_len - 1) / fh->hints->striping_unit;
        ntimes = (last_stripe - first_stripe) + 1;

#if GIO_DEBUG_MODE == 1
        /* fview's offsets should have already sorted into a monotonically
         * non-decreasing order without overlaps. In addition, earlier checks
         * have ensured all fh->fview.len[] > 0.
         */
        assert(fh->fview.len[0] > 0);
        for (i=1; i<fh->fview.npairs; i++) {
            assert(fh->fview.len[i] > 0);
            assert(fh->fview.off[i-1] < fh->fview.off[i]);
            assert(fh->fview.off[i-1] + fh->fview.len[i-1] <=
                   fh->fview.off[i]);
            /* Note the file view's offset-length pairs if passed from PnetCDF
             * have been coalesced. Thus,
             * fh->fview.off[i-1] + fh->fview.len[i-1] < fh->fview.off[i]);
             */
        }
#endif

        /* initialize loop i's local variables with the 1st pair of fview and
         * bview
         */
        file_off = fh->fview.off[0];
        file_rem = fh->fview.len[0];
        buf_rem  = fh->bview.len[0];

        /* pointer to buf, starting location to copy over to tmp_buf */
        cpy_ptr = (char*)buf + fh->bview.off[0];

        disp = 0;
        k = 0; /* index pointed to bview's  offset-length pairs */
        j = 0; /* index pointed to fview's offset-length pairs */
        for (i=0; i<ntimes; i++) { /* perform write in ntimes rounds */
            MPI_Offset req_len, tmp_buf_rem, gap;

            /* adjust tmp_buf_size to achieve striping_unit aligned file
             * access
             */
            tmp_buf_size = fh->hints->striping_unit
                         - (file_off % fh->hints->striping_unit);

            if (disp >= tmp_buf_size) {
                /* This displacement at the beginning of read-modify-write
                 * region of this round i is too large, containing no data to
                 * be copied from the user write buffer. This allows to skip a
                 * few rounds.
                 */
                MPI_Offset skip = disp / tmp_buf_size;
                i        += skip - 1;
                disp     -= skip * tmp_buf_size;
                lock_rem -= skip * tmp_buf_size;
                file_off += skip * tmp_buf_size;
                continue;
            }

            /* read a chunk from the file into tmp_buf */
            req_len = MIN(tmp_buf_size, lock_rem);

            if (!fh->atomicity) /* lock the read-modify-write region */
                GIOI_WRITE_LOCK(fh, file_off, SEEK_SET, req_len);

            len = GIOI_UFS_read_contig(fh, tmp_buf, req_len, file_off);
            if (len < 0) return len;

            /* In case read's len is shorter than requested, zero-out the
             * remaining contents of tmp_buf.
             */
            if (len < req_len)
                memset(tmp_buf + len, 0, req_len - len);

            /* Copy data from buf to tmp_buf. Skip 'disp' bytes at the front
             * for both buffers.
             */
            tmp_buf_rem = MIN(tmp_buf_size, lock_rem) - disp;
            ptr = tmp_buf + disp;

            /* disp will be reset at the end of each round */
            disp = 0;

            while (tmp_buf_rem > 0) {
                /* cpy_len is the amount to be copied over. It should be no
                 * more than remaining of temporary buffer size of this round
                 * i, remaining of the fview offset-length pair j, and
                 * remaining of bview's offset-length pair k.
                 */
                cpy_len = MIN(file_rem, buf_rem);
                cpy_len = MIN(cpy_len, tmp_buf_rem);
                memcpy(ptr, cpy_ptr, cpy_len);
                total_len += cpy_len;

                /* Deduct remaining of temp buffer.
                 * Note even if tmp_buf_rem == 0, we still need continue to
                 * calculate disp for the next round.
                 */
                tmp_buf_rem -= cpy_len;

                /* advance buffer pointer */
                ptr += cpy_len;

                if (buf_rem == cpy_len) { /* done with pair k */
                    k++;
                    if (k == fh->bview.npairs) /* all data have been copied */
                        break;
                    cpy_ptr = (char*)buf + fh->bview.off[k];
                    buf_rem = fh->bview.len[k];
                }
                else { /* there is still data remained in pair k */
                    cpy_ptr += cpy_len;
                    buf_rem -= cpy_len;
                }

                if (file_rem == cpy_len) { /* done with pair j */
                    /* When j is the last pair of this round i, the end offset
                     * of tmp_buf may fall between the end of pair j and the
                     * beginning of pair j+1. In this case, the beginning of
                     * file offset region for the next round should advance
                     * disp bytes.
                     */

                    j++;
                    assert(j < fh->fview.npairs);
                    /* Note j should never become fview.npairs, as the above
                     * check of if (k == bview.npairs) should short-circuit
                     * the while loop. This is because GIO requires
                     * fview.size == bview.size.
                     */

                    file_rem = fh->fview.len[j];

                    /* calculate the empty size between pairs j-1 and j */
                    gap = fh->fview.off[j]
                        - (fh->fview.off[j-1] + fh->fview.len[j-1]);

                    if (tmp_buf_rem <= gap) {
                        /* j-1 is last pair of this round */
                        disp = gap - tmp_buf_rem;
                        break;
                    }
                    tmp_buf_rem -= gap;
                    ptr += gap;
                }
                else { /* there is still data remained in pair j */
                    file_rem -= cpy_len;
                }
            }

            /* write the modified chunk to the file */
            len = GIOI_UFS_write_contig(fh, tmp_buf, req_len, file_off, 0);
            if (len < 0) return len;

            if (!fh->atomicity) /* unlock the read-modify-write region */
                GIOI_UNLOCK(fh, file_off, SEEK_SET, req_len);

            /* reduce remaining size to be locked */
            lock_rem -= req_len;

            /* update file offset for the next round of read-modify-write */
            file_off += req_len;
        }

        /* free tmp_buf if allocated */
        if (tmp_buf != NULL) GIOI_Free(tmp_buf);
    }

    /* if atomicity is true, unlock (exclusive) the whole region */
    if (fh->atomicity)
        GIOI_UNLOCK(fh, lock_off, SEEK_SET, lock_len);

    return total_len;
}

