/***************************************************************************
 *
 *   SPDX-License-Identifier: BSD-3-Clause
 *   Copyright(c) 2007-2026 Intel Corporation
 * 
 *   These contents may have been developed with support from one or more
 *   Intel-operated generative artificial intelligence solutions.
 *
 ***************************************************************************/
/**
 ****************************************************************************
 * @file qae_mem_utils_common.h
 *
 * This file provides for Linux user space memory allocation. It uses
 * a driver that allocates the memory in kernel memory space (to ensure
 * physically contiguous memory) and maps it to
 * user space for use by the  quick assist sample code
 *
 ***************************************************************************/

#ifndef QAE_MEM_UTILS_COMMON_H
#define QAE_MEM_UTILS_COMMON_H

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <sys/types.h>
#include <dirent.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/param.h>
#ifndef ICP_WITHOUT_THREAD
#include <pthread.h>
#endif
#include <errno.h>
#include <stdarg.h>
#include <stdbool.h>
#include <unistd.h>
#include "qae_mem.h"
#include "qae_mem_utils.h"
#include "qae_mem_user_utils.h"
#include "qae_page_table_common.h"
#include "qae_mem_hugepage_utils.h"

/* Maximum cached memory size, 8 Mb by default */
#define MAX_CACHE_DEPTH_MB (0x800000)

/* Maximum supported alignment is 4M. */
#define QAE_MAX_PHYS_ALIGN (0x400000ULL)

/* Maximum supported allocation is 64M for vfio. */
#define QAE_MAX_ALLOC_SIZE (0x4000000ULL)

#ifdef MADV_WIPEONFORK
#define CACHE_PID
#endif

#ifdef CACHE_PID
extern void *cache_pid;
#endif

typedef struct
{
    dev_mem_info_t *head;
    dev_mem_info_t *tail;
} slab_list_t;

/* User space page table for fast virtual to physical address translation */
extern page_table_t g_page_table;
extern const uint64_t __qae_bitmask[65];
extern int g_fd;

extern free_page_table_fptr_t free_page_table_fptr;
extern load_addr_fptr_t load_addr_fptr;
extern load_key_fptr_t load_key_fptr;
extern store_mmap_range_fptr_t store_mmap_range_fptr;

API_LOCAL
int __qae_open(void);

API_LOCAL
void *__qae_mem_alloc(block_ctrl_t *block_ctrl, size_t size, size_t align);

API_LOCAL
bool __qae_mem_free(block_ctrl_t *block_ctrl, void *block, bool secure_free);

API_LOCAL
void __qae_finish_free_slab(const int fd, dev_mem_info_t *slab);

API_LOCAL
void *__qae_alloc_addr(size_t size,
                       const int node,
                       const size_t phys_alignment_byte);
API_LOCAL
void __qae_free_addr(void **p_va, bool secure_free);

API_LOCAL
uint64_t allocate_iova(const uint32_t size, uint32_t alignment);

API_LOCAL
void iova_release(uint64_t iova, uint32_t size);

API_LOCAL
int dma_map_slab(const void *virt, const uint64_t iova, const size_t size);

API_LOCAL
int dma_unmap_slab(const uint64_t iova, const size_t size);

API_LOCAL
void __qae_memFreeNUMA(void **ptr, bool secure_free);

static inline size_t div_round_up(const size_t n, const size_t d)
{
    return (n + d - 1) / d;
}

static inline size_t round_up(const size_t n, const size_t s)
{
    return ((n + s - 1) / s) * s;
}

static inline size_t get_header_size(const size_t num_blocks)
{
    return sizeof(block_ctrl_t) +
           (num_blocks / QWORD_WIDTH + 1) * sizeof(uint64_t) +
           num_blocks * sizeof(uint32_t);
}

/* Compute the base pointer of the sizes[] region that lives immediately
 * after bitmap[] in the slab. The sizes array holds the allocation length
 * (in units) for each block index: 0 = free, >0 = number of consecutive
 * units owned by the allocation starting at that index.
 */
static inline uint32_t *block_ctrl_sizes(block_ctrl_t *bc)
{
    return (uint32_t *)(bc->bitmap + bc->max_num_bitmaps + 1);
}

static inline void set_bitmap(uint64_t *bitmap, const size_t index, size_t len);

/* Initialise the geometry fields of a freshly-allocated block_ctrl_t and
 * set up the sizes[] pointer to reference the slab region immediately
 * after the bitmap sentinel word. The caller is responsible for zeroing
 * the bitmap[] and sizes[] regions before use.
 */
static inline void init_block_ctrl(block_ctrl_t *bc, const size_t unit_size)
{
    bc->unit_size = unit_size;
    bc->max_num_blocks = bc->mem_info.size / unit_size;
    bc->max_num_bitmaps = bc->max_num_blocks / QWORD_WIDTH;
    bc->header_size = get_header_size(bc->max_num_blocks);
    bc->reserved_units = div_round_up(bc->header_size, UNIT_SIZE);
    bc->sizes = block_ctrl_sizes(bc);

    /* initialise the bitmap to 1 for reserved blocks */
    set_bitmap(bc->bitmap, 0, bc->reserved_units);
    /* make a barrier to stop search at the end of the bitmap */
    bc->bitmap[bc->max_num_bitmaps] = QWORD_ALL_ONE;
}

/* mem_ctzll function
 * input: a 64-bit bitmap window
 * output: number of contiguous 0s from least significant bit position
 * __GNUC__ predefined macro and __builtin_ctz() are supported by Intel C
 */
static inline int32_t mem_ctzll(uint64_t bitmap_window)
{
    if (bitmap_window)
    {
#ifdef __GNUC__
        return __builtin_ctzll(bitmap_window);
#else
#error "Undefined built-in function"
#endif
    }
    return QWORD_WIDTH;
}

/* clear_bitmap function
 * clear the BITMAP_LENx64-bit bitmap from pos
 * for len length
 * input : map - pointer to the bitmap
 *         pos - bit position
 *         len - number of contiguous bits
 */
static inline void clear_bitmap(uint64_t *bitmap,
                                const size_t index,
                                size_t len)
{
    size_t qword = index / QWORD_WIDTH;
    const size_t offset = index % QWORD_WIDTH;
    size_t num;

    if (offset > 0)
    {
        const size_t width = MIN(len, QWORD_WIDTH - offset);
        const uint64_t mask = __qae_bitmask[width] << offset;

        /* Clear required bits */
        bitmap[qword] &= ~mask;

        len -= width;
        qword += 1;
    }

    num = len / QWORD_WIDTH;
    len %= QWORD_WIDTH;

    while (num--)
    {
        bitmap[qword++] = 0;
    }

    /* Clear remaining bits */
    bitmap[qword] &= ~__qae_bitmask[len];
}

/* set_bitmap function
 * set the BITMAP_LENx64-bit bitmap from pos
 * for len length
 * input : map - pointer to the bitmap
 *         pos - bit position
 *         len - number of contiguous bits
 */
static inline void set_bitmap(uint64_t *bitmap, const size_t index, size_t len)
{
    size_t qword = index / QWORD_WIDTH;
    const size_t offset = index % QWORD_WIDTH;
    size_t num;

    if (offset > 0)
    {
        const size_t width = MIN(len, QWORD_WIDTH - offset);
        const uint64_t mask = __qae_bitmask[width] << offset;

        /* Set required bits */
        bitmap[qword] |= mask;

        len -= width;
        qword += 1;
    }

    num = len / QWORD_WIDTH;
    len %= QWORD_WIDTH;

    while (num--)
    {
        bitmap[qword++] = ~0ULL;
    }

    /* Set remaining bits */
    bitmap[qword] |= __qae_bitmask[len];
}

#ifdef CACHE_PID
static inline void uncache_process_id(void)
{
    int ret = 0;
    if (cache_pid != NULL)
    {
        ret = qae_munmap(cache_pid, getpagesize());
        if (ret)
        {
            CMD_ERROR("%s:%d munmap call for cache failed, ret = %d\n",
                      __func__,
                      __LINE__,
                      ret);
        }
        cache_pid = NULL;
    }
}
static inline int cache_process_id(void)
{
    if (!cache_pid)
    {
        int page_size = getpagesize();

        cache_pid = qae_mmap(NULL,
                             page_size,
                             PROT_READ | PROT_WRITE,
                             MAP_PRIVATE | MAP_ANON,
                             -1,
                             0);
        if (cache_pid == NULL)
        {
            CMD_ERROR(
                "%s:%d Unable to mmap aligned memory \n", __func__, __LINE__);
            return -ENOMEM;
        }

        if (qae_madvise(cache_pid, page_size, MADV_WIPEONFORK))
        {

            CMD_ERROR(
                "%s:%d Unable to update page properties\n", __func__, __LINE__);
            qae_munmap(cache_pid, page_size);
            close(g_fd);
            g_fd = -1;
            cache_pid = NULL;
            return -ENOMEM;
        }
    }

    *((pid_t *)cache_pid) = getpid();
    return 0;
}
#endif /* CACHE_PID */

#ifndef CACHE_PID
static inline int check_pid(void)
{
    static pid_t pid = 0;
    pid_t current_pid = getpid();

    if (pid != current_pid)
    {
        pid = current_pid;
        return 1;
    }
    return 0;
}
#endif /* !CACHE_PID */

static inline int is_new_process()
{
    /* Check if it is a new process or child. */
#ifdef CACHE_PID
    const int is_new_pid =
        cache_pid == NULL || (cache_pid != NULL && *((pid_t *)cache_pid) == 0);
#else
    const int is_new_pid = check_pid();
#endif
    return is_new_pid;
}

#endif /* QAE_MEM_UTILS_COMMON_H */
