/*
 * =======================================================================================
 *
 *      Filename:  nvmon_cupti.h
 *
 *      Description:  Header File of nvmon module (CUPTI backend).
 *
 *      Version:   <VERSION>
 *      Released:  <DATE>
 *
 *      Author:   Thomas Gruber (tg), thomas.gruber@googlemail.com
 *      Project:  likwid
 *
 *      Copyright (C) 2019 RRZE, University Erlangen-Nuremberg
 *
 *      This program is free software: you can redistribute it and/or modify it under
 *      the terms of the GNU General Public License as published by the Free Software
 *      Foundation, either version 3 of the License, or (at your option) any later
 *      version.
 *
 *      This program is distributed in the hope that it will be useful, but WITHOUT ANY
 *      WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 *      PARTICULAR PURPOSE.  See the GNU General Public License for more details.
 *
 *      You should have received a copy of the GNU General Public License along with
 *      this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * =======================================================================================
 */

#ifndef LIKWID_NVMON_CUPTI_H
#define LIKWID_NVMON_CUPTI_H


#include <cuda.h>
#include <cupti.h>

/* Copy from PAPI's cuda component (BSD License)
 * @author  Asim YarKhan yarkhan@icl.utk.edu (updated in 2017 to support CUDA metrics)
 * @author  Asim YarKhan yarkhan@icl.utk.edu (updated in 2015 for multiple CUDA contexts/devices)
 * @author  Heike Jagode (First version, in collaboration with Robert Dietrich, TU Dresden) jagode@icl.utk.edu
 */
void (*_dl_non_dynamic_init) (void) __attribute__ ((weak));

#ifndef CU_CALL
#define CU_CALL(func, ...)                            \
    do {                                                \
        CUresult s_ = (*func##_ptr)(__VA_ARGS__);    \
        if (s_ != CUDA_SUCCESS) {                        \
            const char *errstr = NULL;\
            cuGetErrorString_ptr(s_, &errstr);\
            ERROR_PRINT("Error: function %s failed with error: '%s' (CUresult=%d).", #func, errstr, s_);   \
            return -EPERM;                              \
        }                                               \
    } while (0)
#endif

#ifndef CUPTI_CALL
#define CUPTI_CALL(func, ...)                            \
    do {                                                \
        CUptiResult s_ = (*func##_ptr)(__VA_ARGS__);    \
        if (s_ != CUPTI_SUCCESS) {                        \
            const char *errstr = NULL; \
            cuptiGetResultString_ptr(s_, &errstr); \
            ERROR_PRINT("Error: function %s failed with error: '%s' (CUptiResult=%d)", #func, errstr, s_);   \
            return -EPERM;                              \
        }                                               \
    } while (0)
#endif

#define CUDA_CALL(func, ...)                            \
    do {                                                \
        cudaError_t s_ = (*func##_ptr)(__VA_ARGS__);    \
        if (s_ != cudaSuccess) {                        \
            ERROR_PRINT("Error: function %s failed with error: '%s' (cudaError_t=%d).", #func, cudaGetErrorString_ptr(s_), s_);   \
            return -EPERM;                              \
        }                                               \
    } while (0)

#define DECLARECUFUNC(funcname, ...) CUresult __attribute__((weak)) funcname(__VA_ARGS__);  static CUresult (*funcname##_ptr)(__VA_ARGS__);
DECLARECUFUNC(cuCtxGetCurrent, CUcontext *);
DECLARECUFUNC(cuCtxSetCurrent, CUcontext);
DECLARECUFUNC(cuCtxDestroy, CUcontext);
DECLARECUFUNC(cuDeviceGet, CUdevice *, int);
DECLARECUFUNC(cuDeviceGetCount, int *);
DECLARECUFUNC(cuDeviceGetName, char *, int, CUdevice);
DECLARECUFUNC(cuGetErrorString, CUresult, const char **);
DECLARECUFUNC(cuInit, unsigned int);
DECLARECUFUNC(cuCtxPopCurrent, CUcontext * pctx);
DECLARECUFUNC(cuCtxPushCurrent, CUcontext pctx);
DECLARECUFUNC(cuCtxSynchronize);

#define DECLARECUDAFUNC(funcname, ...) cudaError_t __attribute__((weak)) funcname(__VA_ARGS__);  static cudaError_t (*funcname##_ptr)(__VA_ARGS__);
DECLARECUDAFUNC(cudaGetDevice, int *);
DECLARECUDAFUNC(cudaSetDevice, int);
DECLARECUDAFUNC(cudaFree, void *);
const char * __attribute__((weak)) cudaGetErrorString(cudaError_t);
static const char *(*cudaGetErrorString_ptr)(cudaError_t);

#define DECLARECUPTIFUNC(funcname, ...) CUptiResult __attribute__((weak)) funcname(__VA_ARGS__);  static CUptiResult (*funcname##_ptr)(__VA_ARGS__);
DECLARECUPTIFUNC(cuptiEventGroupGetAttribute, CUpti_EventGroup eventGroup, CUpti_EventGroupAttribute attrib, size_t * valueSize, void *value);
DECLARECUPTIFUNC(cuptiDeviceGetEventDomainAttribute, CUdevice device, CUpti_EventDomainID eventDomain, CUpti_EventDomainAttribute attrib, size_t * valueSize, void *value);
DECLARECUPTIFUNC(cuptiEventGroupReadEvent, CUpti_EventGroup eventGroup, CUpti_ReadEventFlags flags, CUpti_EventID event, size_t * eventValueBufferSizeBytes, uint64_t *        eventValueBuffer);
DECLARECUPTIFUNC(cuptiEventGroupSetAttribute, CUpti_EventGroup eventGroup, CUpti_EventGroupAttribute attrib, size_t valueSize, void *value);
DECLARECUPTIFUNC(cuptiEventGroupSetDisable, CUpti_EventGroupSet * eventGroupSet);
DECLARECUPTIFUNC(cuptiEventGroupSetEnable, CUpti_EventGroupSet * eventGroupSet);
DECLARECUPTIFUNC(cuptiEventGroupSetsCreate, CUcontext context, size_t eventIdArraySizeBytes, CUpti_EventID * eventIdArray, CUpti_EventGroupSets ** eventGroupPasses);
DECLARECUPTIFUNC(cuptiEventGroupSetsDestroy, CUpti_EventGroupSets * eventGroupSets);
DECLARECUPTIFUNC(cuptiGetTimestamp, uint64_t * timestamp);
DECLARECUPTIFUNC(cuptiSetEventCollectionMode, CUcontext context, CUpti_EventCollectionMode mode);
DECLARECUPTIFUNC(cuptiDeviceEnumEventDomains, CUdevice, size_t *, CUpti_EventDomainID *);
DECLARECUPTIFUNC(cuptiDeviceGetNumEventDomains, CUdevice, uint32_t *);
DECLARECUPTIFUNC(cuptiEventDomainEnumEvents, CUpti_EventDomainID, size_t *, CUpti_EventID *);
DECLARECUPTIFUNC(cuptiEventDomainGetNumEvents, CUpti_EventDomainID, uint32_t *);
DECLARECUPTIFUNC(cuptiEventGetAttribute, CUpti_EventID, CUpti_EventAttribute, size_t *, void *);
DECLARECUPTIFUNC(cuptiGetResultString, CUptiResult result, const char **str);


static void *dl_libcuda = NULL;
static void *dl_libcudart = NULL;
static void *dl_libcupti = NULL;

static int
link_cputi_libraries(void)
{
#define DLSYM_AND_CHECK( dllib, name ) dlsym( dllib, name ); if ( dlerror() != NULL ) { return -1; }

    /* Attempt to guess if we were statically linked to libc, if so bail */
    if(_dl_non_dynamic_init != NULL) {
        return -1;
    }
    /* Need to link in the cuda libraries, if not found disable the component */
    dl_libcuda = dlopen("libcuda.so", RTLD_NOW | RTLD_GLOBAL);
    if (!dl_libcuda)
    {
        fprintf(stderr, "CUDA library libcuda.so not found.");
        return -1;
    }
    cuCtxGetCurrent_ptr = DLSYM_AND_CHECK(dl_libcuda, "cuCtxGetCurrent");
    cuCtxSetCurrent_ptr = DLSYM_AND_CHECK(dl_libcuda, "cuCtxSetCurrent");
    cuDeviceGet_ptr = DLSYM_AND_CHECK(dl_libcuda, "cuDeviceGet");
    cuDeviceGetCount_ptr = DLSYM_AND_CHECK(dl_libcuda, "cuDeviceGetCount");
    cuDeviceGetName_ptr = DLSYM_AND_CHECK(dl_libcuda, "cuDeviceGetName");
    cuGetErrorString_ptr = DLSYM_AND_CHECK(dl_libcuda, "cuGetErrorString");
    cuInit_ptr = DLSYM_AND_CHECK(dl_libcuda, "cuInit");
    cuCtxPopCurrent_ptr = DLSYM_AND_CHECK(dl_libcuda, "cuCtxPopCurrent");
    cuCtxPushCurrent_ptr = DLSYM_AND_CHECK(dl_libcuda, "cuCtxPushCurrent");
    cuCtxSynchronize_ptr = DLSYM_AND_CHECK(dl_libcuda, "cuCtxSynchronize");
    cuCtxDestroy_ptr = DLSYM_AND_CHECK(dl_libcuda, "cuCtxDestroy");

    dl_libcudart = dlopen("libcudart.so", RTLD_NOW | RTLD_GLOBAL | RTLD_NODELETE);
    if (!dl_libcudart)
    {
        fprintf(stderr, "CUDA runtime library libcudart.so not found.");
        return -1;
    }
    cudaGetDevice_ptr = DLSYM_AND_CHECK(dl_libcudart, "cudaGetDevice");
    cudaSetDevice_ptr = DLSYM_AND_CHECK(dl_libcudart, "cudaSetDevice");
    cudaFree_ptr = DLSYM_AND_CHECK(dl_libcudart, "cudaFree");
    cudaGetErrorString_ptr = DLSYM_AND_CHECK(dl_libcudart, "cudaGetErrorString");

    dl_libcupti = dlopen("libcupti.so", RTLD_NOW | RTLD_GLOBAL);
    if (!dl_libcupti)
    {
        fprintf(stderr, "CUDA runtime library libcupti.so not found.");
        return -1;
    }
    cuptiDeviceGetEventDomainAttribute_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiDeviceGetEventDomainAttribute");
    cuptiEventGroupGetAttribute_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiEventGroupGetAttribute");
    cuptiEventGroupReadEvent_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiEventGroupReadEvent");
    cuptiEventGroupSetAttribute_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiEventGroupSetAttribute");
    cuptiEventGroupSetDisable_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiEventGroupSetDisable");
    cuptiEventGroupSetEnable_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiEventGroupSetEnable");
    cuptiEventGroupSetsCreate_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiEventGroupSetsCreate");
    cuptiEventGroupSetsDestroy_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiEventGroupSetsDestroy");
    cuptiGetTimestamp_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiGetTimestamp");
    cuptiSetEventCollectionMode_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiSetEventCollectionMode");
    cuptiDeviceEnumEventDomains_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiDeviceEnumEventDomains");
    cuptiDeviceGetNumEventDomains_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiDeviceGetNumEventDomains");
    cuptiEventDomainEnumEvents_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiEventDomainEnumEvents");
    cuptiEventDomainGetNumEvents_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiEventDomainGetNumEvents");
    cuptiEventGetAttribute_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiEventGetAttribute");
    cuptiGetResultString_ptr = DLSYM_AND_CHECK(dl_libcupti, "cuptiGetResultString");
    return 0;
}


static int check_nv_context(NvmonDevice_t device, CUcontext currentContext)
{
    int j = 0;
    int need_pop = 0;
    GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Current context %p DevContext %p", currentContext, device->context);
    if (!device->context)
    {
        int context_of_dev = -1;
        for (j = 0; j < nvGroupSet->numberOfGPUs; j++)
        {
            NvmonDevice_t dev = &nvGroupSet->gpus[j];
            if (dev->context == currentContext)
            {
                context_of_dev = j;
                break;
            }
        }
        if (context_of_dev < 0 && !device->context)
        {
            device->context = currentContext;
            GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Reuse context %p for device %d", device->context, device->deviceId);
        }
        else
        {
            CUDA_CALL(cudaSetDevice, device->deviceId);
            CUDA_CALL(cudaFree, NULL);
            CU_CALL(cuCtxGetCurrent, &device->context);
            GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "New context %p for device %d", device->context, device->deviceId);
        }
    }
    else if (device->context != currentContext)
    {
        GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Use context %p for device %d", device->context, device->deviceId);
        CU_CALL(cuCtxPushCurrent, device->context);
        need_pop = 1;
    }
    else
    {
        GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Context %p fits for device %d", device->context, device->deviceId);
    }
    return need_pop;
}

void nvmon_cupti_freeDevice(NvmonDevice_t dev)
{
    if (dev)
    {
        if (dev->context)
        {
            cuCtxDestroy_ptr(dev->context);
            dev->context = NULL;
        }
        if (dev->cuEventSets)
        {
            cuptiEventGroupSetsDestroy_ptr(dev->cuEventSets);
            dev->cuEventSets = NULL;
        }

        // The event objects are freed here

        if (dev->activeEvents)
        {
            free(dev->activeEvents);
            dev->activeEvents = NULL;
            dev->numActiveEvents = 0;
        }

        if (dev->nvEventSets)
        {
            for (int j = 0; j < dev->numNvEventSets; j++)
            {
                free(dev->nvEventSets[j].results);
            }
            free(dev->nvEventSets);
            dev->nvEventSets = NULL;
            dev->numNvEventSets = 0;
        }
        if (dev->allevents)
        {
            for (int j = 0; j < dev->numAllEvents; j++)
            {
                if (dev->allevents[j])
                {
                    free(dev->allevents[j]);
                }
            }
        }
        memset(dev, 0, sizeof(NvmonDevice));
    }
    return;
}

int
nvmon_cupti_createDevice(int id, NvmonDevice *dev)
{
    int c = 0;
    unsigned numDomains = 0;
    CUpti_EventDomainID* eventDomainIds = NULL;
    uint32_t totalEvents = 0;


    dev->deviceId = id;
    dev->cuEventSets = NULL;
    dev->context = 0UL;
    dev->activeEvents = NULL;
    dev->numActiveEvents = 0;
    dev->numNvEventSets = 0;
    dev->nvEventSets = NULL;

    if ((!dl_libcuda) || (!dl_libcudart) || (!dl_libcupti))
    {
        int err = link_cputi_libraries();
        if (err < 0)
        {
            return -1;
        }
    }

    // Assign device ID and get cuDevice from CUDA
    CU_CALL(cuDeviceGet, &dev->cuDevice, id);

    // Get the number of event domains of the device
    CUPTI_CALL(cuptiDeviceGetNumEventDomains, dev->cuDevice, &numDomains);
    GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Nvmon: Dev %d Domains %d", id, numDomains);

    // Get the domain IDs for the device
    size_t domainarraysize = numDomains * sizeof(CUpti_EventDomainID);
    eventDomainIds = malloc(domainarraysize);
    if (!eventDomainIds) return -ENOMEM;
    CUPTI_CALL(cuptiDeviceEnumEventDomains, dev->cuDevice, &domainarraysize, eventDomainIds);


    // Count the events in all domains to allocate the event list
    dev->numAllEvents = 0;
    for (unsigned j = 0; j < numDomains; j++)
    {
        uint32_t domainNumEvents = 0;
        CUpti_EventDomainID domainID = eventDomainIds[j];
        CUPTI_CALL(cuptiEventDomainGetNumEvents, domainID, &domainNumEvents);
        GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Nvmon: Dev %d Domain %d Events %d", id, j, domainNumEvents);
        totalEvents += domainNumEvents;
    }
    // Now we now how many events are provided by the device, so allocate a big enough event list
    dev->allevents = malloc(totalEvents * sizeof(NvmonEvent_t));
    if (!dev->allevents)
    {
        free(eventDomainIds);
        return -ENOMEM;
    }

    // We use hash tables for faster access
    dev->eventHash = g_hash_table_new(g_str_hash, g_str_equal);
    dev->evIdHash = g_hash_table_new(g_int64_hash, g_int64_equal);

    for (unsigned j = 0; j < numDomains; j++)
    {
        uint32_t domainNumEvents = 0;
        CUpti_EventDomainID domainID = eventDomainIds[j];
        // How many events are provided by the domain
        CUPTI_CALL(cuptiEventDomainGetNumEvents, domainID, &domainNumEvents);
        size_t tmpSize = domainNumEvents * sizeof(CUpti_EventID);
        // Allocate space for all CUPTI event IDs in the domain
        CUpti_EventID* cuEventIds = malloc(tmpSize);
        // Get the CUPTI events
        CUPTI_CALL(cuptiEventDomainEnumEvents, domainID, &tmpSize, cuEventIds);
        for (unsigned k = 0; k < domainNumEvents; k++)
        {
            CUpti_EventID eventId = cuEventIds[k];
            // Name and description are limited in length
            size_t tmpSizeBytesName = (NVMON_DEFAULT_STR_LEN-1) * sizeof(char);
            size_t tmpSizeBytesDesc = (NVMON_DEFAULT_STR_LEN-1) * sizeof(char);

            NvmonEvent_t event = malloc(sizeof(NvmonEvent));
            if (!event)
            {
                free(cuEventIds);
                free(dev->allevents);
                free(eventDomainIds);
                return -ENOMEM;
            }

            // Get event name and description
            CUPTI_CALL(cuptiEventGetAttribute, eventId, CUPTI_EVENT_ATTR_NAME, &tmpSizeBytesName, event->name);
            CUPTI_CALL(cuptiEventGetAttribute, eventId, CUPTI_EVENT_ATTR_LONG_DESCRIPTION, &tmpSizeBytesDesc, event->description);
            event->name[tmpSizeBytesName/sizeof(char)] = '\0';
            event->description[tmpSizeBytesDesc/sizeof(char)] = '\0';
            // LIKWID events are all uppercase, so transform the event names
            c = 0;
            while (event->name[c] != '\0')
            {
                event->name[c] = toupper(event->name[c]);
                c++;
            }
            // Save all gathered information in a NvmonEvent object

            event->cuEventId = eventId;
            event->eventId = dev->numAllEvents;
            event->cuDomainId = domainID;
            event->domainId = j;
            event->type = NVMON_CUPTI_EVENT;
            event->active = 0;
            //GPUDEBUG_PRINT(DEBUGLEV_DETAIL, "New Event %d CuEvent %d Domain %d CuDomain %d Name %s", event->eventId, (int)event->cuEventId, event->domainId, (int)event->cuDomainId, event->name);
            // Add the object to the event list
            dev->allevents[dev->numAllEvents] = event;
            dev->numAllEvents++;
            // Add the object to the hash tables
            char* nameKey = g_strdup(event->name);

            CUpti_EventID* idKey = malloc(sizeof(CUpti_EventID));
            if (idKey)
            {
                *idKey = event->cuEventId;
                // Key is event name
                g_hash_table_insert(dev->eventHash, (gpointer)nameKey, (gpointer)event);
                // Key is CUPTI event ID
                g_hash_table_insert(dev->evIdHash, (gpointer)idKey, (gpointer)event);
            }
        }
    }

/*    CUDA_CALL(cudaSetDevice, dev->deviceId);*/
/*    CUDA_CALL(cudaFree, NULL);*/
/*    CU_CALL(cuCtxGetCurrent, &dev->context);*/
    return 0;
}



int nvmon_cupti_getEventsOfGpu(int gpuId, NvmonEventList_t* list)
{
    int ret = 0;
    NvmonDevice device;
    int err = nvmon_cupti_createDevice(gpuId, &device);
    if (!err)
    {
        NvmonEventList_t l = malloc(sizeof(NvmonEventList));
        if (l)
        {
            l->events = malloc(sizeof(NvmonEventListEntry) * device.numAllEvents);
            if (l->events)
            {
                for (int i = 0; i < device.numAllEvents; i++)
                {
                    NvmonEventListEntry* out = &l->events[i];
                    NvmonEvent_t event = device.allevents[i];
                    out->name = malloc(strlen(event->name)+2);
                    if (out->name)
                    {
                        ret = snprintf(out->name, strlen(event->name)+1, "%s", event->name);
                        if (ret > 0)
                        {
                            out->name[ret] = '\0';
                        }
                    }
                    out->desc = malloc(strlen(event->description)+2);
                    if (out->desc)
                    {
                        ret = snprintf(out->desc, strlen(event->description)+1, "%s", event->description);
                        if (ret > 0)
                        {
                            out->desc[ret] = '\0';
                        }
                    }
                    out->limit = malloc(10*sizeof(char));
                    if (out->limit)
                    {
                        switch (event->type)
                        {
                            case NVMON_CUPTI_EVENT:
                                ret = snprintf(out->limit, 9, "GPU");
                                if (ret > 0) out->limit[ret] = '\0';
                                break;
                            default:
                                break;
                        }
                    }
                }
                l->numEvents = device.numAllEvents;
                *list = l;
            }
            else
            {
                free(l);
                nvmon_cupti_freeDevice(&device);
                return -ENOMEM;
            }
        }
    }
    else
    {
        ERROR_PRINT("No such device %d", gpuId);
    }
    return 0;
}



int
nvmon_cupti_addEventSets(NvmonDevice_t device, const char* eventString)
{
    int i = 0;
    int curDeviceId = -1;
    CUcontext curContext;
    struct bstrList* eventtokens = NULL;

    CUDA_CALL(cudaGetDevice, &curDeviceId);
    CUDA_CALL(cudaFree, NULL);
    CU_CALL(cuCtxGetCurrent, &curContext);

    bstring eventBString = bfromcstr(eventString);
    eventtokens = bsplit(eventBString, ',');
    bdestroy(eventBString);

    if (curDeviceId != device->deviceId)
    {
        CUDA_CALL(cudaSetDevice, device->deviceId);
    }

    int popContext = 0;
    CUpti_EventGroupSets * cuEventSets = NULL;

    NvmonEventSet* tmpEventSet = realloc(device->nvEventSets, (device->numNvEventSets+1)*sizeof(NvmonEventSet));
    if (!tmpEventSet)
    {
        ERROR_PRINT("Cannot enlarge GPU %d eventSet list", device->deviceId);
        return -ENOMEM;
    }
    device->nvEventSets = tmpEventSet;
    NvmonEventSet* devEventSet = &device->nvEventSets[device->numNvEventSets];

    devEventSet->nvEvents = (NvmonEvent_t*) malloc(eventtokens->qty * sizeof(NvmonEvent_t));
    if (devEventSet->nvEvents == NULL)
    {
        ERROR_PRINT("Cannot allocate event list for group %d", groupSet->numberOfActiveGroups);
        return -ENOMEM;
    }
    devEventSet->cuEventIDs = (CUpti_EventID*) malloc(eventtokens->qty * sizeof(CUpti_EventID));
    if (devEventSet->cuEventIDs == NULL)
    {
        ERROR_PRINT("Cannot allocate event ID list for group %d", groupSet->numberOfActiveGroups);
        free(devEventSet->nvEvents);
        return -ENOMEM;
    }
    devEventSet->results = malloc(eventtokens->qty * sizeof(NvmonEventResult));
    if (devEventSet->cuEventIDs == NULL)
    {
        ERROR_PRINT("Cannot allocate result list for group %d", groupSet->numberOfActiveGroups);
        free(devEventSet->cuEventIDs);
        free(devEventSet->nvEvents);
        return -ENOMEM;
    }
    memset(devEventSet->results, 0, eventtokens->qty * sizeof(NvmonEventResult));
    devEventSet->numberOfEvents = 0;

    // If the device has no context, check whether the current context is
    // used already by another device
    popContext = check_nv_context(device, curContext);


    CUPTI_CALL(cuptiSetEventCollectionMode, device->context, CUPTI_EVENT_COLLECTION_MODE_KERNEL);

    for (i = 0; i < eventtokens->qty; i++)
    {
        struct bstrList* evset = bsplit(eventtokens->entry[i], ':');
        if (evset->qty != 2)
        {
            ERROR_PRINT("NVMON: Event %s invalid: Format <event>:<gpucounter>", bdata(eventtokens->entry[i]));
        }
        if (blength(evset->entry[0]) == 0 || blength(evset->entry[1]) == 0)
        {
            ERROR_PRINT("NVMON: Event %s invalid: Format <event>:<gpucounter>", bdata(eventtokens->entry[i]));
        }
        NvmonEvent_t event = g_hash_table_lookup(device->eventHash, (gpointer)bdata(evset->entry[0]));
        if (!event)
        {
            GPUDEBUG_PRINT(DEBUGLEV_INFO, "NVMON: Event %s unknown. Skipping...", bdata(evset->entry[0]));
            continue; //unknown event
        }
        else
        {
            devEventSet->cuEventIDs[devEventSet->numberOfEvents] = event->cuEventId;
            devEventSet->nvEvents[devEventSet->numberOfEvents] = event;
            devEventSet->numberOfEvents++;
            size_t s = devEventSet->numberOfEvents*sizeof(CUpti_EventID);

            CUPTI_CALL(cuptiEventGroupSetsCreate, device->context, s, devEventSet->cuEventIDs, &cuEventSets);
            if (cuEventSets->numSets > 1)
            {
                ERROR_PRINT("Error adding event %s. Multiple measurement runs are required. skipping event ...", bdata(evset->entry[i]));
                continue;
            }
        }
    }
    devEventSet->id = nvGroupSet->numberOfActiveGroups;
    // Create an eventset with the currently configured and current event
    if(popContext)
    {
        CU_CALL(cuCtxPopCurrent, &device->context);
    }
    device->numNvEventSets++;
    return 0;
}

int nvmon_cupti_setupCounters(NvmonDevice_t device, NvmonEventSet* eventSet)
{
    int popContext = 0;
    int oldDevId = -1;
    CUpti_EventGroupSets * cuEventSets = NULL;
    CUcontext curContext;

    if (eventSet->numberOfEvents == 0)
    {
        GPUDEBUG_PRINT(DEBUGLEV_DETAIL, "Skipping GPU%d it has no events in group %d", device->deviceId, eventSet->id);
        return -EINVAL;
    }
    // Currently we are on which device?
    CUDA_CALL(cudaGetDevice, &oldDevId);
    // This is a workaround to (eventually create and) get the current context
    CUDA_CALL(cudaFree, NULL);
    CU_CALL(cuCtxGetCurrent, &curContext);

    popContext = check_nv_context(device, curContext);


    size_t grpEventIdsSize = eventSet->numberOfEvents * sizeof(CUpti_EventID);
    CUPTI_CALL(cuptiEventGroupSetsCreate, device->context, grpEventIdsSize, eventSet->cuEventIDs, &cuEventSets);
    // Allocate temporary array to hold the group event IDs
    CUpti_EventID *grpEventIds = malloc(grpEventIdsSize);
    if (!grpEventIds)
    {
        return -ENOMEM;
    }
    // Delete current activeEvent list
    if (device->activeEvents)
    {
        free(device->activeEvents);
        device->activeEvents = NULL;
        device->numActiveEvents = 0;
    }
    // Delete current activeCuGroups list
    if (device->activeCuGroups)
    {
        free(device->activeCuGroups);
        device->activeCuGroups = NULL;
        device->numActiveCuGroups = 0;
    }

    // Create a new activeEvent list
    device->activeEvents = malloc(eventSet->numberOfEvents * sizeof(NvmonActiveEvent));
    if (!device->activeEvents)
    {
        free(grpEventIds);
        grpEventIds = NULL;
        return -ENOMEM;
    }
    // Create a new activeCuGroups list
    device->activeCuGroups = malloc(cuEventSets->numSets * sizeof(CUpti_EventGroupSet**));
    if (!device->activeCuGroups)
    {
        free(device->activeEvents);
        device->activeEvents = NULL;
        free(grpEventIds);
        grpEventIds = NULL;
        return -ENOMEM;
    }


    // Run over eventset and store all information we need for start/stop/reads in NvmonActiveEvent_t
    for (unsigned j = 0; j < cuEventSets->numSets; j++)
    {
        size_t sizeofuint32t = sizeof(uint32_t);
        uint32_t numGEvents = 0, numGInstances = 0, numTotalGInstances = 0;
        CUpti_EventGroupSet* groupset = &cuEventSets->sets[j];

        for (unsigned k = 0; k < groupset->numEventGroups; k++)
        {
            uint32_t one = 1;
            CUpti_EventGroup group = groupset->eventGroups[k];
            // Get the number of events in the group
            CUPTI_CALL(cuptiEventGroupGetAttribute, group, CUPTI_EVENT_GROUP_ATTR_NUM_EVENTS, &sizeofuint32t, &numGEvents);
            // Get the CUPTI event IDs in the group
            CUPTI_CALL(cuptiEventGroupGetAttribute, group, CUPTI_EVENT_GROUP_ATTR_EVENTS, &grpEventIdsSize, grpEventIds);
            // If we don't set this, each event has only a single instance but we want to measure all instances
            CUPTI_CALL(cuptiEventGroupSetAttribute, group, CUPTI_EVENT_GROUP_ATTR_PROFILE_ALL_DOMAIN_INSTANCES, sizeof(uint32_t), &one);
            // Get instance count for a group
            CUPTI_CALL(cuptiEventGroupGetAttribute, group, CUPTI_EVENT_GROUP_ATTR_INSTANCE_COUNT, &sizeofuint32t, &numGInstances);

            for (unsigned l = 0; l < numGEvents; l++)
            {
                for (int m = 0; m <  eventSet->numberOfEvents; m++)
                {
                    if (eventSet->cuEventIDs[m] == grpEventIds[l])
                    {
                        CUpti_EventDomainID did = eventSet->nvEvents[m]->cuDomainId;
                        // Get total instance count for a group
                        CUPTI_CALL(cuptiDeviceGetEventDomainAttribute, device->cuDevice, did, CUPTI_EVENT_DOMAIN_ATTR_TOTAL_INSTANCE_COUNT, &sizeofuint32t, &numTotalGInstances);
                        device->activeEvents[m].eventId = eventSet->nvEvents[m]->eventId;
                        device->activeEvents[m].idxInSet = m;
                        device->activeEvents[m].groupId = eventSet->id;
                        device->activeEvents[m].cuEventId = eventSet->nvEvents[m]->cuEventId;
                        device->activeEvents[m].cuDomainId = did;
                        device->activeEvents[m].numTotalInstances = numTotalGInstances;
                        device->activeEvents[m].cuGroup = group;
                        device->activeEvents[m].cuGroupSet = groupset;
                        device->activeEvents[m].numInstances = numGInstances;
                        device->activeEvents[m].deviceId = device->deviceId;

                        int found = 0;
                        for (int i = 0; i < device->numActiveCuGroups; i++)
                        {
                            if (device->activeCuGroups[i] == groupset)
                            {
                                found = 1;
                            }
                        }
                        if (!found)
                        {
                            device->activeCuGroups[device->numActiveCuGroups] = groupset;
                            device->numActiveCuGroups++;
                        }
                        // Mark event as active. This is used to avoid measuring the same event on the same device twice
                        eventSet->nvEvents[m]->active = 1;
                        GPUDEBUG_PRINT(DEBUGLEV_INFO, "Setup event %s (%d) for GPU %d", eventSet->nvEvents[m]->name, device->activeEvents[m].cuEventId, device->deviceId);
                        device->numActiveEvents++;
                    }
                }
            }
        }
    }
    free(grpEventIds);
    if(popContext)
    {
        CU_CALL(cuCtxPopCurrent, &device->context);
    }
    return 0;
}

int nvmon_cupti_startCounters(NvmonDevice_t device)
{
    int j = 0;
    CUcontext curContext;
    int popContext = 0;
    uint64_t timestamp = 0;
    int oldDevId = -1;

    // Currently we are on which device?
    CUDA_CALL(cudaGetDevice, &oldDevId);
    // Take the timestamp, we assign it later for all devices
    CUPTI_CALL(cuptiGetTimestamp, &timestamp);
    // This is a workaround to (eventually create and) get the current context
    CUDA_CALL(cudaFree, NULL);
    CU_CALL(cuCtxGetCurrent, &curContext);


    //NvmonDevice_t device = &nvGroupSet->gpus[i];
    if (device->numActiveCuGroups == 0)
    {
        GPUDEBUG_PRINT(DEBUGLEV_DETAIL, "Skipping GPU%d it has no events in group %d", device->deviceId, nvGroupSet->activeGroup);
        return 0;
    }
    if (device->deviceId != oldDevId)
    {
        GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Change GPU device %d -> %d", oldDevId, device->deviceId);
        CUDA_CALL(cudaSetDevice, device->deviceId);
        CUDA_CALL(cudaGetDevice, &oldDevId);
    }
    CU_CALL(cuCtxGetCurrent, &curContext);
    device->timeStart = timestamp;
    device->timeRead = timestamp;

    // Are we in the proper context?
    popContext = check_nv_context(device, curContext);

    NvmonEventSet* nvEventSet = &device->nvEventSets[nvGroupSet->activeGroup];
    for (j = 0; j < nvEventSet->numberOfEvents; j++)
    {
        NvmonEventResult* res = &nvEventSet->results[j];
        res->startValue = 0.0;
        res->stopValue = 0.0;
        res->currentValue = 0.0;
        res->fullValue = 0.0;
        res->overflows = 0;
    }

    for (j = 0; j < device->numActiveCuGroups; j++)
    {
        GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Enable group %p on Dev %d", device->activeCuGroups[j], device->deviceId);
        CUPTI_CALL(cuptiEventGroupSetEnable, device->activeCuGroups[j]);
    }

    // If we added the device context to the stack, pop it again
    if (popContext)
    {
        GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Pop Context %p for device %d", device->context, device->deviceId);
        CU_CALL(cuCtxPopCurrent, &device->context);
    }
    return 0;
}


int nvmon_cupti_stopCounters(NvmonDevice_t device)
{
    (void)device; // unused

    int oldDevId = -1;
    uint64_t timestamp = 0;
    CUcontext curContext;

    CUDA_CALL(cudaGetDevice, &oldDevId);
    // Take the timestamp, we assign it later for all devices
    CUPTI_CALL(cuptiGetTimestamp, &timestamp);


    for (int i = 0; i < nvGroupSet->numberOfGPUs; i++)
    {
        int popContext = 0;
        unsigned maxTotalInstances = 0;
        size_t valuesSize = 0;
        NvmonDevice_t device = &nvGroupSet->gpus[i];
        if (device->deviceId != oldDevId)
        {
            GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Change GPU device %d -> %d", oldDevId, device->deviceId);
            CUDA_CALL(cudaSetDevice, device->deviceId);
            CUDA_CALL(cudaGetDevice, &oldDevId);
        }
        CU_CALL(cuCtxGetCurrent, &curContext);
        NvmonEventSet* nvEventSet = &device->nvEventSets[nvGroupSet->activeGroup];
        if (device->numActiveCuGroups == 0)
        {
            GPUDEBUG_PRINT(DEBUGLEV_DETAIL, "Skipping GPU%d it has no events in group %d", device->deviceId, nvGroupSet->activeGroup);
            continue;
        }
        device->timeStop = timestamp;

        // Are we in the proper context?
        popContext = check_nv_context(device, curContext);

        for (int j = 0; j < device->numActiveEvents; j++)
        {
            maxTotalInstances = MAX(maxTotalInstances, device->activeEvents[j].numTotalInstances);
        }
        uint64_t *tmpValues = (uint64_t *) malloc(maxTotalInstances * sizeof(uint64_t));

        for (int j = 0; j < device->numActiveEvents; j++)
        {
            NvmonActiveEvent_t event = &device->activeEvents[j];
            valuesSize = sizeof(uint64_t) * event->numTotalInstances;
            memset(tmpValues, 0, valuesSize);
            GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Read Grp %p Ev %u for device %d", event->cuGroup, event->cuEventId, device->deviceId);
            CUptiResult s = cuptiEventGroupReadEvent_ptr(event->cuGroup, CUPTI_EVENT_READ_FLAG_NONE, event->cuEventId, &valuesSize, tmpValues);
            if (s != CUPTI_SUCCESS) {
                ERROR_PRINT("cuptiEventGroupReadEvent failed: %d\n", s);
                free(tmpValues);
                return -EFAULT;
            }
            uint64_t valuesum = 0;
            for (unsigned k = 0; k < event->numInstances; k++)
            {
                valuesum += tmpValues[k];
            }
            NvmonEventResult* res = &nvEventSet->results[event->idxInSet];
            res->stopValue = (double)valuesum;
            res->lastValue = res->currentValue;
            res->fullValue += res->stopValue - res->startValue;
            res->lastValue += res->stopValue - res->currentValue;
            res->currentValue = (double)valuesum;
        }
        for (int j = 0; j < device->numActiveCuGroups; j++)
        {
            GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Disable group %p", device->activeCuGroups[j]);
            CUPTI_CALL(cuptiEventGroupSetDisable, device->activeCuGroups[j]);
        }
        // If we added the device context to the stack, pop it again
        if (popContext)
        {
            GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Pop Context %p for device %d", device->context, device->deviceId);
            CU_CALL(cuCtxPopCurrent, &device->context);
        }
    }
    return 0;
}


int nvmon_cupti_readCounters(NvmonDevice_t device)
{
    int oldDevId = -1;
    uint64_t timestamp = 0;
    CUcontext curContext;
    unsigned maxTotalInstances = 0;


    for (int j = 0; j < device->numActiveEvents; j++)
    {
        maxTotalInstances = MAX(maxTotalInstances, device->activeEvents[j].numTotalInstances);
    }


    // In this array we collect the instance values of an events (summed up later)
    size_t valuesSize = sizeof(uint64_t) * maxTotalInstances;
    uint64_t *tmpValues = (uint64_t *) malloc(valuesSize);
    if (!tmpValues)
    {
        ERROR_PRINT("Not enough memory to allocate space for instance values");
        return -ENOMEM;
    }

    // Currently we are on which device?
    CUDA_CALL(cudaGetDevice, &oldDevId);
    // Take the timestamp, we assign it later for all devices
    CUPTI_CALL(cuptiGetTimestamp, &timestamp);
/*    // This is a workaround to (eventually create and) get the current context*/
/*    CUDA_CALL(cudaFree, NULL);*/
    CU_CALL(cuCtxGetCurrent, &curContext);


    int popContext = 0;
    if (device->numActiveEvents == 0)
    {
        GPUDEBUG_PRINT(DEBUGLEV_DETAIL, "Skipping GPU%d it has no events in group %d", device->deviceId, nvGroupSet->activeGroup);
        return 0;
    }
    if (device->deviceId != oldDevId)
    {
        GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Change GPU device %d -> %d", oldDevId, device->deviceId);
        CUDA_CALL(cudaSetDevice, device->deviceId);
        CUDA_CALL(cudaGetDevice, &oldDevId);
    }
    CUDA_CALL(cudaFree, NULL);
    CU_CALL(cuCtxGetCurrent, &curContext);
    device->timeRead = timestamp;

    // Are we in the proper context?
    popContext = check_nv_context(device, curContext);
    // Synchronize devices. I'm not sure whether this is required as each
    // device measures it's own events
    CU_CALL(cuCtxSynchronize);
    NvmonEventSet* nvEventSet = &device->nvEventSets[nvGroupSet->activeGroup];

    for (int j = 0; j < device->numActiveEvents; j++)
    {
        NvmonActiveEvent_t event = &device->activeEvents[j];
        // Empty space for instance values
        valuesSize = sizeof(uint64_t) * event->numTotalInstances;
        memset(tmpValues, 0, valuesSize);
        // Read all instance values
        GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Read Grp %p Ev %u for device %d", event->cuGroup, event->cuEventId, device->deviceId);
        CUPTI_CALL(cuptiEventGroupReadEvent, event->cuGroup, CUPTI_EVENT_READ_FLAG_NONE, event->cuEventId, &valuesSize, tmpValues);
        // Sum all instance values
        uint64_t valuesum = 0;
        for (unsigned k = 0; k < event->numInstances; k++)
        {
            valuesum += tmpValues[k];
        }

        NvmonEventResult* res = &nvEventSet->results[event->idxInSet];
        res->lastValue = res->currentValue;
        res->currentValue = (double)valuesum;
        res->fullValue += res->currentValue - res->startValue;
        res->lastValue += res->currentValue - res->lastValue;
    }
    if (popContext)
    {
        GPUDEBUG_PRINT(DEBUGLEV_DEVELOP, "Pop Context %p for device %d", device->context, device->deviceId);
        CU_CALL(cuCtxPopCurrent, &device->context);
    }
    free(tmpValues);
    return 0;
}


NvmonFunctions nvmon_cupti_functions = {
    .freeDevice = nvmon_cupti_freeDevice,
    .createDevice = nvmon_cupti_createDevice,
    .getEventList = nvmon_cupti_getEventsOfGpu,
    .addEvents = nvmon_cupti_addEventSets,
    .setupCounters = nvmon_cupti_setupCounters,
    .startCounters = nvmon_cupti_startCounters,
    .stopCounters = nvmon_cupti_stopCounters,
    .readCounters = nvmon_cupti_readCounters,
};

#endif
