/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
 * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
 *                         reserved.
 * Copyright (c) 2011-2013 UT-Battelle, LLC. All rights reserved.
 * Copyright (c) 2017-2019 Intel, Inc.  All rights reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */

#include "btl_ugni_endpoint.h"
#include "btl_ugni_smsg.h"
#include "opal/mca/pmix/pmix-internal.h"

static void mca_btl_ugni_ep_construct(mca_btl_base_endpoint_t *ep)
{
    memset((char *) ep + sizeof(ep->super), 0, sizeof(*ep) - sizeof(ep->super));
    OBJ_CONSTRUCT(&ep->frag_wait_list, opal_list_t);
    OBJ_CONSTRUCT(&ep->lock, opal_recursive_mutex_t);
}

static void mca_btl_ugni_ep_destruct(mca_btl_base_endpoint_t *ep)
{
    OBJ_DESTRUCT(&ep->frag_wait_list);
    OBJ_DESTRUCT(&ep->lock);
    free(ep->remote_attr);
}

OBJ_CLASS_INSTANCE(mca_btl_ugni_endpoint_t, opal_list_item_t, mca_btl_ugni_ep_construct,
                   mca_btl_ugni_ep_destruct);

static int mca_btl_ugni_endpoint_get_modex(mca_btl_base_endpoint_t *ep)
{
    mca_btl_ugni_modex_t *modex;
    size_t msg_size;
    int rc;

    assert(NULL != ep && NULL != ep->peer_proc);

    /* Receive the modex */
    OPAL_MODEX_RECV(rc, &mca_btl_ugni_component.super.btl_version, &ep->peer_proc->proc_name,
                    (void **) &modex, &msg_size);
    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
        BTL_ERROR(("error receiving modex"));
        return rc;
    }

    ep->ep_rem_addr = modex->addr;
    ep->ep_rem_id = modex->id;

    BTL_VERBOSE(("received modex for ep %p. addr: %d, id: %d", (void *) ep, ep->ep_rem_addr,
                 ep->ep_rem_id));

    free(modex);

    return OPAL_SUCCESS;
}

int mca_btl_ugni_init_ep(mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_endpoint_t **ep,
                         mca_btl_ugni_module_t *btl, opal_proc_t *peer_proc)
{
    mca_btl_ugni_endpoint_t *endpoint;
    int rc;

    endpoint = OBJ_NEW(mca_btl_ugni_endpoint_t);
    assert(endpoint != NULL);

    endpoint->smsg_progressing = 0;
    endpoint->state = MCA_BTL_UGNI_EP_STATE_INIT;
    endpoint->peer_proc = peer_proc;

    /* get the modex info for this endpoint and setup a ugni endpoint. this call may lead
     * to re-entry through opal_progress(). */
    rc = mca_btl_ugni_endpoint_get_modex(endpoint);
    if (OPAL_SUCCESS != rc) {
        assert(0);
        return rc;
    }

    /* add this endpoint to the pointer array */
    endpoint->index = opal_pointer_array_add(&ugni_module->endpoints, endpoint);

    *ep = endpoint;

    return OPAL_SUCCESS;
}

void mca_btl_ugni_release_ep(mca_btl_ugni_endpoint_t *ep)
{
    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl(ep);
    int rc;

    opal_mutex_lock(&ep->lock);

    rc = mca_btl_ugni_ep_disconnect(ep, false);
    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
        BTL_VERBOSE(("btl/ugni error disconnecting endpoint"));
    }

    /* TODO -- Clear space at the end of the endpoint array */
    opal_pointer_array_set_item(&ugni_module->endpoints, ep->index, NULL);

    opal_mutex_unlock(&ep->lock);

    OBJ_RELEASE(ep);
}

static inline int mca_btl_ugni_ep_smsg_get_mbox(mca_btl_base_endpoint_t *ep)
{
    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl(ep);
    opal_free_list_item_t *mbox;

    assert(NULL == ep->mailbox);

    mbox = opal_free_list_get(&ugni_module->smsg_mboxes);
    if (OPAL_UNLIKELY(NULL == mbox)) {
        return OPAL_ERR_OUT_OF_RESOURCE;
    }

    ep->mailbox = (mca_btl_ugni_smsg_mbox_t *) mbox;
    ep->mailbox->attr.index = ep->index;

    /* per ugni spec we need to zero mailbox data before connecting */
    memset((char *) ep->mailbox->attr.smsg_attr.msg_buffer
               + ep->mailbox->attr.smsg_attr.mbox_offset,
           0, ep->mailbox->attr.smsg_attr.buff_size);
    return OPAL_SUCCESS;
}

static int mca_btl_ugni_ep_send_disconnect(mca_btl_base_endpoint_t *ep)
{
    int rc;

    do {
        rc = mca_btl_ugni_endpoint_smsg_send_wtag(ep, NULL, 0, NULL, 0, -1,
                                                  MCA_BTL_UGNI_TAG_DISCONNECT);
        if (OPAL_LIKELY(GNI_RC_NOT_DONE != rc)) {
            break;
        }

        /* most likely got here because we are out of credits. check the remote CQ to get credit
         * return */
        (void) mca_btl_ugni_progress_remote_smsg(mca_btl_ugni_ep_btl(ep));
    } while (1);

    return mca_btl_rc_ugni_to_opal(rc);
}

int mca_btl_ugni_ep_disconnect(mca_btl_base_endpoint_t *ep, bool send_disconnect)
{
    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl(ep);
    mca_btl_ugni_device_t *device;
    int rc;

    if (MCA_BTL_UGNI_EP_STATE_INIT == ep->state) {
        /* nothing to do */
        return OPAL_SUCCESS;
    }

    device = ep->smsg_ep_handle.device;

    while (device->dev_smsg_local_cq.active_operations) {
        /* ensure all sends are complete before removing and procs */
        rc = mca_btl_ugni_progress_local_smsg(ugni_module, device);
        if (OPAL_SUCCESS != rc) {
            break;
        }
    }

    if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state && send_disconnect) {
        rc = mca_btl_ugni_ep_send_disconnect(ep);
        if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
            BTL_VERBOSE(("could not send disconnect message to peer"));
        }

        /* wait for the disconnect messagse to go */
        do {
            /* ensure all sends are complete before removing and procs */
            rc = mca_btl_ugni_progress_local_smsg(ugni_module, device);
            if (OPAL_SUCCESS != rc) {
                break;
            }
        } while (device->dev_smsg_local_cq.active_operations);

        (void) opal_atomic_add_fetch_32(&ep->smsg_ep_handle.device->smsg_connections, -1);
    }

    mca_btl_ugni_device_lock(device);

    /* NTH: this call may not need the device lock. seems to work without it but
     * the lock is here to be safe. */
    (void) mca_btl_ugni_ep_handle_cleanup(&ep->smsg_ep_handle);

    mca_btl_ugni_device_unlock(device);

    if (ep->mailbox) {
        opal_free_list_return(&ugni_module->smsg_mboxes, ((opal_free_list_item_t *) ep->mailbox));
        ep->mailbox = NULL;
    }

    ep->state = MCA_BTL_UGNI_EP_STATE_INIT;

    return OPAL_SUCCESS;
}

static inline int mca_btl_ugni_ep_connect_start(mca_btl_base_endpoint_t *ep)
{
    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl(ep);
    mca_btl_ugni_device_t *device = ugni_module->devices;
    int rc;

    /* protect against re-entry from opal_progress */
    if (OPAL_UNLIKELY(MCA_BTL_UGNI_EP_STATE_CONNECTING == ep->state)) {
        return OPAL_ERR_RESOURCE_BUSY;
    }

    ep->state = MCA_BTL_UGNI_EP_STATE_CONNECTING;

    BTL_VERBOSE(("initiating connection to remote peer with address: %u id: %u proc: %p",
                 ep->ep_rem_addr, ep->ep_rem_id, (void *) ep->peer_proc));

    /* bind endpoint to remote address */
    /* we bind two endpoints to separate out local smsg completion and local fma completion */
    mca_btl_ugni_device_lock(device);
    rc = mca_btl_ugni_ep_handle_init(ep, device->dev_smsg_local_cq.gni_handle, device,
                                     &ep->smsg_ep_handle);
    mca_btl_ugni_device_unlock(device);
    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
        return rc;
    }

    /* build connection data */
    rc = mca_btl_ugni_ep_smsg_get_mbox(ep);
    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
        return rc;
    }

    ep->remote_attr = calloc(1, sizeof(*ep->remote_attr));
    if (OPAL_UNLIKELY(NULL == ep->remote_attr)) {
        return OPAL_ERR_OUT_OF_RESOURCE;
    }

    BTL_VERBOSE(("btl/ugni connection to remote peer initiated"));

    return OPAL_SUCCESS;
}

static inline int mca_btl_ugni_ep_connect_finish(mca_btl_base_endpoint_t *ep)
{
    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl(ep);
    gni_return_t grc;
    int rc;

    BTL_VERBOSE(
        ("finishing connection. remote attributes: msg_type = %d, msg_buffer = %p, buff_size = %d, "
         "mem_hndl = {qword1 = %" PRIu64 ", qword2 = %" PRIu64 "}, mbox = %d, mbox_maxcredit = %d, "
         "msg_maxsize = %d",
         ep->remote_attr->smsg_attr.msg_type, ep->remote_attr->smsg_attr.msg_buffer,
         ep->remote_attr->smsg_attr.buff_size, ep->remote_attr->smsg_attr.mem_hndl.qword1,
         ep->remote_attr->smsg_attr.mem_hndl.qword2, ep->remote_attr->smsg_attr.mbox_offset,
         ep->remote_attr->smsg_attr.mbox_maxcredit, ep->remote_attr->smsg_attr.msg_maxsize));

    BTL_VERBOSE(
        ("finishing connection. local attributes: msg_type = %d, msg_buffer = %p, buff_size = %d, "
         "mem_hndl = {qword1 = %" PRIu64 ", qword2 = %" PRIu64 "}, mbox = %d, mbox_maxcredit = %d, "
         "msg_maxsize = %d",
         ep->mailbox->attr.smsg_attr.msg_type, ep->mailbox->attr.smsg_attr.msg_buffer,
         ep->mailbox->attr.smsg_attr.buff_size, ep->mailbox->attr.smsg_attr.mem_hndl.qword1,
         ep->mailbox->attr.smsg_attr.mem_hndl.qword2, ep->mailbox->attr.smsg_attr.mbox_offset,
         ep->mailbox->attr.smsg_attr.mbox_maxcredit, ep->mailbox->attr.smsg_attr.msg_maxsize));

    grc = GNI_SmsgInit(ep->smsg_ep_handle.gni_handle, &ep->mailbox->attr.smsg_attr,
                       &ep->remote_attr->smsg_attr);
    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc)) {
        BTL_ERROR(("error initializing SMSG protocol. rc = %d", grc));

        return mca_btl_rc_ugni_to_opal(grc);
    }

    /* set the local event data to the local index and the remote event data to my
     * index on the remote peer. This makes lookup of endpoints on completion take
     * a single lookup in the endpoints array. we will not be able to change the
     * remote peer's index in the endpoint's array after this point. */
    GNI_EpSetEventData(ep->smsg_ep_handle.gni_handle, ep->index, ep->remote_attr->index);

    ep->rmt_irq_mem_hndl = ep->remote_attr->rmt_irq_mem_hndl;
    ep->state = MCA_BTL_UGNI_EP_STATE_CONNECTED;
    (void) opal_atomic_add_fetch_32(&ep->smsg_ep_handle.device->smsg_connections, 1);

    /* send all pending messages */
    BTL_VERBOSE(("endpoint connected. posting %u sends",
                 (unsigned int) opal_list_get_size(&ep->frag_wait_list)));

    rc = mca_btl_ugni_progress_send_wait_list(ep);
    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
        OPAL_THREAD_LOCK(&ugni_module->ep_wait_list_lock);
        if (false == ep->wait_listed) {
            opal_list_append(&ugni_module->ep_wait_list, &ep->super);
            ep->wait_listed = true;
        }
        OPAL_THREAD_UNLOCK(&ugni_module->ep_wait_list_lock);
    }

    free(ep->remote_attr);
    ep->remote_attr = NULL;

    return OPAL_SUCCESS;
}

static int mca_btl_ugni_directed_ep_post(mca_btl_base_endpoint_t *ep)
{
    mca_btl_ugni_module_t *ugni_module = mca_btl_ugni_ep_btl(ep);
    gni_return_t rc;

    BTL_VERBOSE(
        ("posting directed datagram to remote id: %d for endpoint %p", ep->ep_rem_id, (void *) ep));
    /* the irq cq is associated with only the first device */
    ep->mailbox->attr.rmt_irq_mem_hndl = ugni_module->devices->smsg_irq_mhndl;

    rc = GNI_EpPostDataWId(ep->smsg_ep_handle.gni_handle, &ep->mailbox->attr,
                           sizeof(ep->mailbox->attr), ep->remote_attr, sizeof(*ep->remote_attr),
                           MCA_BTL_UGNI_CONNECT_DIRECTED_ID | ep->index);
    if (OPAL_LIKELY(GNI_RC_SUCCESS == rc)) {
        (void) opal_atomic_add_fetch_32(&ugni_module->active_datagrams, 1);
    }

    return mca_btl_rc_ugni_to_opal(rc);
}

int mca_btl_ugni_wildcard_ep_post(mca_btl_ugni_module_t *ugni_module)
{
    gni_return_t rc;

    BTL_VERBOSE(("posting wildcard datagram"));

    memset(&ugni_module->wc_local_attr, 0, sizeof(ugni_module->wc_local_attr));
    memset(&ugni_module->wc_remote_attr, 0, sizeof(ugni_module->wc_remote_attr));
    rc = GNI_EpPostDataWId(ugni_module->wildcard_ep, &ugni_module->wc_local_attr,
                           sizeof(ugni_module->wc_local_attr), &ugni_module->wc_remote_attr,
                           sizeof(ugni_module->wc_remote_attr), MCA_BTL_UGNI_CONNECT_WILDCARD_ID);

    return mca_btl_rc_ugni_to_opal(rc);
}

int mca_btl_ugni_ep_connect_progress(mca_btl_base_endpoint_t *ep)
{
    int rc;

    BTL_VERBOSE(("progressing connection for endpoint %p with state %d", (void *) ep, ep->state));

    if (MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state) {
        return OPAL_SUCCESS;
    }

    if (MCA_BTL_UGNI_EP_STATE_INIT == ep->state) {
        rc = mca_btl_ugni_ep_connect_start(ep);
        if (OPAL_SUCCESS != rc) {
            return rc;
        }
    }

    BTL_VERBOSE(("ep->remote_attr->smsg_attr = {.msg_type = %d, .msg_buffer = %p}",
                 ep->remote_attr->smsg_attr.msg_type,
                 (void *) ep->remote_attr->smsg_attr.msg_buffer));

    if (GNI_SMSG_TYPE_INVALID == ep->remote_attr->smsg_attr.msg_type) {
        /* use datagram to exchange connection information with the remote peer */
        if (!ep->dg_posted) {
            rc = mca_btl_ugni_directed_ep_post(ep);
            if (OPAL_SUCCESS == rc) {
                ep->dg_posted = true;
                rc = OPAL_ERR_RESOURCE_BUSY;
            }

            return rc;
        }

        return OPAL_SUCCESS;
    }

    return mca_btl_ugni_ep_connect_finish(ep);
}

int mca_btl_ugni_ep_handle_init(mca_btl_ugni_endpoint_t *ep, gni_cq_handle_t cq,
                                mca_btl_ugni_device_t *device,
                                mca_btl_ugni_endpoint_handle_t *ep_handle)
{
    gni_return_t grc;

    ep_handle->device = device;

    /* create a uGNI endpoint handle and bind it to the remote peer */
    grc = GNI_EpCreate(device->dev_handle, cq, &ep_handle->gni_handle);
    if (OPAL_LIKELY(GNI_RC_SUCCESS == grc)) {
        grc = GNI_EpBind(ep_handle->gni_handle, ep->ep_rem_addr, ep->ep_rem_id);
    }

    return mca_btl_rc_ugni_to_opal(grc);
}

int mca_btl_ugni_ep_handle_cleanup(mca_btl_ugni_endpoint_handle_t *ep_handle)
{
    int rc;

    if (0 == ep_handle->gni_handle) {
        return OPAL_SUCCESS;
    }

    /* TODO: need to fix, may be outstanding tx's, etc. */
    rc = GNI_EpUnbind(ep_handle->gni_handle);
    if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
        /* should warn */
    } else {
        (void) GNI_EpDestroy(ep_handle->gni_handle);
    }

    ep_handle->gni_handle = 0;

    return OPAL_SUCCESS;
}