From: Konstantin Taranov Date: Tue, 6 Aug 2024 10:02:24 +0200 Subject: providers/mana: Process Error cqes Add processing of error CQEs to mana RC connections. Signed-off-by: Konstantin Taranov Origin: upstream, https://github.com/linux-rdma/rdma-core/pull/1556 --- providers/mana/cq.c | 223 ++++++++++++++++++++++++++++++++++-------- providers/mana/gdma.h | 160 ++++++++++++++++++++---------- providers/mana/mana.c | 31 +++++- providers/mana/mana.h | 18 ++-- providers/mana/qp.c | 172 ++++++++++++++++++++------------ providers/mana/shadow_queue.h | 28 +++++- providers/mana/wr.c | 18 +++- 7 files changed, 477 insertions(+), 173 deletions(-) diff --git a/providers/mana/cq.c b/providers/mana/cq.c index bb9a1b0..dd8c54c 100644 --- a/providers/mana/cq.c +++ b/providers/mana/cq.c @@ -68,7 +68,6 @@ struct ibv_cq *mana_create_cq(struct ibv_context *context, int cqe, cq->cqe = cqe; // to preserve old behaviour for DPDK cq->head = INITIALIZED_OWNER_BIT(ilog32(cq->cqe) - 1); cq->last_armed_head = cq->head - 1; - cq->ready_wcs = 0; cmd_drv = &cmd.drv_payload; cmd_drv->buf_addr = (uintptr_t)cq->buf; @@ -146,25 +145,26 @@ int mana_arm_cq(struct ibv_cq *ibcq, int solicited) return 0; } -static inline uint32_t handle_rc_requester_cqe(struct mana_qp *qp, struct gdma_cqe *cqe) +static inline bool get_next_signal_psn(struct mana_qp *qp, uint32_t *psn) { - struct mana_gdma_queue *recv_queue = &qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER]; - struct mana_gdma_queue *send_queue = &qp->rc_qp.queues[USER_RC_SEND_QUEUE_REQUESTER]; - uint32_t syndrome = cqe->rdma_cqe.rc_armed_completion.syndrome; - uint32_t psn = cqe->rdma_cqe.rc_armed_completion.psn; - struct rc_sq_shadow_wqe *shadow_wqe; - uint32_t wcs = 0; + struct rc_sq_shadow_wqe *shadow_wqe = + (struct rc_sq_shadow_wqe *)shadow_queue_get_next_to_signal(&qp->shadow_sq); - if (!IB_IS_ACK(syndrome)) - return 0; + if (!shadow_wqe) + return false; - if (!PSN_GT(psn, qp->rc_qp.sq_highest_completed_psn)) - return 0; + *psn = shadow_wqe->end_psn; + return true; +} - qp->rc_qp.sq_highest_completed_psn = psn; +static inline void advance_send_completions(struct mana_qp *qp, uint32_t psn) +{ + struct mana_gdma_queue *recv_queue = &qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER]; + struct mana_gdma_queue *send_queue = &qp->rc_qp.queues[USER_RC_SEND_QUEUE_REQUESTER]; + struct rc_sq_shadow_wqe *shadow_wqe; if (!PSN_LT(psn, qp->rc_qp.sq_psn)) - return 0; + return; while ((shadow_wqe = (struct rc_sq_shadow_wqe *) shadow_queue_get_next_to_complete(&qp->shadow_sq)) != NULL) { @@ -182,34 +182,41 @@ static inline uint32_t handle_rc_requester_cqe(struct mana_qp *qp, struct gdma_c mana_ib_update_shared_mem_left_offset(qp, offset & GDMA_QUEUE_OFFSET_MASK); shadow_queue_advance_next_to_complete(&qp->shadow_sq); - if (shadow_wqe->header.flags != MANA_NO_SIGNAL_WC) - wcs++; } +} - uint32_t prev_psn = PSN_DEC(qp->rc_qp.sq_psn); +static inline void handle_rc_requester_cqe(struct mana_qp *qp, struct gdma_cqe *cqe) +{ + struct mana_gdma_queue *recv_queue = &qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER]; + uint32_t syndrome = cqe->rdma_cqe.rc_armed_completion.syndrome; + uint32_t psn = cqe->rdma_cqe.rc_armed_completion.psn; + uint32_t arm_psn; - if (qp->rc_qp.sq_highest_completed_psn == prev_psn) - gdma_arm_normal_cqe(recv_queue, qp->rc_qp.sq_psn); - else - gdma_arm_normal_cqe(recv_queue, prev_psn); + if (!IB_IS_ACK(syndrome)) + return; - return wcs; + advance_send_completions(qp, psn); + + if (!get_next_signal_psn(qp, &arm_psn)) + arm_psn = PSN_INC(psn); + + gdma_arm_normal_cqe(recv_queue, arm_psn); } -static inline uint32_t handle_rc_responder_cqe(struct mana_qp *qp, struct gdma_cqe *cqe) +static inline void handle_rc_responder_cqe(struct mana_qp *qp, struct gdma_cqe *cqe) { struct mana_gdma_queue *recv_queue = &qp->rc_qp.queues[USER_RC_RECV_QUEUE_RESPONDER]; struct rc_rq_shadow_wqe *shadow_wqe; shadow_wqe = (struct rc_rq_shadow_wqe *)shadow_queue_get_next_to_complete(&qp->shadow_rq); if (!shadow_wqe) - return 0; + return; uint32_t offset_cqe = cqe->rdma_cqe.rc_recv.rx_wqe_offset / GDMA_WQE_ALIGNMENT_UNIT_SIZE; uint32_t offset_wqe = shadow_wqe->header.unmasked_queue_offset & GDMA_QUEUE_OFFSET_MASK; if (offset_cqe != offset_wqe) - return 0; + return; shadow_wqe->byte_len = cqe->rdma_cqe.rc_recv.msg_len; shadow_wqe->imm_or_rkey = cqe->rdma_cqe.rc_recv.imm_data; @@ -232,24 +239,59 @@ static inline uint32_t handle_rc_responder_cqe(struct mana_qp *qp, struct gdma_c recv_queue->cons_idx &= GDMA_QUEUE_OFFSET_MASK; shadow_queue_advance_next_to_complete(&qp->shadow_rq); - return 1; } -static inline uint32_t mana_handle_cqe(struct mana_context *ctx, struct gdma_cqe *cqe) +static inline bool error_cqe_is_send(struct mana_qp *qp, struct gdma_cqe *cqe) { - struct mana_qp *qp; + if (cqe->is_sq && + qp->rc_qp.queues[USER_RC_SEND_QUEUE_REQUESTER].id == cqe->wqid) + return true; + if (!cqe->is_sq && + qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER].id == cqe->wqid) + return true; + + return false; +} + +static inline uint32_t error_cqe_get_psn(struct gdma_cqe *cqe) +{ + return cqe->rdma_cqe.error.psn; +} + +static inline void handle_rc_error_cqe(struct mana_qp *qp, struct gdma_cqe *cqe) +{ + uint32_t vendor_error = cqe->rdma_cqe.error.vendor_error; + bool is_send_error = error_cqe_is_send(qp, cqe); + uint32_t psn = error_cqe_get_psn(cqe); + struct shadow_queue *queue_with_error; + struct shadow_wqe_header *shadow_wqe; + + mana_qp_move_flush_err(&qp->ibqp.qp); + advance_send_completions(qp, psn); + + queue_with_error = is_send_error ? &qp->shadow_sq : &qp->shadow_rq; + shadow_wqe = shadow_queue_get_next_to_complete(queue_with_error); + + if (shadow_wqe) { + shadow_wqe->flags = 0; + shadow_wqe->vendor_error = vendor_error; + shadow_queue_advance_next_to_complete(queue_with_error); + } +} - if (cqe->is_sq) // impossible for rc - return 0; +static inline void mana_handle_cqe(struct mana_context *ctx, struct gdma_cqe *cqe) +{ + struct mana_qp *qp = mana_get_qp(ctx, cqe->wqid, cqe->is_sq); - qp = mana_get_qp_from_rq(ctx, cqe->wqid); if (!qp) - return 0; + return; - if (cqe->rdma_cqe.cqe_type == CQE_TYPE_ARMED_CMPL) - return handle_rc_requester_cqe(qp, cqe); + if (cqe->rdma_cqe.cqe_type == CQE_TYPE_ERROR) + handle_rc_error_cqe(qp, cqe); + else if (cqe->rdma_cqe.cqe_type == CQE_TYPE_ARMED_CMPL) + handle_rc_requester_cqe(qp, cqe); else - return handle_rc_responder_cqe(qp, cqe); + handle_rc_responder_cqe(qp, cqe); } static inline int gdma_read_cqe(struct mana_cq *cq, struct gdma_cqe *cqe) @@ -275,15 +317,86 @@ static inline int gdma_read_cqe(struct mana_cq *cq, struct gdma_cqe *cqe) return 1; } +static enum ibv_wc_status vendor_error_to_wc_error(uint32_t vendor_error) +{ + switch (vendor_error) { + case VENDOR_ERR_OK: + return IBV_WC_SUCCESS; + case VENDOR_ERR_RX_PKT_LEN: + case VENDOR_ERR_RX_MSG_LEN_OVFL: + case VENDOR_ERR_RX_READRESP_LEN_MISMATCH: + return IBV_WC_LOC_LEN_ERR; + case VENDOR_ERR_TX_GDMA_CORRUPTED_WQE: + case VENDOR_ERR_TX_PCIE_WQE: + case VENDOR_ERR_TX_PCIE_MSG: + case VENDOR_ERR_RX_MALFORMED_WQE: + case VENDOR_ERR_TX_GDMA_INVALID_STATE: + case VENDOR_ERR_TX_MISBEHAVING_CLIENT: + case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE: + case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD: + case VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED: + case VENDOR_ERR_TX_RDMA_WQE_LEN_ERR: + case VENDOR_ERR_TX_RDMA_MTU_ERR: + return IBV_WC_LOC_QP_OP_ERR; + case VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION: + case VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE: + case VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR: + case VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION: + case VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE: + case VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR: + case VENDOR_ERR_TX_RDMA_ATB_CMD_MISS: + case VENDOR_ERR_TX_RDMA_ATB_CMD_IDX_ERROR: + case VENDOR_ERR_TX_RDMA_ATB_CMD_TAG_MISMATCH_ERROR: + case VENDOR_ERR_TX_RDMA_ATB_CMD_PDID_MISMATCH_ERROR: + case VENDOR_ERR_TX_RDMA_ATB_CMD_AR_ERROR: + case VENDOR_ERR_TX_RDMA_ATB_CMD_PT_OVF: + case VENDOR_ERR_TX_RDMA_ATB_CMD_PT_LENGHT_MISMATCH: + case VENDOR_ERR_TX_RDMA_ATB_CMD_ILLEGAL_CMD: + return IBV_WC_LOC_PROT_ERR; + case VENDOR_ERR_RX_ATB_SGE_MISSCONFIG: + case VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT: + case VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE: + case VENDOR_ERR_RX_GFID: + return IBV_WC_LOC_ACCESS_ERR; + case VENDOR_ERR_RX_OP_REQ: + return IBV_WC_REM_INV_REQ_ERR; + case VENDOR_ERR_RX_ATB_RKEY_MISCONFIG_ERR: + case VENDOR_ERR_RX_ATB_RKEY_ADDR_RIGHT: + case VENDOR_ERR_RX_ATB_RKEY_ADDR_RANGE: + case VENDOR_ERR_RX_REMOTE_ACCESS_NAK: + return IBV_WC_REM_ACCESS_ERR; + case VENDOR_ERR_RX_INVALID_REQ_NAK: + case VENDOR_ERR_RX_REMOTE_OP_ERR_NAK: + return IBV_WC_REM_OP_ERR; + case VENDOR_ERR_RX_MISBEHAVING_CLIENT: + case VENDOR_ERR_RX_CLIENT_ID: + case VENDOR_ERR_RX_PCIE: + case VENDOR_ERR_RX_NO_AVAIL_WQE: + case VENDOR_ERR_RX_ATB_WQE_MISCONFIG: + case VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT: + case VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE: + case VENDOR_ERR_TX_RDMA_INVALID_STATE: + case VENDOR_ERR_TX_RDMA_INVALID_NPT: + case VENDOR_ERR_TX_RDMA_INVALID_SGID: + case VENDOR_ERR_TX_RDMA_VFID_MISMATCH: + return IBV_WC_FATAL_ERR; + case VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE: + case VENDOR_ERR_SW_FLUSHED: + return IBV_WC_WR_FLUSH_ERR; + default: + return IBV_WC_GENERAL_ERR; + } +} + static void fill_verbs_from_shadow_wqe(struct mana_qp *qp, struct ibv_wc *wc, const struct shadow_wqe_header *shadow_wqe) { const struct rc_rq_shadow_wqe *rc_wqe = (const struct rc_rq_shadow_wqe *)shadow_wqe; wc->wr_id = shadow_wqe->wr_id; - wc->status = IBV_WC_SUCCESS; + wc->status = vendor_error_to_wc_error(shadow_wqe->vendor_error); wc->opcode = shadow_wqe->opcode; - wc->vendor_err = 0; + wc->vendor_err = shadow_wqe->vendor_error; wc->wc_flags = shadow_wqe->flags; wc->qp_num = qp->ibqp.qp.qp_num; wc->pkey_index = 0; @@ -332,17 +445,44 @@ out: return wc_index; } +static void mana_flush_completions(struct mana_cq *cq) +{ + struct shadow_wqe_header *shadow_wqe; + struct mana_qp *qp; + + list_for_each(&cq->send_qp_list, qp, send_cq_node) { + if (qp->ibqp.qp.state != IBV_QPS_ERR) + continue; + while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq)) + != NULL) { + shadow_wqe->vendor_error = VENDOR_ERR_SW_FLUSHED; + shadow_wqe->flags = 0; + shadow_queue_advance_next_to_complete(&qp->shadow_sq); + } + } + + list_for_each(&cq->recv_qp_list, qp, recv_cq_node) { + if (qp->ibqp.qp.state != IBV_QPS_ERR) + continue; + while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq)) + != NULL) { + shadow_wqe->vendor_error = VENDOR_ERR_SW_FLUSHED; + shadow_queue_advance_next_to_complete(&qp->shadow_rq); + } + } +} + int mana_poll_cq(struct ibv_cq *ibcq, int nwc, struct ibv_wc *wc) { struct mana_cq *cq = container_of(ibcq, struct mana_cq, ibcq); struct mana_context *ctx = to_mctx(ibcq->context); struct gdma_cqe gdma_cqe; int num_polled = 0; - int ret; + int ret, i; pthread_spin_lock(&cq->lock); - while (cq->ready_wcs < nwc) { + for (i = 0; i < nwc; i++) { ret = gdma_read_cqe(cq, &gdma_cqe); if (ret < 0) { num_polled = -1; @@ -350,11 +490,12 @@ int mana_poll_cq(struct ibv_cq *ibcq, int nwc, struct ibv_wc *wc) } if (ret == 0) break; - cq->ready_wcs += mana_handle_cqe(ctx, &gdma_cqe); + mana_handle_cqe(ctx, &gdma_cqe); } + mana_flush_completions(cq); num_polled = mana_process_completions(cq, nwc, wc); - cq->ready_wcs -= num_polled; + out: pthread_spin_unlock(&cq->lock); diff --git a/providers/mana/gdma.h b/providers/mana/gdma.h index 7c69cff..fd02c23 100644 --- a/providers/mana/gdma.h +++ b/providers/mana/gdma.h @@ -164,65 +164,127 @@ enum { CQE_TYPE_RC_WRITE_IMM = 6, CQE_TYPE_ARMED_CMPL = 7, CQE_TYPE_LWR = 8, - CQE_TYPE_RC_FENCE = 9, - CQE_TYPE_MAX + CQE_TYPE_ERROR = 34, }; /* HW DATA */ -struct mana_rdma_cqe { - uint32_t cqe_type : 8; - uint32_t vendor_error : 8; - uint32_t reserved1 : 16; - union { - uint32_t data[GDMA_COMP_DATA_SIZE / sizeof(uint32_t) - 4]; - struct { - uint32_t msg_len; - uint32_t psn : 24; - uint32_t reserved : 8; - uint32_t imm_data; - uint32_t rx_wqe_offset; - } rc_recv; - struct { - uint32_t sge_offset : 5; - uint32_t rx_wqe_offset : 27; - uint32_t sge_byte_offset; - } ud_send; - struct { - uint32_t msg_len; - uint32_t src_qpn : 24; - uint32_t reserved : 8; - uint32_t imm_data; - uint32_t rx_wqe_offset; - } ud_recv; - - struct { - uint32_t reserved1; - uint32_t psn : 24; - uint32_t reserved2 : 8; - uint32_t imm_data; - uint32_t rx_wqe_offset; - } rc_write_with_imm; - struct { - uint32_t msn : 24; - uint32_t syndrome : 8; - uint32_t psn : 24; - uint32_t reserved : 8; - uint32_t read_resp_psn : 24; - } rc_armed_completion; +union mana_rdma_cqe { + struct { + uint8_t cqe_type; + uint8_t data[GDMA_COMP_DATA_SIZE - 1]; }; - uint32_t timestamp_hi; - uint32_t timestamp_lo; - uint32_t reserved3; + struct { + uint32_t cqe_type : 8; + uint32_t reserved1 : 24; + uint32_t msg_len; + uint32_t psn : 24; + uint32_t reserved2 : 8; + uint32_t imm_data; + uint32_t rx_wqe_offset; + } rc_recv; + struct { + uint32_t cqe_type : 8; + uint32_t vendor_error : 9; + uint32_t reserved1 : 15; + uint32_t sge_offset : 5; + uint32_t tx_wqe_offset : 27; + } ud_send; + struct { + uint32_t cqe_type : 8; + uint32_t reserved1 : 24; + uint32_t msg_len; + uint32_t src_qpn : 24; + uint32_t reserved2 : 8; + uint32_t imm_data; + uint32_t rx_wqe_offset; + } ud_recv; + struct { + uint32_t cqe_type : 8; + uint32_t vendor_error : 10; + uint32_t reserved1 : 14; + uint32_t msn : 24; + uint32_t syndrome : 8; + uint32_t psn : 24; + uint32_t opcode : 8; + uint32_t rsp_msn : 24; + uint32_t reserved2 : 8; + uint32_t rsp_psn : 24; + uint32_t reserved3 : 8; + } error; + struct { + uint32_t cqe_type : 8; + uint32_t reserved1 : 24; + uint32_t msn : 24; + uint32_t syndrome : 8; + uint32_t psn : 24; + uint32_t reserved2 : 8; + } rc_armed_completion; }; /* HW DATA */ +static_assert(sizeof(union mana_rdma_cqe) == GDMA_COMP_DATA_SIZE, "bad size"); struct gdma_cqe { - union { - uint8_t data[GDMA_COMP_DATA_SIZE]; - struct mana_rdma_cqe rdma_cqe; - }; + union mana_rdma_cqe rdma_cqe; uint32_t wqid : 24; uint32_t is_sq : 1; uint32_t reserved : 4; uint32_t owner_bits : 3; }; /* HW DATA */ +enum mana_error_code { + VENDOR_ERR_OK = 0x0, + VENDOR_ERR_RX_OP_REQ = 0x03, + VENDOR_ERR_RX_PKT_LEN = 0x05, + VENDOR_ERR_RX_ATB_RKEY_MISCONFIG_ERR = 0x43, + VENDOR_ERR_RX_ATB_RKEY_ADDR_RIGHT = 0x83, + VENDOR_ERR_RX_ATB_RKEY_ADDR_RANGE = 0xc3, + VENDOR_ERR_RX_MSG_LEN_OVFL = 0x102, + VENDOR_ERR_RX_MISBEHAVING_CLIENT = 0x108, + VENDOR_ERR_RX_MALFORMED_WQE = 0x109, + VENDOR_ERR_RX_CLIENT_ID = 0x10a, + VENDOR_ERR_RX_GFID = 0x10b, + VENDOR_ERR_RX_READRESP_LEN_MISMATCH = 0x10f, + VENDOR_ERR_RX_PCIE = 0x10c, + VENDOR_ERR_RX_NO_AVAIL_WQE = 0x111, + VENDOR_ERR_RX_ATB_SGE_MISSCONFIG = 0x143, + VENDOR_ERR_RX_ATB_WQE_MISCONFIG = 0x145, + VENDOR_ERR_RX_INVALID_REQ_NAK = 0x161, + VENDOR_ERR_RX_REMOTE_ACCESS_NAK = 0x162, + VENDOR_ERR_RX_REMOTE_OP_ERR_NAK = 0x163, + VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT = 0x183, + VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT = 0x185, + VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE = 0x1c3, + VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE = 0x1c5, + VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE = 0x1c7, + VENDOR_ERR_TX_GDMA_CORRUPTED_WQE = 0x201, + VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION = 0x202, + VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE = 0x203, + VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR = 0x204, + VENDOR_ERR_TX_PCIE_WQE = 0x205, + VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION = 0x206, + VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE = 0x207, + VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR = 0x208, + VENDOR_ERR_TX_PCIE_MSG = 0x209, + VENDOR_ERR_TX_GDMA_INVALID_STATE = 0x20a, + VENDOR_ERR_TX_MISBEHAVING_CLIENT = 0x20b, + VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE = 0x210, + VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD = 0x211, + VENDOR_ERR_TX_RDMA_INVALID_STATE = 0x212, + VENDOR_ERR_TX_RDMA_INVALID_NPT = 0x213, + VENDOR_ERR_TX_RDMA_INVALID_SGID = 0x214, + VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED = 0x215, + VENDOR_ERR_TX_RDMA_WQE_LEN_ERR = 0x216, + VENDOR_ERR_TX_RDMA_MTU_ERR = 0x217, + VENDOR_ERR_TX_RDMA_VFID_MISMATCH = 0x218, + VENDOR_ERR_TX_RDMA_ATB_CMD_MISS = 0x220, + VENDOR_ERR_TX_RDMA_ATB_CMD_IDX_ERROR = 0x221, + VENDOR_ERR_TX_RDMA_ATB_CMD_TAG_MISMATCH_ERROR = 0x222, + VENDOR_ERR_TX_RDMA_ATB_CMD_PDID_MISMATCH_ERROR = 0x223, + VENDOR_ERR_TX_RDMA_ATB_CMD_AR_ERROR = 0x224, + VENDOR_ERR_TX_RDMA_ATB_CMD_PT_OVF = 0x225, + VENDOR_ERR_TX_RDMA_ATB_CMD_PT_LENGHT_MISMATCH = 0x226, + VENDOR_ERR_TX_RDMA_ATB_CMD_ILLEGAL_CMD = 0x227, + VENDOR_ERR_HW_MAX = 0x3ff, + /* SW vendor errors */ + VENDOR_ERR_SW_FLUSHED = 0xfff, +}; + #endif //_GDMA_H_ diff --git a/providers/mana/mana.c b/providers/mana/mana.c index 22e5ff9..581725f 100644 --- a/providers/mana/mana.c +++ b/providers/mana/mana.c @@ -204,8 +204,10 @@ static void mana_free_context(struct ibv_context *ibctx) int i; for (i = 0; i < MANA_QP_TABLE_SIZE; ++i) { - if (context->qp_table[i].refcnt) - free(context->qp_table[i].table); + if (context->qp_stable[i].refcnt) + free(context->qp_stable[i].table); + if (context->qp_rtable[i].refcnt) + free(context->qp_rtable[i].table); } pthread_mutex_destroy(&context->qp_table_mutex); @@ -214,8 +216,27 @@ static void mana_free_context(struct ibv_context *ibctx) free(context); } +static void mana_async_event(struct ibv_context *context, + struct ibv_async_event *event) +{ + struct ibv_qp *ibvqp; + + switch (event->event_type) { + case IBV_EVENT_QP_FATAL: + case IBV_EVENT_QP_REQ_ERR: + case IBV_EVENT_QP_ACCESS_ERR: + case IBV_EVENT_PATH_MIG_ERR: + ibvqp = event->element.qp; + mana_qp_move_flush_err(ibvqp); + break; + default: + break; + } +} + static const struct verbs_context_ops mana_ctx_ops = { .alloc_pd = mana_alloc_pd, + .async_event = mana_async_event, .alloc_parent_domain = mana_alloc_parent_domain, .create_cq = mana_create_cq, .create_qp = mana_create_qp, @@ -283,8 +304,10 @@ static struct verbs_context *mana_alloc_context(struct ibv_device *ibdev, verbs_set_ops(&context->ibv_ctx, &mana_ctx_ops); pthread_mutex_init(&context->qp_table_mutex, NULL); - for (i = 0; i < MANA_QP_TABLE_SIZE; ++i) - context->qp_table[i].refcnt = 0; + for (i = 0; i < MANA_QP_TABLE_SIZE; ++i) { + context->qp_stable[i].refcnt = 0; + context->qp_rtable[i].refcnt = 0; + } context->db_page = mmap(NULL, DOORBELL_PAGE_SIZE, PROT_WRITE, MAP_SHARED, context->ibv_ctx.context.cmd_fd, 0); diff --git a/providers/mana/mana.h b/providers/mana/mana.h index 29350d7..7d31cae 100644 --- a/providers/mana/mana.h +++ b/providers/mana/mana.h @@ -71,12 +71,15 @@ static inline uint32_t get_large_wqe_size(uint32_t sge) return align(wqe_size, GDMA_WQE_ALIGNMENT_UNIT_SIZE); } +struct mana_table { + struct mana_qp **table; + int refcnt; +}; + struct mana_context { struct verbs_context ibv_ctx; - struct { - struct mana_qp **table; - int refcnt; - } qp_table[MANA_QP_TABLE_SIZE]; + struct mana_table qp_rtable[MANA_QP_TABLE_SIZE]; + struct mana_table qp_stable[MANA_QP_TABLE_SIZE]; pthread_mutex_t qp_table_mutex; struct manadv_ctx_allocators extern_alloc; @@ -110,10 +113,8 @@ struct mana_ib_raw_qp { struct mana_ib_rc_qp { struct mana_gdma_queue queues[USER_RC_QUEUE_TYPE_MAX]; - uint32_t sq_ssn; uint32_t sq_psn; - uint32_t sq_highest_completed_psn; }; struct mana_qp { @@ -157,7 +158,6 @@ struct mana_cq { pthread_spinlock_t lock; uint32_t head; uint32_t last_armed_head; - uint32_t ready_wcs; void *db_page; /* list of qp's that use this cq for send completions */ struct list_head send_qp_list; @@ -240,5 +240,7 @@ int mana_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, int mana_arm_cq(struct ibv_cq *ibcq, int solicited); -struct mana_qp *mana_get_qp_from_rq(struct mana_context *ctx, uint32_t qpn); +struct mana_qp *mana_get_qp(struct mana_context *ctx, uint32_t qpn, bool is_sq); + +void mana_qp_move_flush_err(struct ibv_qp *ibqp); #endif diff --git a/providers/mana/qp.c b/providers/mana/qp.c index 67c945b..0934804 100644 --- a/providers/mana/qp.c +++ b/providers/mana/qp.c @@ -113,68 +113,114 @@ free_qp: return NULL; } -static int mana_store_qp(struct mana_context *ctx, struct mana_qp *qp, uint32_t qid) +static int mana_store_qid(struct mana_table *qp_table, struct mana_qp *qp, uint32_t qid) { uint32_t tbl_idx, tbl_off; int ret = 0; - pthread_mutex_lock(&ctx->qp_table_mutex); - tbl_idx = qid >> MANA_QP_TABLE_SHIFT; tbl_off = qid & MANA_QP_TABLE_MASK; - if (ctx->qp_table[tbl_idx].refcnt == 0) { - ctx->qp_table[tbl_idx].table = + if (qp_table[tbl_idx].refcnt == 0) { + qp_table[tbl_idx].table = calloc(MANA_QP_TABLE_SIZE, sizeof(struct mana_qp *)); - if (!ctx->qp_table[tbl_idx].table) { + if (!qp_table[tbl_idx].table) { ret = ENOMEM; goto out; } } - if (ctx->qp_table[tbl_idx].table[tbl_off]) { + if (qp_table[tbl_idx].table[tbl_off]) { ret = EBUSY; goto out; } - ctx->qp_table[tbl_idx].table[tbl_off] = qp; - ctx->qp_table[tbl_idx].refcnt++; + qp_table[tbl_idx].table[tbl_off] = qp; + qp_table[tbl_idx].refcnt++; out: - pthread_mutex_unlock(&ctx->qp_table_mutex); return ret; } -static void mana_remove_qp(struct mana_context *ctx, uint32_t qid) +static void mana_remove_qid(struct mana_table *qp_table, uint32_t qid) { uint32_t tbl_idx, tbl_off; - pthread_mutex_lock(&ctx->qp_table_mutex); tbl_idx = qid >> MANA_QP_TABLE_SHIFT; tbl_off = qid & MANA_QP_TABLE_MASK; - ctx->qp_table[tbl_idx].table[tbl_off] = NULL; - ctx->qp_table[tbl_idx].refcnt--; + qp_table[tbl_idx].table[tbl_off] = NULL; + qp_table[tbl_idx].refcnt--; - if (ctx->qp_table[tbl_idx].refcnt == 0) { - free(ctx->qp_table[tbl_idx].table); - ctx->qp_table[tbl_idx].table = NULL; + if (qp_table[tbl_idx].refcnt == 0) { + free(qp_table[tbl_idx].table); + qp_table[tbl_idx].table = NULL; } +} + +static int mana_store_qp(struct mana_context *ctx, struct mana_qp *qp) +{ + uint32_t sreq = qp->rc_qp.queues[USER_RC_SEND_QUEUE_REQUESTER].id; + uint32_t srep = qp->rc_qp.queues[USER_RC_SEND_QUEUE_RESPONDER].id; + uint32_t rreq = qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER].id; + uint32_t rrep = qp->rc_qp.queues[USER_RC_RECV_QUEUE_RESPONDER].id; + int ret; + + pthread_mutex_lock(&ctx->qp_table_mutex); + ret = mana_store_qid(ctx->qp_stable, qp, sreq); + if (ret) + goto error; + ret = mana_store_qid(ctx->qp_stable, qp, srep); + if (ret) + goto remove_sreq; + ret = mana_store_qid(ctx->qp_rtable, qp, rreq); + if (ret) + goto remove_srep; + ret = mana_store_qid(ctx->qp_rtable, qp, rrep); + if (ret) + goto remove_rreq; pthread_mutex_unlock(&ctx->qp_table_mutex); + return 0; + +remove_rreq: + mana_remove_qid(ctx->qp_rtable, rreq); +remove_srep: + mana_remove_qid(ctx->qp_stable, srep); +remove_sreq: + mana_remove_qid(ctx->qp_stable, sreq); +error: + pthread_mutex_unlock(&ctx->qp_table_mutex); + return ret; +} + +static void mana_remove_qp(struct mana_context *ctx, struct mana_qp *qp) +{ + uint32_t sreq = qp->rc_qp.queues[USER_RC_SEND_QUEUE_REQUESTER].id; + uint32_t srep = qp->rc_qp.queues[USER_RC_SEND_QUEUE_RESPONDER].id; + uint32_t rreq = qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER].id; + uint32_t rrep = qp->rc_qp.queues[USER_RC_RECV_QUEUE_RESPONDER].id; + + pthread_mutex_lock(&ctx->qp_table_mutex); + mana_remove_qid(ctx->qp_stable, sreq); + mana_remove_qid(ctx->qp_stable, srep); + mana_remove_qid(ctx->qp_rtable, rreq); + mana_remove_qid(ctx->qp_rtable, rrep); + pthread_mutex_unlock(&ctx->qp_table_mutex); } -struct mana_qp *mana_get_qp_from_rq(struct mana_context *ctx, uint32_t qid) +struct mana_qp *mana_get_qp(struct mana_context *ctx, uint32_t qid, bool is_sq) { + struct mana_table *qp_table = is_sq ? ctx->qp_stable : ctx->qp_rtable; uint32_t tbl_idx, tbl_off; tbl_idx = qid >> MANA_QP_TABLE_SHIFT; tbl_off = qid & MANA_QP_TABLE_MASK; - if (!ctx->qp_table[tbl_idx].table) + if (!qp_table[tbl_idx].table) return NULL; - return ctx->qp_table[tbl_idx].table[tbl_off]; + return qp_table[tbl_idx].table[tbl_off]; } static uint32_t get_queue_size(struct ibv_qp_init_attr *attr, enum user_queue_types type) @@ -284,16 +330,11 @@ static struct ibv_qp *mana_create_qp_rc(struct ibv_pd *ibpd, qp->ibqp.qp.qp_num = qp->rc_qp.queues[USER_RC_RECV_QUEUE_RESPONDER].id; - ret = mana_store_qp(ctx, qp, qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER].id); + ret = mana_store_qp(ctx, qp); if (ret) { errno = ret; goto destroy_qp; } - ret = mana_store_qp(ctx, qp, qp->rc_qp.queues[USER_RC_RECV_QUEUE_RESPONDER].id); - if (ret) { - errno = ret; - goto remove_qp_req; - } pthread_spin_lock(&send_cq->lock); list_add(&send_cq->send_qp_list, &qp->send_cq_node); @@ -305,8 +346,6 @@ static struct ibv_qp *mana_create_qp_rc(struct ibv_pd *ibpd, return &qp->ibqp.qp; -remove_qp_req: - mana_remove_qp(ctx, qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER].id); destroy_qp: ibv_cmd_destroy_qp(&qp->ibqp.qp); free_rb: @@ -346,29 +385,32 @@ static void mana_ib_modify_rc_qp(struct mana_qp *qp, struct ibv_qp_attr *attr, i if (attr_mask & IBV_QP_PATH_MTU) qp->mtu = attr->path_mtu; - switch (attr->qp_state) { - case IBV_QPS_RESET: - for (i = 0; i < USER_RC_QUEUE_TYPE_MAX; ++i) { - qp->rc_qp.queues[i].prod_idx = 0; - qp->rc_qp.queues[i].cons_idx = 0; - } - mana_ib_reset_rb_shmem(qp); - reset_shadow_queue(&qp->shadow_rq); - reset_shadow_queue(&qp->shadow_sq); - case IBV_QPS_INIT: - break; - case IBV_QPS_RTR: - break; - case IBV_QPS_RTS: - if (attr_mask & IBV_QP_SQ_PSN) { - qp->rc_qp.sq_ssn = 1; - qp->rc_qp.sq_psn = attr->sq_psn; - qp->rc_qp.sq_highest_completed_psn = PSN_DEC(attr->sq_psn); - gdma_arm_normal_cqe(&qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER], attr->sq_psn); + if (attr_mask & IBV_QP_STATE) { + qp->ibqp.qp.state = attr->qp_state; + switch (attr->qp_state) { + case IBV_QPS_RESET: + for (i = 0; i < USER_RC_QUEUE_TYPE_MAX; ++i) { + qp->rc_qp.queues[i].prod_idx = 0; + qp->rc_qp.queues[i].cons_idx = 0; + } + mana_ib_reset_rb_shmem(qp); + reset_shadow_queue(&qp->shadow_rq); + reset_shadow_queue(&qp->shadow_sq); + case IBV_QPS_INIT: + break; + case IBV_QPS_RTR: + break; + case IBV_QPS_RTS: + if (attr_mask & IBV_QP_SQ_PSN) { + qp->rc_qp.sq_ssn = 1; + qp->rc_qp.sq_psn = attr->sq_psn; + gdma_arm_normal_cqe(&qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER], + attr->sq_psn); + } + break; + default: + break; } - break; - default: - break; } } @@ -381,18 +423,21 @@ int mana_modify_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, int attr_mask) if (ibqp->qp_type != IBV_QPT_RC) return EOPNOTSUPP; - if (!(attr_mask & IBV_QP_STATE)) - return 0; + pthread_spin_lock(&qp->sq_lock); + pthread_spin_lock(&qp->rq_lock); err = ibv_cmd_modify_qp(ibqp, attr, attr_mask, &cmd, sizeof(cmd)); if (err) { verbs_err(verbs_get_ctx(ibqp->context), "Failed to modify qp\n"); - return err; + goto cleanup; } mana_ib_modify_rc_qp(qp, attr, attr_mask); - return 0; +cleanup: + pthread_spin_unlock(&qp->rq_lock); + pthread_spin_unlock(&qp->sq_lock); + return err; } static void mana_drain_cqes(struct mana_qp *qp) @@ -401,18 +446,10 @@ static void mana_drain_cqes(struct mana_qp *qp) struct mana_cq *recv_cq = container_of(qp->ibqp.qp.recv_cq, struct mana_cq, ibcq); pthread_spin_lock(&send_cq->lock); - while (shadow_queue_get_next_to_consume(&qp->shadow_sq)) { - shadow_queue_advance_consumer(&qp->shadow_sq); - send_cq->ready_wcs--; - } list_del(&qp->send_cq_node); pthread_spin_unlock(&send_cq->lock); pthread_spin_lock(&recv_cq->lock); - while (shadow_queue_get_next_to_consume(&qp->shadow_rq)) { - shadow_queue_advance_consumer(&qp->shadow_rq); - recv_cq->ready_wcs--; - } list_del(&qp->recv_cq_node); pthread_spin_unlock(&recv_cq->lock); } @@ -424,8 +461,7 @@ int mana_destroy_qp(struct ibv_qp *ibqp) int ret, i; if (ibqp->qp_type == IBV_QPT_RC) { - mana_remove_qp(ctx, qp->rc_qp.queues[USER_RC_RECV_QUEUE_REQUESTER].id); - mana_remove_qp(ctx, qp->rc_qp.queues[USER_RC_RECV_QUEUE_RESPONDER].id); + mana_remove_qp(ctx, qp); mana_drain_cqes(qp); } @@ -546,3 +582,11 @@ struct ibv_qp *mana_create_qp_ex(struct ibv_context *context, return NULL; } + +void mana_qp_move_flush_err(struct ibv_qp *ibqp) +{ + struct ibv_qp_attr attr = {}; + + attr.qp_state = IBV_QPS_ERR; + mana_modify_qp(ibqp, &attr, IBV_QP_STATE); +} diff --git a/providers/mana/shadow_queue.h b/providers/mana/shadow_queue.h index 6c86cdb..1073f7c 100644 --- a/providers/mana/shadow_queue.h +++ b/providers/mana/shadow_queue.h @@ -17,13 +17,13 @@ struct shadow_wqe_header { /* ibv_wc_opcode */ - uint8_t opcode; + uint64_t opcode : 8; /* ibv_wc_flags or MANA_NO_SIGNAL_WC */ - uint8_t flags; + uint64_t flags : 8; + uint64_t posted_wqe_size_in_bu : 8; /* ibv_wc_status */ - uint8_t vendor_error_code; - uint8_t posted_wqe_size_in_bu; - uint32_t unmasked_queue_offset; + uint64_t vendor_error : 12; + uint64_t unmasked_queue_offset : 28; uint64_t wr_id; }; @@ -43,6 +43,7 @@ struct shadow_queue { uint64_t prod_idx; uint64_t cons_idx; uint64_t next_to_complete_idx; + uint64_t next_to_signal_idx; uint32_t length; uint32_t stride; void *buffer; @@ -53,6 +54,7 @@ static inline void reset_shadow_queue(struct shadow_queue *queue) queue->prod_idx = 0; queue->cons_idx = 0; queue->next_to_complete_idx = 0; + queue->next_to_signal_idx = 0; } static inline int create_shadow_queue(struct shadow_queue *queue, uint32_t length, uint32_t stride) @@ -148,4 +150,20 @@ static inline void shadow_queue_advance_next_to_complete(struct shadow_queue *qu queue->next_to_complete_idx++; } +static inline struct shadow_wqe_header * +shadow_queue_get_next_to_signal(struct shadow_queue *queue) +{ + struct shadow_wqe_header *wqe = NULL; + + queue->next_to_signal_idx = max(queue->next_to_signal_idx, queue->next_to_complete_idx); + while (queue->next_to_signal_idx < queue->prod_idx) { + wqe = shadow_queue_get_element(queue, queue->next_to_signal_idx); + queue->next_to_signal_idx++; + if (wqe->flags != MANA_NO_SIGNAL_WC) + return wqe; + } + + return NULL; +} + #endif //_SHADOW_QUEUE_H_ diff --git a/providers/mana/wr.c b/providers/mana/wr.c index 5661045..a9a5fe5 100644 --- a/providers/mana/wr.c +++ b/providers/mana/wr.c @@ -146,6 +146,13 @@ static int mana_ib_rc_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, int ret = 0; pthread_spin_lock(&qp->rq_lock); + + if (unlikely(ibqp->state == IBV_QPS_RESET || ibqp->state == IBV_QPS_ERR)) { + verbs_err(verbs_get_ctx(ibqp->context), "Invalid QP state\n"); + ret = EINVAL; + goto cleanup; + } + for (; wr; wr = wr->next) { if (shadow_queue_full(&qp->shadow_rq)) { verbs_err(&mc->ibv_ctx, "recv shadow queue full\n"); @@ -371,8 +378,14 @@ static int mana_ib_rc_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, pthread_spin_lock(&qp->sq_lock); + if (unlikely(ibqp->state != IBV_QPS_RTS)) { + verbs_err(verbs_get_ctx(ibqp->context), "Invalid QP state\n"); + ret = EINVAL; + goto cleanup; + } + for (; wr; wr = wr->next) { - if ((wr->send_flags & IBV_SEND_SIGNALED) && shadow_queue_full(&qp->shadow_sq)) { + if (shadow_queue_full(&qp->shadow_sq)) { verbs_err(verbs_get_ctx(ibqp->context), "shadow queue full\n"); ret = ENOMEM; goto cleanup; @@ -397,7 +410,8 @@ static int mana_ib_rc_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, ring = true; shadow_queue_advance_producer(&qp->shadow_sq); - mana_ib_update_shared_mem_right_offset(qp, shadow_wqe->header.unmasked_queue_offset); + mana_ib_update_shared_mem_right_offset(qp, + shadow_wqe->header.unmasked_queue_offset); } cleanup: