From acd884dec99adcf8c4cdd0aa8a50be79c216f8e8 Mon Sep 17 00:00:00 2001 From: Sumit Saxena Date: Tue, 28 May 2024 10:31:59 +0000 Subject: [PATCH] RDMA/bnxt_re: Add bnxt_re RoCE driver This patch introduces the RoCE driver for the Broadcom NetXtreme-E 10/25/50/100/200G RoCE HCAs. The RoCE driver is a two part driver that relies on the bnxt_en NIC driver to operate. The changes needed in the bnxt_en driver is included through another patch "L2-RoCE driver communication interface" in this set. Presently, There is no user space support, Hence recommendation to use the krping kernel module for testing. User space support will be incorporated in subsequent patch submissions. Reviewed by: imp Approved by: imp Differential revision: https://reviews.freebsd.org/D45011 --- sys/dev/bnxt/bnxt_re/bnxt_re-abi.h | 177 + sys/dev/bnxt/bnxt_re/bnxt_re.h | 1075 ++++++ sys/dev/bnxt/bnxt_re/ib_verbs.c | 5498 ++++++++++++++++++++++++++++ sys/dev/bnxt/bnxt_re/ib_verbs.h | 632 ++++ sys/dev/bnxt/bnxt_re/main.c | 4467 ++++++++++++++++++++++ sys/dev/bnxt/bnxt_re/qplib_fp.c | 3544 ++++++++++++++++++ sys/dev/bnxt/bnxt_re/qplib_fp.h | 638 ++++ sys/dev/bnxt/bnxt_re/qplib_rcfw.c | 1338 +++++++ sys/dev/bnxt/bnxt_re/qplib_rcfw.h | 354 ++ sys/dev/bnxt/bnxt_re/qplib_res.c | 1226 +++++++ sys/dev/bnxt/bnxt_re/qplib_res.h | 840 +++++ sys/dev/bnxt/bnxt_re/qplib_sp.c | 1234 +++++++ sys/dev/bnxt/bnxt_re/qplib_sp.h | 432 +++ sys/dev/bnxt/bnxt_re/qplib_tlv.h | 187 + sys/dev/bnxt/bnxt_re/stats.c | 773 ++++ sys/dev/bnxt/bnxt_re/stats.h | 353 ++ sys/modules/bnxt/bnxt_re/Makefile | 22 + 17 files changed, 22790 insertions(+) create mode 100644 sys/dev/bnxt/bnxt_re/bnxt_re-abi.h create mode 100644 sys/dev/bnxt/bnxt_re/bnxt_re.h create mode 100644 sys/dev/bnxt/bnxt_re/ib_verbs.c create mode 100644 sys/dev/bnxt/bnxt_re/ib_verbs.h create mode 100644 sys/dev/bnxt/bnxt_re/main.c create mode 100644 sys/dev/bnxt/bnxt_re/qplib_fp.c create mode 100644 sys/dev/bnxt/bnxt_re/qplib_fp.h create mode 100644 sys/dev/bnxt/bnxt_re/qplib_rcfw.c create mode 100644 sys/dev/bnxt/bnxt_re/qplib_rcfw.h create mode 100644 sys/dev/bnxt/bnxt_re/qplib_res.c create mode 100644 sys/dev/bnxt/bnxt_re/qplib_res.h create mode 100644 sys/dev/bnxt/bnxt_re/qplib_sp.c create mode 100644 sys/dev/bnxt/bnxt_re/qplib_sp.h create mode 100644 sys/dev/bnxt/bnxt_re/qplib_tlv.h create mode 100644 sys/dev/bnxt/bnxt_re/stats.c create mode 100644 sys/dev/bnxt/bnxt_re/stats.h create mode 100644 sys/modules/bnxt/bnxt_re/Makefile diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h new file mode 100644 index 00000000000..8f48609e7f6 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Uverbs ABI header file + */ + +#ifndef __BNXT_RE_UVERBS_ABI_H__ +#define __BNXT_RE_UVERBS_ABI_H__ + +#include +#include + +#define BNXT_RE_ABI_VERSION 6 + +enum { + BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED = 0x01, + BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED = 0x02, + BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED = 0x04, + BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED = 0x08, + BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED = 0x10, + BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED = 0x20, + BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40 +}; + +enum { + BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT = 0x01, + BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02 +}; + +struct bnxt_re_uctx_req { + __aligned_u64 comp_mask; +}; + +#define BNXT_RE_CHIP_ID0_CHIP_NUM_SFT 0x00 +#define BNXT_RE_CHIP_ID0_CHIP_REV_SFT 0x10 +#define BNXT_RE_CHIP_ID0_CHIP_MET_SFT 0x18 +struct bnxt_re_uctx_resp { + __u32 dev_id; + __u32 max_qp; + __u32 pg_size; + __u32 cqe_sz; + __u32 max_cqd; + __u32 chip_id0; + __u32 chip_id1; + __u32 modes; + __aligned_u64 comp_mask; +} __attribute__((packed)); + +enum { + BNXT_RE_COMP_MASK_PD_HAS_WC_DPI = 0x01, + BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR = 0x02, +}; + +struct bnxt_re_pd_resp { + __u32 pdid; + __u32 dpi; + __u64 dbr; + __u64 comp_mask; + __u32 wcdpi; + __u64 dbr_bar_addr; +} __attribute__((packed)); + +enum { + BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO = 0x01, + BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI = 0x02, + BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE = 0x04, +}; + +enum { + BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK = 0x1 +}; + +enum { + BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_RECOVERY = 0x1, + BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_PACING_NOTIFY = 0x2 +}; + +#define BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(_req) \ + (_req.comp_mask & BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK && \ + _req.cq_capability & BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_PACING_NOTIFY) + +#define BNXT_RE_IS_DBR_RECOV_CQ(_req) \ + (_req.comp_mask & BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK && \ + _req.cq_capability & BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_RECOVERY) + +struct bnxt_re_cq_req { + __u64 cq_va; + __u64 cq_handle; + __aligned_u64 comp_mask; + __u16 cq_capability; +} __attribute__((packed)); + +struct bnxt_re_cq_resp { + __u32 cqid; + __u32 tail; + __u32 phase; + __u32 rsvd; + __aligned_u64 comp_mask; + __u32 dpi; + __u64 dbr; + __u32 wcdpi; + __u64 uctx_cq_page; +} __attribute__((packed)); + +struct bnxt_re_resize_cq_req { + __u64 cq_va; +} __attribute__((packed)); + +struct bnxt_re_qp_req { + __u64 qpsva; + __u64 qprva; + __u64 qp_handle; +} __attribute__((packed)); + +struct bnxt_re_qp_resp { + __u32 qpid; +} __attribute__((packed)); + +struct bnxt_re_srq_req { + __u64 srqva; + __u64 srq_handle; +} __attribute__((packed)); + +struct bnxt_re_srq_resp { + __u32 srqid; +} __attribute__((packed)); + +/* Modify QP */ +enum { + BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK = 0x1, + BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN = 0x1, + BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK = 0x2 +}; + +struct bnxt_re_modify_qp_ex_req { + __aligned_u64 comp_mask; + __u32 dpi; + __u32 rsvd; +} __packed; + +struct bnxt_re_modify_qp_ex_resp { + __aligned_u64 comp_mask; + __u32 ppp_st_idx; + __u32 path_mtu; +} __packed; + +enum bnxt_re_shpg_offt { + BNXT_RE_BEG_RESV_OFFT = 0x00, + BNXT_RE_AVID_OFFT = 0x10, + BNXT_RE_AVID_SIZE = 0x04, + BNXT_RE_END_RESV_OFFT = 0xFF0 +}; +#endif diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re.h b/sys/dev/bnxt/bnxt_re/bnxt_re.h new file mode 100644 index 00000000000..3bf0bbeb106 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/bnxt_re.h @@ -0,0 +1,1075 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: main (header) + */ + +#ifndef __BNXT_RE_H__ +#define __BNXT_RE_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bnxt.h" +#include "bnxt_ulp.h" +#include "hsi_struct_def.h" +#include "qplib_res.h" +#include "qplib_sp.h" +#include "qplib_fp.h" +#include "qplib_rcfw.h" +#include "ib_verbs.h" +#include "stats.h" + +#define ROCE_DRV_MODULE_NAME "bnxt_re" +#define ROCE_DRV_MODULE_VERSION "230.0.133.0" +#define ROCE_DRV_MODULE_RELDATE "April 22, 2024" + +#define BNXT_RE_REF_WAIT_COUNT 20 +#define BNXT_RE_ROCE_V1_ETH_TYPE 0x8915 +#define BNXT_RE_ROCE_V2_PORT_NO 4791 +#define BNXT_RE_RES_FREE_WAIT_COUNT 1000 + +#define BNXT_RE_PAGE_SHIFT_4K (12) +#define BNXT_RE_PAGE_SHIFT_8K (13) +#define BNXT_RE_PAGE_SHIFT_64K (16) +#define BNXT_RE_PAGE_SHIFT_2M (21) +#define BNXT_RE_PAGE_SHIFT_8M (23) +#define BNXT_RE_PAGE_SHIFT_1G (30) + +#define BNXT_RE_PAGE_SIZE_4K BIT(BNXT_RE_PAGE_SHIFT_4K) +#define BNXT_RE_PAGE_SIZE_8K BIT(BNXT_RE_PAGE_SHIFT_8K) +#define BNXT_RE_PAGE_SIZE_64K BIT(BNXT_RE_PAGE_SHIFT_64K) +#define BNXT_RE_PAGE_SIZE_2M BIT(BNXT_RE_PAGE_SHIFT_2M) +#define BNXT_RE_PAGE_SIZE_8M BIT(BNXT_RE_PAGE_SHIFT_8M) +#define BNXT_RE_PAGE_SIZE_1G BIT(BNXT_RE_PAGE_SHIFT_1G) + +#define BNXT_RE_MAX_MR_SIZE_LOW BIT(BNXT_RE_PAGE_SHIFT_1G) +#define BNXT_RE_MAX_MR_SIZE_HIGH BIT(39) +#define BNXT_RE_MAX_MR_SIZE BNXT_RE_MAX_MR_SIZE_HIGH + +/* Number of MRs to reserve for PF, leaving remainder for VFs */ +#define BNXT_RE_RESVD_MR_FOR_PF (32 * 1024) +#define BNXT_RE_MAX_GID_PER_VF 128 + +#define BNXT_RE_MAX_VF_QPS_PER_PF (6 * 1024) + +/** + * min_not_zero - return the minimum that is _not_ zero, unless both are zero + * @x: value1 + * @y: value2 + */ +#define min_not_zero(x, y) ({ \ + typeof(x) __x = (x); \ + typeof(y) __y = (y); \ + __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); }) + +struct ib_mr_init_attr { + int max_reg_descriptors; + u32 flags; +}; + +struct bnxt_re_dev; + +int bnxt_re_register_netdevice_notifier(struct notifier_block *nb); +int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb); +int ib_register_device_compat(struct bnxt_re_dev *rdev); + +#ifndef __struct_group +#define __struct_group(TAG, NAME, ATTRS, MEMBERS...) \ + union { \ + struct { MEMBERS } ATTRS; \ + struct TAG { MEMBERS } ATTRS NAME; \ + } +#endif /* __struct_group */ +#ifndef struct_group_attr +#define struct_group_attr(NAME, ATTRS, MEMBERS...) \ + __struct_group(/* no tag */, NAME, ATTRS, MEMBERS) +#endif /* struct_group_attr */ +/* + * Percentage of resources of each type reserved for PF. + * Remaining resources are divided equally among VFs. + * [0, 100] + */ + +#define BNXT_RE_RQ_WQE_THRESHOLD 32 +#define BNXT_RE_UD_QP_HW_STALL 0x400000 + +/* + * Setting the default ack delay value to 16, which means + * the default timeout is approx. 260ms(4 usec * 2 ^(timeout)) + */ + +#define BNXT_RE_DEFAULT_ACK_DELAY 16 +#define BNXT_RE_BOND_PF_MAX 2 + +#define BNXT_RE_STATS_CTX_UPDATE_TIMER 250 +#define BNXT_RE_30SEC_MSEC (30 * 1000) + +#define BNXT_RE_BOND_RESCHED_CNT 10 + +#define BNXT_RE_CHIP_NUM_57454 0xC454 +#define BNXT_RE_CHIP_NUM_57452 0xC452 + +#define BNXT_RE_CHIP_NUM_5745X(chip_num) \ + ((chip_num) == BNXT_RE_CHIP_NUM_57454 || \ + (chip_num) == BNXT_RE_CHIP_NUM_57452) + +#define BNXT_RE_MIN_KERNEL_QP_TX_DEPTH 4096 +#define BNXT_RE_STOP_QPS_BUDGET 200 + +#define BNXT_RE_HWRM_CMD_TIMEOUT(rdev) \ + ((rdev)->chip_ctx->hwrm_cmd_max_timeout * 1000) + +extern unsigned int min_tx_depth; +extern struct mutex bnxt_re_dev_lock; +extern struct mutex bnxt_re_mutex; +extern struct list_head bnxt_re_dev_list; + +struct bnxt_re_ring_attr { + dma_addr_t *dma_arr; + int pages; + int type; + u32 depth; + u32 lrid; /* Logical ring id */ + u16 flags; + u8 mode; + u8 rsvd; +}; + +#define BNXT_RE_MAX_DEVICES 256 +#define BNXT_RE_MSIX_FROM_MOD_PARAM -1 +#define BNXT_RE_MIN_MSIX 2 +#define BNXT_RE_MAX_MSIX_VF 2 +#define BNXT_RE_MAX_MSIX_PF 9 +#define BNXT_RE_MAX_MSIX_NPAR_PF 5 +#define BNXT_RE_MAX_MSIX 64 +#define BNXT_RE_MAX_MSIX_GEN_P5_PF BNXT_RE_MAX_MSIX +#define BNXT_RE_GEN_P5_MAX_VF 64 + +struct bnxt_re_nq_record { + struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX]; + /* FP Notification Queue (CQ & SRQ) */ + struct bnxt_qplib_nq nq[BNXT_RE_MAX_MSIX]; + int num_msix; + int max_init; + struct mutex load_lock; +}; + +struct bnxt_re_work { + struct work_struct work; + unsigned long event; + struct bnxt_re_dev *rdev; + struct ifnet *vlan_dev; + bool do_lag; + + /* netdev where we received the event */ + struct ifnet *netdev; + struct auxiliary_device *adev; +}; + +/* + * Data structure and defines to handle + * recovery + */ +#define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY 20 +#define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS 30000 /* 30sec timeout */ +#define BNXT_RE_PRE_RECOVERY_REMOVE 0x1 +#define BNXT_RE_COMPLETE_REMOVE 0x2 +#define BNXT_RE_POST_RECOVERY_INIT 0x4 +#define BNXT_RE_COMPLETE_INIT 0x8 +#define BNXT_RE_COMPLETE_SHUTDOWN 0x10 + +/* QP1 SQ entry data strucutre */ +struct bnxt_re_sqp_entries { + u64 wrid; + struct bnxt_qplib_sge sge; + /* For storing the actual qp1 cqe */ + struct bnxt_qplib_cqe cqe; + struct bnxt_re_qp *qp1_qp; +}; + +/* GSI QP mode enum */ +enum bnxt_re_gsi_mode { + BNXT_RE_GSI_MODE_INVALID = 0, + BNXT_RE_GSI_MODE_ALL = 1, + BNXT_RE_GSI_MODE_ROCE_V1, + BNXT_RE_GSI_MODE_ROCE_V2_IPV4, + BNXT_RE_GSI_MODE_ROCE_V2_IPV6, + BNXT_RE_GSI_MODE_UD +}; + +enum bnxt_re_roce_cap { + BNXT_RE_FLAG_ROCEV1_CAP = 1, + BNXT_RE_FLAG_ROCEV2_CAP, + BNXT_RE_FLAG_ROCEV1_V2_CAP, +}; + +#define BNXT_RE_MAX_GSI_SQP_ENTRIES 1024 +struct bnxt_re_gsi_context { + u8 gsi_qp_mode; + bool first_cq_created; + /* Start: used only in gsi_mode_all */ + struct bnxt_re_qp *gsi_qp; + struct bnxt_re_qp *gsi_sqp; + struct bnxt_re_ah *gsi_sah; + struct bnxt_re_sqp_entries *sqp_tbl; + /* End: used only in gsi_mode_all */ +}; + +struct bnxt_re_tc_rec { + u8 cos_id_roce; + u8 tc_roce; + u8 cos_id_cnp; + u8 tc_cnp; + u8 tc_def; + u8 cos_id_def; + u8 max_tc; + u8 roce_prio; + u8 cnp_prio; + u8 roce_dscp; + u8 cnp_dscp; + u8 prio_valid; + u8 dscp_valid; + bool ecn_enabled; + bool serv_type_enabled; + u64 cnp_dscp_bv; + u64 roce_dscp_bv; +}; + +struct bnxt_re_dscp2pri { + u8 dscp; + u8 mask; + u8 pri; +}; + +struct bnxt_re_cos2bw_cfg { + u8 pad[3]; + struct_group_attr(cfg, __packed, + u8 queue_id; + __le32 min_bw; + __le32 max_bw; + u8 tsa; + u8 pri_lvl; + u8 bw_weight; + ); + u8 unused; +}; + +#define BNXT_RE_AEQ_IDX 0 +#define BNXT_RE_MAX_SGID_ENTRIES 256 + +#define BNXT_RE_DBGFS_FILE_MEM 65536 +enum { + BNXT_RE_STATS_QUERY = 1, + BNXT_RE_QP_QUERY = 2, + BNXT_RE_SERVICE_FN_QUERY = 3, +}; + +struct bnxt_re_dbg_file { + struct bnxt_re_dev *rdev; + u32 type; + union { + struct bnxt_qplib_query_stats_info sinfo; + struct bnxt_qplib_query_fn_info fninfo; + }params; + char dbg_buf[BNXT_RE_DBGFS_FILE_MEM]; +}; + +struct bnxt_re_debug_entries { + /* Dir entries */ + struct dentry *qpinfo_dir; + struct dentry *service_fn_dir; + /* file entries */ + struct dentry *stat_query; + struct bnxt_re_dbg_file stat_file; + struct dentry *qplist_query; + struct bnxt_re_dbg_file qp_file; + struct dentry *service_fn_query; + struct bnxt_re_dbg_file service_fn_file; +}; + +struct bnxt_re_en_dev_info { + struct list_head en_list; + struct bnxt_en_dev *en_dev; + struct bnxt_re_dev *rdev; + unsigned long flags; +#define BNXT_RE_FLAG_EN_DEV_NETDEV_REG 0 +#define BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV 1 +#define BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV 2 + u8 wqe_mode; + u8 gsi_mode; + bool te_bypass; + bool ib_uninit_done; + u32 num_msix_requested; + wait_queue_head_t waitq; +}; + +#define BNXT_RE_DB_FIFO_ROOM_MASK_P5 0x1FFF8000 +#define BNXT_RE_MAX_FIFO_DEPTH_P5 0x2c00 +#define BNXT_RE_DB_FIFO_ROOM_SHIFT 15 + +#define BNXT_RE_DB_FIFO_ROOM_MASK_P7 0x3FFF8000 +#define BNXT_RE_MAX_FIFO_DEPTH_P7 0x8000 + +#define BNXT_RE_DB_FIFO_ROOM_MASK(ctx) \ + (_is_chip_p7((ctx)) ? \ + BNXT_RE_DB_FIFO_ROOM_MASK_P7 :\ + BNXT_RE_DB_FIFO_ROOM_MASK_P5) +#define BNXT_RE_MAX_FIFO_DEPTH(ctx) \ + (_is_chip_p7((ctx)) ? \ + BNXT_RE_MAX_FIFO_DEPTH_P7 :\ + BNXT_RE_MAX_FIFO_DEPTH_P5) + +struct bnxt_dbq_nq_list { + int num_nql_entries; + u16 nq_id[16]; +}; + +#define BNXT_RE_ASYNC_ERR_REP_BASE(_type) \ + (ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_##_type) + +#define BNXT_RE_ASYNC_ERR_DBR_TRESH(_type) \ + (ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_##_type) + +#define BNXT_RE_EVENT_DBR_EPOCH(data) \ + (((data) & \ + BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_MASK)) >> \ + BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_SFT)) + +#define BNXT_RE_EVENT_ERROR_REPORT_TYPE(data1) \ + (((data1) & \ + BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_MASK)) >> \ + BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_SFT)) + +#define BNXT_RE_DBR_LIST_ADD(_rdev, _res, _type) \ +{ \ + spin_lock(&(_rdev)->res_list[_type].lock); \ + list_add_tail(&(_res)->dbr_list, \ + &(_rdev)->res_list[_type].head); \ + spin_unlock(&(_rdev)->res_list[_type].lock); \ +} + +#define BNXT_RE_DBR_LIST_DEL(_rdev, _res, _type) \ +{ \ + spin_lock(&(_rdev)->res_list[_type].lock); \ + list_del(&(_res)->dbr_list); \ + spin_unlock(&(_rdev)->res_list[_type].lock); \ +} + +#define BNXT_RE_CQ_PAGE_LIST_ADD(_uctx, _cq) \ +{ \ + mutex_lock(&(_uctx)->cq_lock); \ + list_add_tail(&(_cq)->cq_list, &(_uctx)->cq_list); \ + mutex_unlock(&(_uctx)->cq_lock); \ +} + +#define BNXT_RE_CQ_PAGE_LIST_DEL(_uctx, _cq) \ +{ \ + mutex_lock(&(_uctx)->cq_lock); \ + list_del(&(_cq)->cq_list); \ + mutex_unlock(&(_uctx)->cq_lock); \ +} + +#define BNXT_RE_NETDEV_EVENT(event, x) \ + do { \ + if ((event) == (x)) \ + return #x; \ + } while (0) + +/* Do not change the seq of this enum which is followed by dbr recov */ +enum { + BNXT_RE_RES_TYPE_CQ = 0, + BNXT_RE_RES_TYPE_UCTX, + BNXT_RE_RES_TYPE_QP, + BNXT_RE_RES_TYPE_SRQ, + BNXT_RE_RES_TYPE_MAX +}; + +struct bnxt_re_dbr_res_list { + struct list_head head; + spinlock_t lock; +}; + +struct bnxt_re_dbr_drop_recov_work { + struct work_struct work; + struct bnxt_re_dev *rdev; + u32 curr_epoch; +}; + +struct bnxt_re_aer_work { + struct work_struct work; + struct bnxt_re_dev *rdev; +}; + +struct bnxt_re_dbq_stats { + u64 fifo_occup_slab_1; + u64 fifo_occup_slab_2; + u64 fifo_occup_slab_3; + u64 fifo_occup_slab_4; + u64 fifo_occup_water_mark; + u64 do_pacing_slab_1; + u64 do_pacing_slab_2; + u64 do_pacing_slab_3; + u64 do_pacing_slab_4; + u64 do_pacing_slab_5; + u64 do_pacing_water_mark; +}; + +/* Device debug statistics */ +struct bnxt_re_drv_dbg_stats { + struct bnxt_re_dbq_stats dbq; +}; + +/* DB pacing counters */ +struct bnxt_re_dbr_sw_stats { + u64 dbq_int_recv; + u64 dbq_int_en; + u64 dbq_pacing_resched; + u64 dbq_pacing_complete; + u64 dbq_pacing_alerts; + u64 dbr_drop_recov_events; + u64 dbr_drop_recov_timeouts; + u64 dbr_drop_recov_timeout_users; + u64 dbr_drop_recov_event_skips; +}; + +struct bnxt_re_dev { + struct ib_device ibdev; + struct list_head list; + atomic_t ref_count; + atomic_t sched_count; + unsigned long flags; +#define BNXT_RE_FLAG_NETDEV_REGISTERED 0 +#define BNXT_RE_FLAG_IBDEV_REGISTERED 1 +#define BNXT_RE_FLAG_GOT_MSIX 2 +#define BNXT_RE_FLAG_HAVE_L2_REF 3 +#define BNXT_RE_FLAG_ALLOC_RCFW 4 +#define BNXT_RE_FLAG_NET_RING_ALLOC 5 +#define BNXT_RE_FLAG_RCFW_CHANNEL_EN 6 +#define BNXT_RE_FLAG_ALLOC_CTX 7 +#define BNXT_RE_FLAG_STATS_CTX_ALLOC 8 +#define BNXT_RE_FLAG_STATS_CTX2_ALLOC 9 +#define BNXT_RE_FLAG_RCFW_CHANNEL_INIT 10 +#define BNXT_RE_FLAG_WORKER_REG 11 +#define BNXT_RE_FLAG_TBLS_ALLOCINIT 12 +#define BNXT_RE_FLAG_SETUP_NQ 13 +#define BNXT_RE_FLAG_BOND_DEV_REGISTERED 14 +#define BNXT_RE_FLAG_PER_PORT_DEBUG_INFO 15 +#define BNXT_RE_FLAG_DEV_LIST_INITIALIZED 16 +#define BNXT_RE_FLAG_ERR_DEVICE_DETACHED 17 +#define BNXT_RE_FLAG_INIT_DCBX_CC_PARAM 18 +#define BNXT_RE_FLAG_STOP_IN_PROGRESS 20 +#define BNXT_RE_FLAG_ISSUE_ROCE_STATS 29 +#define BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS 30 + struct ifnet *netdev; + struct auxiliary_device *adev; + struct bnxt_qplib_chip_ctx *chip_ctx; + struct bnxt_en_dev *en_dev; + struct bnxt_re_nq_record nqr; + int id; + struct delayed_work worker; + u16 worker_30s; + struct bnxt_re_tc_rec tc_rec[2]; + u8 cur_prio_map; + /* RCFW Channel */ + struct bnxt_qplib_rcfw rcfw; + /* Device Resources */ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_res qplib_res; + struct bnxt_qplib_dpi dpi_privileged; + struct bnxt_qplib_cc_param cc_param; + struct mutex cc_lock; + struct mutex qp_lock; + struct list_head qp_list; + u8 roce_mode; + + /* Max of 2 lossless traffic class supported per port */ + u16 cosq[2]; + /* Start: QP for handling QP1 packets */ + struct bnxt_re_gsi_context gsi_ctx; + /* End: QP for handling QP1 packets */ + bool is_virtfn; + u32 num_vfs; + u32 espeed; + /* + * For storing the speed of slave interfaces. + * Same as espeed when bond is not configured + */ + u32 sl_espeed; + /* To be used for a workaround for ISER stack */ + u32 min_tx_depth; + /* To enable qp debug info. Disabled during driver load */ + u32 en_qp_dbg; + /* Array to handle gid mapping */ + char *gid_map; + + struct bnxt_re_device_stats stats; + struct bnxt_re_drv_dbg_stats *dbg_stats; + /* debugfs to expose per port information*/ + struct dentry *port_debug_dir; + struct dentry *info; + struct dentry *drv_dbg_stats; + struct dentry *sp_perf_stats; + struct dentry *pdev_debug_dir; + struct dentry *pdev_qpinfo_dir; + struct bnxt_re_debug_entries *dbg_ent; + struct workqueue_struct *resolve_wq; + struct list_head mac_wq_list; + struct workqueue_struct *dcb_wq; + struct workqueue_struct *aer_wq; + u32 event_bitmap[3]; + bool unreg_sched; + u64 dbr_throttling_reg_off; + u64 dbr_aeq_arm_reg_off; + u64 dbr_db_fifo_reg_off; + void *dbr_page; + u64 dbr_bar_addr; + u32 pacing_algo_th; + u32 pacing_en_int_th; + u32 do_pacing_save; + struct workqueue_struct *dbq_wq; + struct workqueue_struct *dbr_drop_recov_wq; + struct work_struct dbq_fifo_check_work; + struct delayed_work dbq_pacing_work; + /* protect DB pacing */ + struct mutex dbq_lock; + /* Control DBR pacing feature. Set if enabled */ + bool dbr_pacing; + /* Control DBR recovery feature. Set if enabled */ + bool dbr_drop_recov; + bool user_dbr_drop_recov; + /* DBR recovery feature. Set if running */ + bool dbr_recovery_on; + u32 user_dbr_drop_recov_timeout; + /* + * Value used for pacing algo when pacing is active + */ +#define BNXT_RE_MAX_DBR_DO_PACING 0xFFFF + u32 dbr_do_pacing; + u32 dbq_watermark; /* Current watermark set in HW registers */ + u32 dbq_nq_id; /* Current NQ ID for DBQ events */ + u32 dbq_pacing_time; /* ms */ + u32 dbr_def_do_pacing; /* do_pacing when no congestion */ + u32 dbr_evt_curr_epoch; + bool dbq_int_disable; + + bool mod_exit; + struct bnxt_re_dbr_sw_stats *dbr_sw_stats; + struct bnxt_re_dbr_res_list res_list[BNXT_RE_RES_TYPE_MAX]; + struct bnxt_dbq_nq_list nq_list; + char dev_name[IB_DEVICE_NAME_MAX]; + atomic_t dbq_intr_running; + u32 num_msix_requested; + unsigned char *dev_addr; /* For netdev->dev_addr */ +}; + +#define BNXT_RE_RESOLVE_RETRY_COUNT_US 5000000 /* 5 sec */ +struct bnxt_re_resolve_dmac_work{ + struct work_struct work; + struct list_head list; + struct bnxt_re_dev *rdev; + struct ib_ah_attr *ah_attr; + struct bnxt_re_ah_info *ah_info; + atomic_t status_wait; +}; + +static inline u8 bnxt_re_get_prio(u8 prio_map) +{ + u8 prio = 0xFF; + + for (prio = 0; prio < 8; prio++) + if (prio_map & (1UL << prio)) + break; + return prio; +} + +/* This should be called with bnxt_re_dev_lock mutex held */ +static inline bool __bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_dev *tmp_rdev; + + list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) { + if (rdev == tmp_rdev) + return true; + } + return false; +} + +static inline bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_dev *tmp_rdev; + + mutex_lock(&bnxt_re_dev_lock); + list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) { + if (rdev == tmp_rdev) { + mutex_unlock(&bnxt_re_dev_lock); + return true; + } + } + mutex_unlock(&bnxt_re_dev_lock); + + pr_debug("bnxt_re: %s : Invalid rdev received rdev = %p\n", + __func__, rdev); + return false; +} + +int bnxt_re_send_hwrm_cmd(struct bnxt_re_dev *rdev, void *cmd, + int cmdlen); +void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev); +int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev, + struct bnxt_re_dscp2pri *d2p, u16 count, + u16 target_id); +int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev, + struct bnxt_re_dscp2pri *d2p, u16 *count, + u16 target_id); +int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev, + struct bnxt_re_tc_rec *cnprec, u16 tid); +int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id, + struct bnxt_re_cos2bw_cfg *cfg); +int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id, + struct bnxt_re_cos2bw_cfg *cfg); +int bnxt_re_hwrm_pri2cos_cfg(struct bnxt_re_dev *rdev, + u16 target_id, u16 port_id, + u8 *cos_id_map, u8 pri_map); +int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev); +int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev); +struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev); +struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev); +u8 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev, u8 selector); +struct bnxt_qplib_nq * bnxt_re_get_nq(struct bnxt_re_dev *rdev); +void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq); + +#define to_bnxt_re(ptr, type, member) \ + container_of(ptr, type, member) + +#define to_bnxt_re_dev(ptr, member) \ + container_of((ptr), struct bnxt_re_dev, member) + +/* Even number functions from port 0 and odd number from port 1 */ +#define BNXT_RE_IS_PORT0(rdev) (!(rdev->en_dev->pdev->devfn & 1)) + +#define BNXT_RE_ROCE_V1_PACKET 0 +#define BNXT_RE_ROCEV2_IPV4_PACKET 2 +#define BNXT_RE_ROCEV2_IPV6_PACKET 3 +#define BNXT_RE_ACTIVE_MAP_PORT1 0x1 /*port-1 active */ +#define BNXT_RE_ACTIVE_MAP_PORT2 0x2 /*port-2 active */ + +#define BNXT_RE_MEMBER_PORT_MAP (BNXT_RE_ACTIVE_MAP_PORT1 | \ + BNXT_RE_ACTIVE_MAP_PORT2) + +#define rdev_to_dev(rdev) ((rdev) ? (&(rdev)->ibdev.dev) : NULL) + +void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev); +bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev); + +#define bnxt_re_rdev_ready(rdev) (bnxt_re_is_rdev_valid(rdev) && \ + (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))) +#define BNXT_RE_SRIOV_CFG_TIMEOUT 6 + +int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev); +void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 removal_type, + struct auxiliary_device *aux_dev); +void bnxt_re_destroy_lag(struct bnxt_re_dev **rdev); +int bnxt_re_add_device(struct bnxt_re_dev **rdev, + struct ifnet *netdev, + u8 qp_mode, u8 op_type, u8 wqe_mode, u32 num_msix_requested, + struct auxiliary_device *aux_dev); +void bnxt_re_create_base_interface(bool primary); +int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event, + struct ifnet *vlan_dev, + struct ifnet *netdev, + struct auxiliary_device *aux_dev); +void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev); +int _bnxt_re_ib_init(struct bnxt_re_dev *rdev); +int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev); +void bnxt_re_init_resolve_wq(struct bnxt_re_dev *rdev); +void bnxt_re_uninit_resolve_wq(struct bnxt_re_dev *rdev); + +/* The rdev ref_count is to protect immature removal of the device */ +static inline void bnxt_re_hold(struct bnxt_re_dev *rdev) +{ + atomic_inc(&rdev->ref_count); + dev_dbg(rdev_to_dev(rdev), + "Hold ref_count = 0x%x", atomic_read(&rdev->ref_count)); +} + +static inline void bnxt_re_put(struct bnxt_re_dev *rdev) +{ + atomic_dec(&rdev->ref_count); + dev_dbg(rdev_to_dev(rdev), + "Put ref_count = 0x%x", atomic_read(&rdev->ref_count)); +} + +/* +* Responder Error reason codes +* FIXME: Remove these when the defs +* are properly included in hsi header +*/ +enum res_err_state_reason { + /* No error. */ + CFCQ_RES_ERR_STATE_REASON_NO_ERROR = 0, + /* + * Incoming Send, RDMA write, or RDMA read exceeds the maximum + * transfer length. Detected on RX first and only packets for + * write. Detected on RX request for read. This is an RX + * Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX, + /* + * RDMA write payload size does not match write length. Detected + * when total write payload is not equal to the RDMA write + * length that was given in the first or only packet of the + * request. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH, + /* + * Send payload exceeds RQ/SRQ WQE buffer capacity. The total + * send payload that arrived is more than the size of the WQE + * buffer that was fetched from the RQ/SRQ. This is an RX + * Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE, + /* + * Responder detected opcode error. * First, only, middle, last + * for incoming requests are improperly ordered with respect to + * previous (PSN) packet. * First or middle packet is not full + * MTU size. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR, + /* + * PSN sequence error retry limit exceeded. The responder + * encountered a PSN sequence error for the same PSN too many + * times. This can occur via implicit or explicit NAK. This is + * an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT, + /* + * Invalid R_Key. An incoming request contained an R_Key that + * did not reference a valid MR/MW. This error may be detected + * by the RX engine for RDMA write or by the TX engine for RDMA + * read (detected while servicing IRRQ). This is an RX Detected + * Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY, + /* + * Domain error. An incoming request specified an R_Key which + * referenced a MR/MW that was not in the same PD as the QP on + * which the request arrived. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR, + /* + * No permission. An incoming request contained an R_Key that + * referenced a MR/MW which did not have the access permission + * needed for the operation. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION, + /* + * Range error. An incoming request had a combination of R_Key, + * VA, and length that was out of bounds of the associated + * MR/MW. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR, + /* + * Invalid R_Key. An incoming request contained an R_Key that + * did not reference a valid MR/MW. This error may be detected + * by the RX engine for RDMA write or by the TX engine for RDMA + * read (detected while servicing IRRQ). This is a TX Detected + * Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY, + /* + * Domain error. An incoming request specified an R_Key which + * referenced a MR/MW that was not in the same PD as the QP on + * which the request arrived. This is a TX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR, + /* + * No permission. An incoming request contained an R_Key that + * referenced a MR/MW which did not have the access permission + * needed for the operation. This is a TX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION, + /* + * Range error. An incoming request had a combination of R_Key, + * VA, and length that was out of bounds of the associated + * MR/MW. This is a TX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR, + /* + * IRRQ overflow. The peer sent us more RDMA read or atomic + * requests than the negotiated maximum. This is an RX Detected + * Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW, + /* + * Unsupported opcode. The peer sent us a request with an opcode + * for a request type that is not supported on this QP. This is + * an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE, + /* + * Unaligned atomic operation. The VA of an atomic request is on + * a memory boundary that prevents atomic execution. This is an + * RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC, + /* + * Remote invalidate error. A send with invalidate request + * arrived in which the R_Key to invalidate did not describe a + * MR/MW which could be invalidated. RQ WQE completes with error + * status. This error is only reported if the send operation did + * not fail. If the send operation failed then the remote + * invalidate error is not reported. This is an RX Detected + * Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE, + /* + * Local memory error. An RQ/SRQ SGE described an inaccessible + * memory. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR, + /* + * SRQ in error. The QP is moving to error state because it + * found SRQ it uses in error. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR, + /* + * Completion error. No CQE space available on queue or CQ not + * in VALID state. This is a Completion Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR, + /* + * Invalid R_Key while resending responses to duplicate request. + * This is a TX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY, + /* + * Problem was found in the format of a WQE in the RQ/SRQ. This + * is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR, + /* + * A load error occurred on an attempt to load the CQ Context. + * This is a Completion Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR = 0x18, + /* + * A load error occurred on an attempt to load the SRQ Context. + * This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR, + /* + * A fatal error was detected on an attempt to read from or + * write to PCIe on the transmit side. This error is detected by + * the TX side, but has the priority of a Completion Detected + * Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR = 0x1b, + /* + * A fatal error was detected on an attempt to read from or + * write to PCIe on the receive side. This error is detected by + * the RX side (or CAGR), but has the priority of a Completion + * Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR = 0x1c +}; + +int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev, + struct bnxt_qplib_query_fn_info *fn_info, + u32 *pf_mask, u32 *first_pf); + +/* Default DCBx and CC values */ +#define BNXT_RE_DEFAULT_CNP_DSCP 48 +#define BNXT_RE_DEFAULT_CNP_PRI 7 +#define BNXT_RE_DEFAULT_ROCE_DSCP 26 +#define BNXT_RE_DEFAULT_ROCE_PRI 3 + +#define BNXT_RE_DEFAULT_L2_BW 50 +#define BNXT_RE_DEFAULT_ROCE_BW 50 + +#define ROCE_PRIO_VALID 0x0 +#define CNP_PRIO_VALID 0x1 +#define ROCE_DSCP_VALID 0x0 +#define CNP_DSCP_VALID 0x1 + +int bnxt_re_get_pri_dscp_settings(struct bnxt_re_dev *rdev, + u16 target_id, + struct bnxt_re_tc_rec *tc_rec); + +int bnxt_re_setup_dscp(struct bnxt_re_dev *rdev); +int bnxt_re_clear_dscp(struct bnxt_re_dev *rdev); +int bnxt_re_setup_cnp_cos(struct bnxt_re_dev *rdev, bool reset); + +static inline enum ib_port_state bnxt_re_get_link_state(struct bnxt_re_dev *rdev) +{ + if (rdev->netdev->if_drv_flags & IFF_DRV_RUNNING && + rdev->netdev->if_link_state == LINK_STATE_UP) + return IB_PORT_ACTIVE; + return IB_PORT_DOWN; +} + +static inline int bnxt_re_link_state(struct bnxt_re_dev *rdev) +{ + return bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE ? 1:0; +} + +static inline int is_cc_enabled(struct bnxt_re_dev *rdev) +{ + return rdev->cc_param.enable; +} + +static inline void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, + struct input *hdr, u16 opcd, + u16 crid, u16 trid) +{ + hdr->req_type = cpu_to_le16(opcd); + hdr->cmpl_ring = cpu_to_le16(crid); + hdr->target_id = cpu_to_le16(trid); +} + +static inline void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, + void *msg, int msg_len, void *resp, + int resp_max_len, int timeout) +{ + fw_msg->msg = msg; + fw_msg->msg_len = msg_len; + fw_msg->resp = resp; + fw_msg->resp_max_len = resp_max_len; + fw_msg->timeout = timeout; +} + +static inline bool is_qport_service_type_supported(struct bnxt_re_dev *rdev) +{ + return rdev->tc_rec[0].serv_type_enabled; +} + +static inline bool is_bnxt_roce_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type) +{ + if (is_qport_service_type_supported(rdev)) + return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_ROCE); + else + return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS_ROCE); +} + +static inline bool is_bnxt_cnp_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type) +{ + if (is_qport_service_type_supported(rdev)) + return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_CNP); + else + return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSY_ROCE_CNP); +} + +#define BNXT_RE_MAP_SH_PAGE 0x0 +#define BNXT_RE_MAP_WC 0x1 +#define BNXT_RE_DBR_PAGE 0x2 +#define BNXT_RE_MAP_DB_RECOVERY_PAGE 0x3 + +#define BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT (20) /* 20 ms */ +#define BNXT_RE_DBR_INT_TIME 5 /* ms */ +#define BNXT_RE_PACING_EN_INT_THRESHOLD 50 /* Entries in DB FIFO */ +#define BNXT_RE_PACING_ALGO_THRESHOLD 250 /* Entries in DB FIFO */ +/* Percentage of DB FIFO depth */ +#define BNXT_RE_PACING_DBQ_THRESHOLD BNXT_RE_PACING_DBQ_HIGH_WATERMARK + +#define BNXT_RE_PACING_ALARM_TH_MULTIPLE(ctx) (_is_chip_p7(ctx) ? 0 : 2) + +/* + * Maximum Percentage of configurable DB FIFO depth. + * The Doorbell FIFO depth is 0x2c00. But the DBR_REG_DB_THROTTLING register has only 12 bits + * to program the high watermark. This means user can configure maximum 36% only(4095/11264). + */ +#define BNXT_RE_PACING_DBQ_HIGH_WATERMARK 36 + +/* Default do_pacing value when there is no congestion */ +#define BNXT_RE_DBR_DO_PACING_NO_CONGESTION 0x7F /* 1 in 512 probability */ + +enum { + BNXT_RE_DBQ_EVENT_SCHED = 0, + BNXT_RE_DBR_PACING_EVENT = 1, + BNXT_RE_DBR_NQ_PACING_NOTIFICATION = 2, +}; + +struct bnxt_re_dbq_work { + struct work_struct work; + struct bnxt_re_dev *rdev; + struct hwrm_async_event_cmpl cmpl; + u32 event; +}; + +int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev); +int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev); +int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev); +int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, + u16 nq_id, u32 throttle); +void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev); +int bnxt_re_hwrm_pri2cos_qcfg(struct bnxt_re_dev *rdev, struct bnxt_re_tc_rec *tc_rec, + u16 target_id); +void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32); +u32 readl_fbsd(struct bnxt_softc *bp, u32, u8); + +static inline unsigned int bnxt_re_get_total_mr_mw_count(struct bnxt_re_dev *rdev) +{ + return (atomic_read(&rdev->stats.rsors.mr_count) + + atomic_read(&rdev->stats.rsors.mw_count)); +} + +static inline void bnxt_re_set_def_pacing_threshold(struct bnxt_re_dev *rdev) +{ + rdev->qplib_res.pacing_data->pacing_th = rdev->pacing_algo_th; + rdev->qplib_res.pacing_data->alarm_th = + rdev->pacing_algo_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); +} + +static inline void bnxt_re_set_def_do_pacing(struct bnxt_re_dev *rdev) +{ + rdev->qplib_res.pacing_data->do_pacing = rdev->dbr_def_do_pacing; +} + +static inline void bnxt_re_set_pacing_dev_state(struct bnxt_re_dev *rdev) +{ + rdev->qplib_res.pacing_data->dev_err_state = + test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); +} +#endif diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.c b/sys/dev/bnxt/bnxt_re/ib_verbs.c new file mode 100644 index 00000000000..8d43fa96c04 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/ib_verbs.c @@ -0,0 +1,5498 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: IB Verbs interpreter + */ + +#include +#include +#include + +#include "bnxt_re.h" +#include "ib_verbs.h" + +static inline +struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap) +{ + + *nmap = umem->nmap; + return umem->sg_head.sgl; +} + +static inline void bnxt_re_peer_mem_release(struct ib_umem *umem) +{ + dev_dbg(NULL, "ib_umem_release getting invoked \n"); + ib_umem_release(umem); +} + +void bnxt_re_resolve_dmac_task(struct work_struct *work) +{ + int rc = -1; + struct bnxt_re_dev *rdev; + struct ib_ah_attr *ah_attr; + struct bnxt_re_resolve_dmac_work *dmac_work = + container_of(work, struct bnxt_re_resolve_dmac_work, work); + + rdev = dmac_work->rdev; + ah_attr = dmac_work->ah_attr; + rc = ib_resolve_eth_dmac(&rdev->ibdev, ah_attr); + if (rc) + dev_err(rdev_to_dev(dmac_work->rdev), + "Failed to resolve dest mac rc = %d\n", rc); + atomic_set(&dmac_work->status_wait, rc << 8); +} + +static int __from_ib_access_flags(int iflags) +{ + int qflags = 0; + + if (iflags & IB_ACCESS_LOCAL_WRITE) + qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; + if (iflags & IB_ACCESS_REMOTE_READ) + qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; + if (iflags & IB_ACCESS_REMOTE_WRITE) + qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; + if (iflags & IB_ACCESS_REMOTE_ATOMIC) + qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; + if (iflags & IB_ACCESS_MW_BIND) + qflags |= BNXT_QPLIB_ACCESS_MW_BIND; + if (iflags & IB_ZERO_BASED) + qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; + if (iflags & IB_ACCESS_ON_DEMAND) + qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; + return qflags; +}; + +static enum ib_access_flags __to_ib_access_flags(int qflags) +{ + enum ib_access_flags iflags = 0; + + if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) + iflags |= IB_ACCESS_LOCAL_WRITE; + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) + iflags |= IB_ACCESS_REMOTE_WRITE; + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) + iflags |= IB_ACCESS_REMOTE_READ; + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) + iflags |= IB_ACCESS_REMOTE_ATOMIC; + if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) + iflags |= IB_ACCESS_MW_BIND; + if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) + iflags |= IB_ZERO_BASED; + if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) + iflags |= IB_ACCESS_ON_DEMAND; + return iflags; +}; + +static int bnxt_re_copy_to_udata(struct bnxt_re_dev *rdev, void *data, + int len, struct ib_udata *udata) +{ + int rc; + + rc = ib_copy_to_udata(udata, data, len); + if (rc) + dev_err(rdev_to_dev(rdev), + "ucontext copy failed from %ps rc %d\n", + __builtin_return_address(0), rc); + + return rc; +} + +struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev, + u8 port_num) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct ifnet *netdev = NULL; + + rcu_read_lock(); + + if (!rdev || !rdev->netdev) + goto end; + + netdev = rdev->netdev; + + /* In case of active-backup bond mode, return active slave */ + if (netdev) + dev_hold(netdev); + +end: + rcu_read_unlock(); + return netdev; +} + +int bnxt_re_query_device(struct ib_device *ibdev, + struct ib_device_attr *ib_attr, + struct ib_udata *udata) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; + + memset(ib_attr, 0, sizeof(*ib_attr)); + + memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 4); + bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ib_attr->sys_image_guid); + ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; + ib_attr->page_size_cap = dev_attr->page_size_cap; + ib_attr->vendor_id = rdev->en_dev->pdev->vendor; + ib_attr->vendor_part_id = rdev->en_dev->pdev->device; + ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; + ib_attr->max_qp = dev_attr->max_qp; + ib_attr->max_qp_wr = dev_attr->max_qp_wqes; + /* + * Read and set from the module param 'min_tx_depth' + * only once after the driver load + */ + if (rdev->min_tx_depth == 1 && + min_tx_depth < dev_attr->max_qp_wqes) + rdev->min_tx_depth = min_tx_depth; + ib_attr->device_cap_flags = + IB_DEVICE_CURR_QP_STATE_MOD + | IB_DEVICE_RC_RNR_NAK_GEN + | IB_DEVICE_SHUTDOWN_PORT + | IB_DEVICE_SYS_IMAGE_GUID + | IB_DEVICE_LOCAL_DMA_LKEY + | IB_DEVICE_RESIZE_MAX_WR + | IB_DEVICE_PORT_ACTIVE_EVENT + | IB_DEVICE_N_NOTIFY_CQ + | IB_DEVICE_MEM_WINDOW + | IB_DEVICE_MEM_WINDOW_TYPE_2B + | IB_DEVICE_MEM_MGT_EXTENSIONS; + ib_attr->max_send_sge = dev_attr->max_qp_sges; + ib_attr->max_recv_sge = dev_attr->max_qp_sges; + ib_attr->max_sge_rd = dev_attr->max_qp_sges; + ib_attr->max_cq = dev_attr->max_cq; + ib_attr->max_cqe = dev_attr->max_cq_wqes; + ib_attr->max_mr = dev_attr->max_mr; + ib_attr->max_pd = dev_attr->max_pd; + ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; + ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; + if (dev_attr->is_atomic) { + ib_attr->atomic_cap = IB_ATOMIC_GLOB; + ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB; + } + ib_attr->max_ee_rd_atom = 0; + ib_attr->max_res_rd_atom = 0; + ib_attr->max_ee_init_rd_atom = 0; + ib_attr->max_ee = 0; + ib_attr->max_rdd = 0; + ib_attr->max_mw = dev_attr->max_mw; + ib_attr->max_raw_ipv6_qp = 0; + ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; + ib_attr->max_mcast_grp = 0; + ib_attr->max_mcast_qp_attach = 0; + ib_attr->max_total_mcast_qp_attach = 0; + ib_attr->max_ah = dev_attr->max_ah; + ib_attr->max_srq = dev_attr->max_srq; + ib_attr->max_srq_wr = dev_attr->max_srq_wqes; + ib_attr->max_srq_sge = dev_attr->max_srq_sges; + + ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; + ib_attr->max_pkeys = 1; + ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; + ib_attr->sig_prot_cap = 0; + ib_attr->sig_guard_cap = 0; + ib_attr->odp_caps.general_caps = 0; + + return 0; +} + +int bnxt_re_modify_device(struct ib_device *ibdev, + int device_modify_mask, + struct ib_device_modify *device_modify) +{ + dev_dbg(rdev_to_dev(rdev), "Modify device with mask 0x%x\n", + device_modify_mask); + + switch (device_modify_mask) { + case IB_DEVICE_MODIFY_SYS_IMAGE_GUID: + /* Modify the GUID requires the modification of the GID table */ + /* GUID should be made as READ-ONLY */ + break; + case IB_DEVICE_MODIFY_NODE_DESC: + /* Node Desc should be made as READ-ONLY */ + break; + default: + break; + } + return 0; +} + +static void __to_ib_speed_width(u32 espeed, u8 *speed, u8 *width) +{ + switch (espeed) { + case SPEED_1000: + *speed = IB_SPEED_SDR; + *width = IB_WIDTH_1X; + break; + case SPEED_10000: + *speed = IB_SPEED_QDR; + *width = IB_WIDTH_1X; + break; + case SPEED_20000: + *speed = IB_SPEED_DDR; + *width = IB_WIDTH_4X; + break; + case SPEED_25000: + *speed = IB_SPEED_EDR; + *width = IB_WIDTH_1X; + break; + case SPEED_40000: + *speed = IB_SPEED_QDR; + *width = IB_WIDTH_4X; + break; + case SPEED_50000: + *speed = IB_SPEED_EDR; + *width = IB_WIDTH_2X; + break; + case SPEED_100000: + *speed = IB_SPEED_EDR; + *width = IB_WIDTH_4X; + break; + case SPEED_200000: + *speed = IB_SPEED_HDR; + *width = IB_WIDTH_4X; + break; + default: + *speed = IB_SPEED_SDR; + *width = IB_WIDTH_1X; + break; + } +} + +/* Port */ +int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, + struct ib_port_attr *port_attr) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; + u8 active_speed = 0, active_width = 0; + + dev_dbg(rdev_to_dev(rdev), "QUERY PORT with port_num 0x%x\n", port_num); + memset(port_attr, 0, sizeof(*port_attr)); + + port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; + port_attr->state = bnxt_re_get_link_state(rdev); + if (port_attr->state == IB_PORT_ACTIVE) + port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; + port_attr->max_mtu = IB_MTU_4096; + port_attr->active_mtu = iboe_get_mtu(rdev->netdev->if_mtu); + port_attr->gid_tbl_len = dev_attr->max_sgid; + port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | + IB_PORT_DEVICE_MGMT_SUP | + IB_PORT_VENDOR_CLASS_SUP | + IB_PORT_IP_BASED_GIDS; + + port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; + port_attr->bad_pkey_cntr = 0; + port_attr->qkey_viol_cntr = 0; + port_attr->pkey_tbl_len = dev_attr->max_pkey; + port_attr->lid = 0; + port_attr->sm_lid = 0; + port_attr->lmc = 0; + port_attr->max_vl_num = 4; + port_attr->sm_sl = 0; + port_attr->subnet_timeout = 0; + port_attr->init_type_reply = 0; + rdev->espeed = rdev->en_dev->espeed; + + if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) + __to_ib_speed_width(rdev->espeed, &active_speed, + &active_width); + + port_attr->active_speed = active_speed; + port_attr->active_width = active_width; + + return 0; +} + +int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, + int port_modify_mask, + struct ib_port_modify *port_modify) +{ + dev_dbg(rdev_to_dev(rdev), "Modify port with mask 0x%x\n", + port_modify_mask); + + switch (port_modify_mask) { + case IB_PORT_SHUTDOWN: + break; + case IB_PORT_INIT_TYPE: + break; + case IB_PORT_RESET_QKEY_CNTR: + break; + default: + break; + } + return 0; +} + +int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct ib_port_attr port_attr; + + if (bnxt_re_query_port(ibdev, port_num, &port_attr)) + return -EINVAL; + + immutable->pkey_tbl_len = port_attr.pkey_tbl_len; + immutable->gid_tbl_len = port_attr.gid_tbl_len; + if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV1_CAP) + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; + else if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV2_CAP) + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; + else + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE | + RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + return 0; +} + +void bnxt_re_compat_qfwstr(void) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + + sprintf(str, "%d.%d.%d.%d", rdev->dev_attr->fw_ver[0], + rdev->dev_attr->fw_ver[1], rdev->dev_attr->fw_ver[2], + rdev->dev_attr->fw_ver[3]); +} + +int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, + u16 index, u16 *pkey) +{ + if (index > 0) + return -EINVAL; + + *pkey = IB_DEFAULT_PKEY_FULL; + + return 0; +} + +int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, + int index, union ib_gid *gid) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + int rc = 0; + + /* Ignore port_num */ + memset(gid, 0, sizeof(*gid)); + rc = bnxt_qplib_get_sgid(&rdev->qplib_res, + &rdev->qplib_res.sgid_tbl, index, + (struct bnxt_qplib_gid *)gid); + return rc; +} + +int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, + unsigned int index, void **context) +{ + int rc = 0; + struct bnxt_re_gid_ctx *ctx, **ctx_tbl; + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; + struct bnxt_qplib_gid *gid_to_del; + u16 vlan_id = 0xFFFF; + + /* Delete the entry from the hardware */ + ctx = *context; + if (!ctx) { + dev_err(rdev_to_dev(rdev), "GID entry has no ctx?!\n"); + return -EINVAL; + } + if (sgid_tbl && sgid_tbl->active) { + if (ctx->idx >= sgid_tbl->max) { + dev_dbg(rdev_to_dev(rdev), "GID index out of range?!\n"); + return -EINVAL; + } + gid_to_del = &sgid_tbl->tbl[ctx->idx].gid; + vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id; + ctx->refcnt--; + /* DEL_GID is called via WQ context(netdevice_event_work_handler) + * or via the ib_unregister_device path. In the former case QP1 + * may not be destroyed yet, in which case just return as FW + * needs that entry to be present and will fail it's deletion. + * We could get invoked again after QP1 is destroyed OR get an + * ADD_GID call with a different GID value for the same index + * where we issue MODIFY_GID cmd to update the GID entry -- TBD + */ + if (ctx->idx == 0 && + rdma_link_local_addr((struct in6_addr *)gid_to_del) && + (rdev->gsi_ctx.gsi_sqp || + rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)) { + dev_dbg(rdev_to_dev(rdev), + "Trying to delete GID0 while QP1 is alive\n"); + if (!ctx->refcnt) { + rdev->gid_map[index] = -1; + ctx_tbl = sgid_tbl->ctx; + ctx_tbl[ctx->idx] = NULL; + kfree(ctx); + } + return 0; + } + rdev->gid_map[index] = -1; + if (!ctx->refcnt) { + rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, + vlan_id, true); + if (!rc) { + dev_dbg(rdev_to_dev(rdev), "GID remove success\n"); + ctx_tbl = sgid_tbl->ctx; + ctx_tbl[ctx->idx] = NULL; + kfree(ctx); + } else { + dev_err(rdev_to_dev(rdev), + "Remove GID failed rc = 0x%x\n", rc); + } + } + } else { + dev_dbg(rdev_to_dev(rdev), "GID sgid_tbl does not exist!\n"); + return -EINVAL; + } + return rc; +} + +int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, + unsigned int index, const union ib_gid *gid, + const struct ib_gid_attr *attr, void **context) +{ + int rc; + u32 tbl_idx = 0; + u16 vlan_id = 0xFFFF; + struct bnxt_re_gid_ctx *ctx, **ctx_tbl; + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; + if ((attr->ndev) && is_vlan_dev(attr->ndev)) + vlan_id = vlan_dev_vlan_id(attr->ndev); + + rc = bnxt_qplib_add_sgid(sgid_tbl, gid, + rdev->dev_addr, + vlan_id, true, &tbl_idx); + if (rc == -EALREADY) { + dev_dbg(rdev_to_dev(rdev), "GID %pI6 is already present\n", gid); + ctx_tbl = sgid_tbl->ctx; + if (!ctx_tbl[tbl_idx]) { + ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + ctx->idx = tbl_idx; + ctx->refcnt = 1; + ctx_tbl[tbl_idx] = ctx; + } else { + ctx_tbl[tbl_idx]->refcnt++; + } + *context = ctx_tbl[tbl_idx]; + /* tbl_idx is the HW table index and index is the stack index */ + rdev->gid_map[index] = tbl_idx; + return 0; + } else if (rc < 0) { + dev_err(rdev_to_dev(rdev), "Add GID failed rc = 0x%x\n", rc); + return rc; + } else { + ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) { + dev_err(rdev_to_dev(rdev), "Add GID ctx failed\n"); + return -ENOMEM; + } + ctx_tbl = sgid_tbl->ctx; + ctx->idx = tbl_idx; + ctx->refcnt = 1; + ctx_tbl[tbl_idx] = ctx; + /* tbl_idx is the HW table index and index is the stack index */ + rdev->gid_map[index] = tbl_idx; + *context = ctx; + } + return rc; +} + +enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, + u8 port_num) +{ + return IB_LINK_LAYER_ETHERNET; +} + +static void bnxt_re_legacy_create_fence_wqe(struct bnxt_re_pd *pd) +{ + struct bnxt_re_legacy_fence_data *fence = &pd->fence; + struct ib_mr *ib_mr = &fence->mr->ib_mr; + struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; + struct bnxt_re_dev *rdev = pd->rdev; + + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + return; + + memset(wqe, 0, sizeof(*wqe)); + wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; + wqe->wr_id = BNXT_QPLIB_FENCE_WRID; + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + wqe->bind.zero_based = false; + wqe->bind.parent_l_key = ib_mr->lkey; + wqe->bind.va = (u64)fence->va; + wqe->bind.length = fence->size; + wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); + wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; + + /* Save the initial rkey in fence structure for now; + * wqe->bind.r_key will be set at (re)bind time. + */ + fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); +} + +static int bnxt_re_legacy_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp) +{ + struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, + qplib_qp); + struct ib_pd *ib_pd = qp->ib_qp.pd; + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_legacy_fence_data *fence = &pd->fence; + struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; + struct bnxt_qplib_swqe wqe; + int rc; + + /* TODO: Need SQ locking here when Fence WQE + * posting moves up into bnxt_re from bnxt_qplib. + */ + memcpy(&wqe, fence_wqe, sizeof(wqe)); + wqe.bind.r_key = fence->bind_rkey; + fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); + + dev_dbg(rdev_to_dev(qp->rdev), + "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", + wqe.bind.r_key, qp->qplib_qp.id, pd); + rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); + if (rc) { + dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n"); + return rc; + } + bnxt_qplib_post_send_db(&qp->qplib_qp); + + return rc; +} + +static int bnxt_re_legacy_create_fence_mr(struct bnxt_re_pd *pd) +{ + int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND; + struct bnxt_re_legacy_fence_data *fence = &pd->fence; + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_qplib_mrinfo mrinfo; + struct bnxt_re_mr *mr = NULL; + struct ib_mw *ib_mw = NULL; + dma_addr_t dma_addr = 0; + u32 max_mr_count; + u64 pbl_tbl; + int rc; + + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + return 0; + + memset(&mrinfo, 0, sizeof(mrinfo)); + /* Allocate a small chunk of memory and dma-map it */ + fence->va = kzalloc(BNXT_RE_LEGACY_FENCE_BYTES, GFP_KERNEL); + if (!fence->va) + return -ENOMEM; + dma_addr = ib_dma_map_single(&rdev->ibdev, fence->va, + BNXT_RE_LEGACY_FENCE_BYTES, + DMA_BIDIRECTIONAL); + rc = ib_dma_mapping_error(&rdev->ibdev, dma_addr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n"); + rc = -EIO; + fence->dma_addr = 0; + goto free_va; + } + fence->dma_addr = dma_addr; + + /* Allocate a MR */ + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + goto free_dma_addr; + fence->mr = mr; + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); + if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n"); + goto free_mr; + } + /* Register MR */ + mr->ib_mr.lkey = mr->qplib_mr.lkey; + } + mr->qplib_mr.va = (u64)fence->va; + mr->qplib_mr.total_size = BNXT_RE_LEGACY_FENCE_BYTES; + pbl_tbl = dma_addr; + + mrinfo.mrw = &mr->qplib_mr; + mrinfo.ptes = &pbl_tbl; + mrinfo.sg.npages = BNXT_RE_LEGACY_FENCE_PBL_SIZE; + + mrinfo.sg.nmap = 0; + mrinfo.sg.sghead = 0; + mrinfo.sg.pgshft = PAGE_SHIFT; + mrinfo.sg.pgsize = PAGE_SIZE; + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n"); + goto free_mr; + } + mr->ib_mr.lkey = mr->qplib_mr.lkey; + mr->ib_mr.rkey = mr->qplib_mr.rkey; + atomic_inc(&rdev->stats.rsors.mr_count); + max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); + if (max_mr_count > (atomic_read(&rdev->stats.rsors.max_mr_count))) + atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); + + ib_mw = bnxt_re_alloc_mw(&pd->ibpd, IB_MW_TYPE_1, NULL); + /* Create a fence MW only for kernel consumers */ + if (!ib_mw) { + dev_err(rdev_to_dev(rdev), + "Failed to create fence-MW for PD: %p\n", pd); + rc = -EINVAL; + goto free_mr; + } + fence->mw = ib_mw; + + bnxt_re_legacy_create_fence_wqe(pd); + return 0; + +free_mr: + if (mr->ib_mr.lkey) { + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); + atomic_dec(&rdev->stats.rsors.mr_count); + } + kfree(mr); + fence->mr = NULL; + +free_dma_addr: + ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, + BNXT_RE_LEGACY_FENCE_BYTES, DMA_BIDIRECTIONAL); + fence->dma_addr = 0; + +free_va: + kfree(fence->va); + fence->va = NULL; + return rc; +} + +static void bnxt_re_legacy_destroy_fence_mr(struct bnxt_re_pd *pd) +{ + struct bnxt_re_legacy_fence_data *fence = &pd->fence; + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_mr *mr = fence->mr; + + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + return; + + if (fence->mw) { + bnxt_re_dealloc_mw(fence->mw); + fence->mw = NULL; + } + if (mr) { + if (mr->ib_mr.rkey) + bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, + false); + if (mr->ib_mr.lkey) + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); + kfree(mr); + fence->mr = NULL; + atomic_dec(&rdev->stats.rsors.mr_count); + } + if (fence->dma_addr) { + ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, + BNXT_RE_LEGACY_FENCE_BYTES, + DMA_BIDIRECTIONAL); + fence->dma_addr = 0; + } + kfree(fence->va); + fence->va = NULL; +} + + +static int bnxt_re_get_user_dpi(struct bnxt_re_dev *rdev, + struct bnxt_re_ucontext *cntx) +{ + struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; + int ret = 0; + u8 type; + /* Allocate DPI in alloc_pd or in create_cq to avoid failing of + * ibv_devinfo and family of application when DPIs are depleted. + */ + type = BNXT_QPLIB_DPI_TYPE_UC; + ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->dpi, cntx, type); + if (ret) { + dev_err(rdev_to_dev(rdev), "Alloc doorbell page failed!\n"); + goto out; + } + + if (cctx->modes.db_push) { + type = BNXT_QPLIB_DPI_TYPE_WC; + ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->wcdpi, + cntx, type); + if (ret) + dev_err(rdev_to_dev(rdev), "push dp alloc failed\n"); + } +out: + return ret; +} + +/* Protection Domains */ +void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_dev *rdev = pd->rdev; + int rc; + + bnxt_re_legacy_destroy_fence_mr(pd); + + rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res, + &rdev->qplib_res.pd_tbl, + &pd->qplib_pd); + if (rc) + dev_err_ratelimited(rdev_to_dev(rdev), + "%s failed rc = %d\n", __func__, rc); + atomic_dec(&rdev->stats.rsors.pd_count); + + return; +} + +int bnxt_re_alloc_pd(struct ib_pd *pd_in, + struct ib_udata *udata) +{ + struct ib_pd *ibpd = pd_in; + struct ib_device *ibdev = ibpd->device; + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_re_ucontext *ucntx = + rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, + ibucontext); + u32 max_pd_count; + int rc; + struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ibpd); + + pd->rdev = rdev; + if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) { + dev_err(rdev_to_dev(rdev), + "Allocate HW Protection Domain failed!\n"); + rc = -ENOMEM; + goto fail; + } + + if (udata) { + struct bnxt_re_pd_resp resp = {}; + + if (!ucntx->dpi.dbr) { + rc = bnxt_re_get_user_dpi(rdev, ucntx); + if (rc) + goto dbfail; + } + + resp.pdid = pd->qplib_pd.id; + /* Still allow mapping this DBR to the new user PD. */ + resp.dpi = ucntx->dpi.dpi; + resp.dbr = (u64)ucntx->dpi.umdbr; + /* Copy only on a valid wcpdi */ + if (ucntx->wcdpi.dpi) { + resp.wcdpi = ucntx->wcdpi.dpi; + resp.comp_mask = BNXT_RE_COMP_MASK_PD_HAS_WC_DPI; + } + if (rdev->dbr_pacing) { + WARN_ON(!rdev->dbr_bar_addr); + resp.dbr_bar_addr = (u64)rdev->dbr_bar_addr; + resp.comp_mask |= BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR; + } + + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) + goto dbfail; + } + + if (!udata) + if (bnxt_re_legacy_create_fence_mr(pd)) + dev_warn(rdev_to_dev(rdev), + "Failed to create Fence-MR\n"); + + atomic_inc(&rdev->stats.rsors.pd_count); + max_pd_count = atomic_read(&rdev->stats.rsors.pd_count); + if (max_pd_count > atomic_read(&rdev->stats.rsors.max_pd_count)) + atomic_set(&rdev->stats.rsors.max_pd_count, max_pd_count); + + return 0; +dbfail: + (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, + &pd->qplib_pd); +fail: + return rc; +} + +/* Address Handles */ +void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) +{ + struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah); + struct bnxt_re_dev *rdev = ah->rdev; + int rc = 0; + bool block = true; + + block = !(flags & RDMA_DESTROY_AH_SLEEPABLE); + + rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block); + if (rc) + dev_err_ratelimited(rdev_to_dev(rdev), + "%s id = %d blocking %d failed rc = %d\n", + __func__, ah->qplib_ah.id, block, rc); + atomic_dec(&rdev->stats.rsors.ah_count); + + return; +} + +static u8 _to_bnxt_re_nw_type(enum rdma_network_type ntype) +{ + u8 nw_type; + switch (ntype) { + case RDMA_NETWORK_IPV4: + nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; + break; + case RDMA_NETWORK_IPV6: + nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; + break; + default: + nw_type = CMDQ_CREATE_AH_TYPE_V1; + break; + } + return nw_type; +} + +static inline int +bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index, + union ib_gid *sgid, struct ib_gid_attr **sgid_attr, + struct ib_global_route *grh, struct ib_ah *ah) +{ + int ret = 0; + + ret = ib_get_cached_gid(dev, port_num, index, sgid, *sgid_attr); + return ret; +} + +static inline enum rdma_network_type +bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr, + union ib_gid *sgid) +{ + return ib_gid_to_network_type(sgid_attr->gid_type, sgid); +} + +static int bnxt_re_get_ah_info(struct bnxt_re_dev *rdev, + struct ib_ah_attr *ah_attr, + struct bnxt_re_ah_info *ah_info) +{ + struct ib_gid_attr *gattr; + enum rdma_network_type ib_ntype; + u8 ntype; + union ib_gid *gid; + int rc = 0; + + gid = &ah_info->sgid; + gattr = &ah_info->sgid_attr; + + rc = bnxt_re_get_cached_gid(&rdev->ibdev, 1, ah_attr->grh.sgid_index, + gid, &gattr, &ah_attr->grh, NULL); + if (rc) + return rc; + + /* Get vlan tag */ + if (gattr->ndev) { + if (is_vlan_dev(gattr->ndev)) + ah_info->vlan_tag = vlan_dev_vlan_id(gattr->ndev); + if_rele(gattr->ndev); + } + + /* Get network header type for this GID */ + + ib_ntype = bnxt_re_gid_to_network_type(gattr, gid); + ntype = _to_bnxt_re_nw_type(ib_ntype); + ah_info->nw_type = ntype; + + return rc; +} + +static u8 _get_sgid_index(struct bnxt_re_dev *rdev, u8 gindx) +{ + gindx = rdev->gid_map[gindx]; + return gindx; +} + +static int bnxt_re_init_dmac(struct bnxt_re_dev *rdev, struct ib_ah_attr *ah_attr, + struct bnxt_re_ah_info *ah_info, bool is_user, + struct bnxt_re_ah *ah) +{ + int rc = 0; + u8 *dmac; + + if (is_user && !rdma_is_multicast_addr((struct in6_addr *) + ah_attr->grh.dgid.raw) && + !rdma_link_local_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) { + + u32 retry_count = BNXT_RE_RESOLVE_RETRY_COUNT_US; + struct bnxt_re_resolve_dmac_work *resolve_dmac_work; + + + resolve_dmac_work = kzalloc(sizeof(*resolve_dmac_work), GFP_ATOMIC); + + resolve_dmac_work->rdev = rdev; + resolve_dmac_work->ah_attr = ah_attr; + resolve_dmac_work->ah_info = ah_info; + + atomic_set(&resolve_dmac_work->status_wait, 1); + INIT_WORK(&resolve_dmac_work->work, bnxt_re_resolve_dmac_task); + queue_work(rdev->resolve_wq, &resolve_dmac_work->work); + + do { + rc = atomic_read(&resolve_dmac_work->status_wait) & 0xFF; + if (!rc) + break; + udelay(1); + } while (--retry_count); + if (atomic_read(&resolve_dmac_work->status_wait)) { + INIT_LIST_HEAD(&resolve_dmac_work->list); + list_add_tail(&resolve_dmac_work->list, + &rdev->mac_wq_list); + return -EFAULT; + } + kfree(resolve_dmac_work); + } + dmac = ROCE_DMAC(ah_attr); + if (dmac) + memcpy(ah->qplib_ah.dmac, dmac, ETH_ALEN); + return rc; +} + +int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr, + u32 flags, struct ib_udata *udata) +{ + + struct ib_ah *ib_ah = ah_in; + struct ib_pd *ib_pd = ib_ah->pd; + struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ibah); + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_ah_info ah_info; + u32 max_ah_count; + bool is_user; + int rc; + bool block = true; + struct ib_ah_attr *ah_attr = attr; + block = !(flags & RDMA_CREATE_AH_SLEEPABLE); + + if (!(ah_attr->ah_flags & IB_AH_GRH)) + dev_err(rdev_to_dev(rdev), "ah_attr->ah_flags GRH is not set\n"); + + ah->rdev = rdev; + ah->qplib_ah.pd = &pd->qplib_pd; + is_user = ib_pd->uobject ? true : false; + + /* Supply the configuration for the HW */ + memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw, + sizeof(union ib_gid)); + ah->qplib_ah.sgid_index = _get_sgid_index(rdev, ah_attr->grh.sgid_index); + if (ah->qplib_ah.sgid_index == 0xFF) { + dev_err(rdev_to_dev(rdev), "invalid sgid_index!\n"); + rc = -EINVAL; + goto fail; + } + ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index; + ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class; + ah->qplib_ah.flow_label = ah_attr->grh.flow_label; + ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit; + ah->qplib_ah.sl = ah_attr->sl; + rc = bnxt_re_get_ah_info(rdev, ah_attr, &ah_info); + if (rc) + goto fail; + ah->qplib_ah.nw_type = ah_info.nw_type; + + rc = bnxt_re_init_dmac(rdev, ah_attr, &ah_info, is_user, ah); + if (rc) + goto fail; + + rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, block); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Allocate HW Address Handle failed!\n"); + goto fail; + } + + /* Write AVID to shared page. */ + if (ib_pd->uobject) { + struct ib_ucontext *ib_uctx = ib_pd->uobject->context; + struct bnxt_re_ucontext *uctx; + unsigned long flag; + u32 *wrptr; + + uctx = to_bnxt_re(ib_uctx, struct bnxt_re_ucontext, ibucontext); + spin_lock_irqsave(&uctx->sh_lock, flag); + wrptr = (u32 *)((u8 *)uctx->shpg + BNXT_RE_AVID_OFFT); + *wrptr = ah->qplib_ah.id; + wmb(); /* make sure cache is updated. */ + spin_unlock_irqrestore(&uctx->sh_lock, flag); + } + atomic_inc(&rdev->stats.rsors.ah_count); + max_ah_count = atomic_read(&rdev->stats.rsors.ah_count); + if (max_ah_count > atomic_read(&rdev->stats.rsors.max_ah_count)) + atomic_set(&rdev->stats.rsors.max_ah_count, max_ah_count); + + return 0; +fail: + return rc; +} + +int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) +{ + return 0; +} + +int bnxt_re_query_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) +{ + struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah); + + memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data, + sizeof(union ib_gid)); + ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index; + ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class; + ah_attr->sl = ah->qplib_ah.sl; + memcpy(ROCE_DMAC(ah_attr), ah->qplib_ah.dmac, ETH_ALEN); + ah_attr->ah_flags = IB_AH_GRH; + ah_attr->port_num = 1; + ah_attr->static_rate = 0; + + return 0; +} + +/* Shared Receive Queues */ +void bnxt_re_destroy_srq(struct ib_srq *ib_srq, + struct ib_udata *udata) +{ + struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, ibsrq); + struct bnxt_re_dev *rdev = srq->rdev; + struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; + int rc = 0; + + + rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); + if (rc) + dev_err_ratelimited(rdev_to_dev(rdev), + "%s id = %d failed rc = %d\n", + __func__, qplib_srq->id, rc); + + if (srq->umem && !IS_ERR(srq->umem)) + ib_umem_release(srq->umem); + + atomic_dec(&rdev->stats.rsors.srq_count); + + return; +} + +static u16 _max_rwqe_sz(int nsge) +{ + return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge)); +} + +static u16 bnxt_re_get_rwqe_size(struct bnxt_qplib_qp *qplqp, + int rsge, int max) +{ + if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + rsge = max; + + return _max_rwqe_sz(rsge); +} + +static inline +struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev, + struct ib_ucontext *ucontext, + struct ib_udata *udata, + unsigned long addr, + size_t size, int access, int dmasync) +{ + return ib_umem_get(ucontext, addr, size, access, dmasync); +} + +static inline +struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev, + struct ib_ucontext *ucontext, + struct ib_udata *udata, + unsigned long addr, + size_t size, int access, int dmasync) +{ + return ib_umem_get_compat(rdev, ucontext, udata, addr, size, + access, 0); +} + +static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem) +{ + return ib_umem_num_pages(umem); +} + +static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, + struct bnxt_re_pd *pd, + struct bnxt_re_srq *srq, + struct ib_udata *udata) +{ + struct bnxt_qplib_sg_info *sginfo; + struct bnxt_qplib_srq *qplib_srq; + struct bnxt_re_ucontext *cntx; + struct ib_ucontext *context; + struct bnxt_re_srq_req ureq; + struct ib_umem *umem; + int rc, bytes = 0; + + context = pd->ibpd.uobject->context; + cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); + qplib_srq = &srq->qplib_srq; + sginfo = &qplib_srq->sginfo; + + if (udata->inlen < sizeof(ureq)) + dev_warn(rdev_to_dev(rdev), + "Update the library ulen %d klen %d\n", + (unsigned int)udata->inlen, + (unsigned int)sizeof(ureq)); + + rc = ib_copy_from_udata(&ureq, udata, + min(udata->inlen, sizeof(ureq))); + if (rc) + return rc; + + bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size); + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get_compat(rdev, context, udata, ureq.srqva, bytes, + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(umem)) { + dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n", + __func__, PTR_ERR(umem)); + return PTR_ERR(umem); + } + + srq->umem = umem; + sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); + sginfo->npages = ib_umem_num_pages_compat(umem); + qplib_srq->srq_handle = ureq.srq_handle; + qplib_srq->dpi = &cntx->dpi; + qplib_srq->is_user = true; + + return 0; +} + +int bnxt_re_create_srq(struct ib_srq *srq_in, struct ib_srq_init_attr *srq_init_attr, + struct ib_udata *udata) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_re_ucontext *cntx = NULL; + struct ib_ucontext *context; + struct bnxt_re_dev *rdev; + struct bnxt_re_pd *pd; + int rc, entries; + struct ib_srq *ib_srq = srq_in; + struct ib_pd *ib_pd = ib_srq->pd; + struct bnxt_re_srq *srq = + container_of(ib_srq, struct bnxt_re_srq, ibsrq); + u32 max_srq_count; + + pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + rdev = pd->rdev; + dev_attr = rdev->dev_attr; + + if (rdev->mod_exit) { + dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); + rc = -EIO; + goto exit; + } + + if (srq_init_attr->srq_type != IB_SRQT_BASIC) { + dev_err(rdev_to_dev(rdev), "SRQ type not supported\n"); + rc = -ENOTSUPP; + goto exit; + } + + if (udata) { + context = pd->ibpd.uobject->context; + cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); + } + + if (atomic_read(&rdev->stats.rsors.srq_count) >= dev_attr->max_srq) { + dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQs)\n"); + rc = -EINVAL; + goto exit; + } + + if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { + dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQ_WQs)\n"); + rc = -EINVAL; + goto exit; + } + + srq->rdev = rdev; + srq->qplib_srq.pd = &pd->qplib_pd; + srq->qplib_srq.dpi = &rdev->dpi_privileged; + + /* Allocate 1 more than what's provided so posting max doesn't + mean empty */ + entries = srq_init_attr->attr.max_wr + 1; + entries = bnxt_re_init_depth(entries, cntx); + if (entries > dev_attr->max_srq_wqes + 1) + entries = dev_attr->max_srq_wqes + 1; + + srq->qplib_srq.wqe_size = _max_rwqe_sz(6); /* 128 byte wqe size */ + srq->qplib_srq.max_wqe = entries; + srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; + srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; + srq->srq_limit = srq_init_attr->attr.srq_limit; + srq->qplib_srq.eventq_hw_ring_id = rdev->nqr.nq[0].ring_id; + srq->qplib_srq.sginfo.pgsize = PAGE_SIZE; + srq->qplib_srq.sginfo.pgshft = PAGE_SHIFT; + + if (udata) { + rc = bnxt_re_init_user_srq(rdev, pd, srq, udata); + if (rc) + goto fail; + } + + rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!\n"); + goto fail; + } + + if (udata) { + struct bnxt_re_srq_resp resp; + + resp.srqid = srq->qplib_srq.id; + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) { + bnxt_qplib_destroy_srq(&rdev->qplib_res, &srq->qplib_srq); + goto fail; + } + } + atomic_inc(&rdev->stats.rsors.srq_count); + max_srq_count = atomic_read(&rdev->stats.rsors.srq_count); + if (max_srq_count > atomic_read(&rdev->stats.rsors.max_srq_count)) + atomic_set(&rdev->stats.rsors.max_srq_count, max_srq_count); + spin_lock_init(&srq->lock); + + return 0; +fail: + if (udata && srq->umem && !IS_ERR(srq->umem)) { + ib_umem_release(srq->umem); + srq->umem = NULL; + } +exit: + return rc; +} + +int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, + enum ib_srq_attr_mask srq_attr_mask, + struct ib_udata *udata) +{ + struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, + ibsrq); + struct bnxt_re_dev *rdev = srq->rdev; + int rc; + + switch (srq_attr_mask) { + case IB_SRQ_MAX_WR: + /* SRQ resize is not supported */ + break; + case IB_SRQ_LIMIT: + /* Change the SRQ threshold */ + if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) + return -EINVAL; + + srq->qplib_srq.threshold = srq_attr->srq_limit; + rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!\n"); + return rc; + } + /* On success, update the shadow */ + srq->srq_limit = srq_attr->srq_limit; + + if (udata) { + /* Build and send response back to udata */ + rc = bnxt_re_copy_to_udata(rdev, srq, 0, udata); + if (rc) + return rc; + } + break; + default: + dev_err(rdev_to_dev(rdev), + "Unsupported srq_attr_mask 0x%x\n", srq_attr_mask); + return -EINVAL; + } + return 0; +} + +int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) +{ + struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, + ibsrq); + struct bnxt_re_dev *rdev = srq->rdev; + int rc; + + rc = bnxt_qplib_query_srq(&rdev->qplib_res, &srq->qplib_srq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Query HW SRQ (0x%x) failed! rc = %d\n", + srq->qplib_srq.id, rc); + return rc; + } + srq_attr->max_wr = srq->qplib_srq.max_wqe; + srq_attr->max_sge = srq->qplib_srq.max_sge; + srq_attr->srq_limit = srq->qplib_srq.threshold; + + return 0; +} + +int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, + ibsrq); + struct bnxt_qplib_swqe wqe = {}; + unsigned long flags; + int rc = 0; + + spin_lock_irqsave(&srq->lock, flags); + while (wr) { + /* Transcribe each ib_recv_wr to qplib_swqe */ + wqe.num_sge = wr->num_sge; + wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; + wqe.wr_id = wr->wr_id; + wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; + rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); + if (rc) { + *bad_wr = wr; + break; + } + wr = wr->next; + } + spin_unlock_irqrestore(&srq->lock, flags); + + return rc; +} + +unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) +{ + unsigned long flags; + + spin_lock_irqsave(&qp->scq->cq_lock, flags); + if (qp->rcq && qp->rcq != qp->scq) + spin_lock(&qp->rcq->cq_lock); + + return flags; +} + +void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, + unsigned long flags) +{ + if (qp->rcq && qp->rcq != qp->scq) + spin_unlock(&qp->rcq->cq_lock); + spin_unlock_irqrestore(&qp->scq->cq_lock, flags); +} + +/* Queue Pairs */ +static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp) +{ + struct bnxt_re_qp *gsi_sqp; + struct bnxt_re_ah *gsi_sah; + struct bnxt_re_dev *rdev; + unsigned long flags; + int rc = 0; + + rdev = qp->rdev; + gsi_sqp = rdev->gsi_ctx.gsi_sqp; + gsi_sah = rdev->gsi_ctx.gsi_sah; + + /* remove from active qp list */ + mutex_lock(&rdev->qp_lock); + list_del(&gsi_sqp->list); + mutex_unlock(&rdev->qp_lock); + + if (gsi_sah) { + dev_dbg(rdev_to_dev(rdev), "Destroy the shadow AH\n"); + rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &gsi_sah->qplib_ah, + true); + if (rc) + dev_err(rdev_to_dev(rdev), + "Destroy HW AH for shadow QP failed!\n"); + atomic_dec(&rdev->stats.rsors.ah_count); + } + + dev_dbg(rdev_to_dev(rdev), "Destroy the shadow QP\n"); + rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp); + if (rc) + dev_err(rdev_to_dev(rdev), "Destroy Shadow QP failed\n"); + + /* Clean the CQ for shadow QP completions */ + flags = bnxt_re_lock_cqs(gsi_sqp); + bnxt_qplib_clean_qp(&gsi_sqp->qplib_qp); + bnxt_re_unlock_cqs(gsi_sqp, flags); + + bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp); + bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &gsi_sqp->qplib_qp); + kfree(rdev->gsi_ctx.sqp_tbl); + kfree(gsi_sah); + kfree(gsi_sqp); + rdev->gsi_ctx.gsi_sqp = NULL; + rdev->gsi_ctx.gsi_sah = NULL; + rdev->gsi_ctx.sqp_tbl = NULL; + atomic_dec(&rdev->stats.rsors.qp_count); + + return 0; +} + +static void bnxt_re_dump_debug_stats(struct bnxt_re_dev *rdev, u32 active_qps) +{ + u32 total_qp = 0; + u64 avg_time = 0; + int i; + + if (!rdev->rcfw.sp_perf_stats_enabled) + return; + + switch (active_qps) { + case 1: + /* Potential hint for Test Stop */ + for (i = 0; i < RCFW_MAX_STAT_INDEX; i++) { + if (rdev->rcfw.qp_destroy_stats[i]) { + total_qp++; + avg_time += rdev->rcfw.qp_destroy_stats[i]; + } + } + if (total_qp >= 0 || avg_time >= 0) + dev_dbg(rdev_to_dev(rdev), + "Perf Debug: %ps Total (%d) QP destroyed in (%ld) msec\n", + __builtin_return_address(0), total_qp, + (long)jiffies_to_msecs(avg_time)); + break; + case 2: + /* Potential hint for Test Start */ + dev_dbg(rdev_to_dev(rdev), + "Perf Debug: %ps active_qps = %d\n", + __builtin_return_address(0), active_qps); + break; + default: + /* Potential hint to know latency of QP destroy. + * Average time taken for 1K QP Destroy. + */ + if (active_qps > 1024 && !(active_qps % 1024)) + dev_dbg(rdev_to_dev(rdev), + "Perf Debug: %ps Active QP (%d) Watermark (%d)\n", + __builtin_return_address(0), active_qps, + atomic_read(&rdev->stats.rsors.max_qp_count)); + break; + } +} + +int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) +{ + struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_re_dev *rdev = qp->rdev; + unsigned long flags; + u32 active_qps; + int rc; + + mutex_lock(&rdev->qp_lock); + list_del(&qp->list); + active_qps = atomic_dec_return(&rdev->stats.rsors.qp_count); + if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC) + atomic_dec(&rdev->stats.rsors.rc_qp_count); + else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD) + atomic_dec(&rdev->stats.rsors.ud_qp_count); + mutex_unlock(&rdev->qp_lock); + + rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) + dev_err_ratelimited(rdev_to_dev(rdev), + "%s id = %d failed rc = %d\n", + __func__, qp->qplib_qp.id, rc); + + if (!ib_qp->uobject) { + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_clean_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + } + + bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); + if (ib_qp->qp_type == IB_QPT_GSI && + rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { + if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && + rdev->gsi_ctx.gsi_sqp) { + bnxt_re_destroy_gsi_sqp(qp); + } + bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &qp->qplib_qp); + } + + if (qp->rumem && !IS_ERR(qp->rumem)) + ib_umem_release(qp->rumem); + if (qp->sumem && !IS_ERR(qp->sumem)) + ib_umem_release(qp->sumem); + kfree(qp); + + bnxt_re_dump_debug_stats(rdev, active_qps); + + return 0; +} + +static u8 __from_ib_qp_type(enum ib_qp_type type) +{ + switch (type) { + case IB_QPT_GSI: + return CMDQ_CREATE_QP1_TYPE_GSI; + case IB_QPT_RC: + return CMDQ_CREATE_QP_TYPE_RC; + case IB_QPT_UD: + return CMDQ_CREATE_QP_TYPE_UD; + case IB_QPT_RAW_ETHERTYPE: + return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE; + default: + return IB_QPT_MAX; + } +} + +static u16 _get_swqe_sz(int nsge) +{ + return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge); +} + +static int bnxt_re_get_swqe_size(int ilsize, int nsge) +{ + u16 wqe_size, calc_ils; + + wqe_size = _get_swqe_sz(nsge); + if (ilsize) { + calc_ils = (sizeof(struct sq_send_hdr) + ilsize); + wqe_size = max_t(int, calc_ils, wqe_size); + wqe_size = ALIGN(wqe_size, 32); + } + return wqe_size; +} + +static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, + struct ib_qp_init_attr *init_attr) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_q *sq; + int align, ilsize; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + sq = &qplqp->sq; + dev_attr = rdev->dev_attr; + + align = sizeof(struct sq_send_hdr); + ilsize = ALIGN(init_attr->cap.max_inline_data, align); + + sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge); + if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges)) + return -EINVAL; + /* For Cu/Wh and gen p5 backward compatibility mode + * wqe size is fixed to 128 bytes + */ + if (sq->wqe_size < _get_swqe_sz(dev_attr->max_qp_sges) && + qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + sq->wqe_size = _get_swqe_sz(dev_attr->max_qp_sges); + + if (init_attr->cap.max_inline_data) { + qplqp->max_inline_data = sq->wqe_size - + sizeof(struct sq_send_hdr); + init_attr->cap.max_inline_data = qplqp->max_inline_data; + if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + sq->max_sge = qplqp->max_inline_data / + sizeof(struct sq_sge); + } + + return 0; +} + +static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_pd *pd, struct bnxt_re_qp *qp, + struct ib_udata *udata) +{ + struct bnxt_qplib_sg_info *sginfo; + struct bnxt_qplib_qp *qplib_qp; + struct bnxt_re_ucontext *cntx; + struct ib_ucontext *context; + struct bnxt_re_qp_req ureq; + struct ib_umem *umem; + int rc, bytes = 0; + int psn_nume; + int psn_sz; + + qplib_qp = &qp->qplib_qp; + context = pd->ibpd.uobject->context; + cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); + sginfo = &qplib_qp->sq.sginfo; + + if (udata->inlen < sizeof(ureq)) + dev_warn(rdev_to_dev(rdev), + "Update the library ulen %d klen %d\n", + (unsigned int)udata->inlen, + (unsigned int)sizeof(ureq)); + + rc = ib_copy_from_udata(&ureq, udata, + min(udata->inlen, sizeof(ureq))); + if (rc) + return rc; + + bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); + /* Consider mapping PSN search memory only for RC QPs. */ + if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { + psn_sz = _is_chip_gen_p5_p7(rdev->chip_ctx) ? + sizeof(struct sq_psn_search_ext) : + sizeof(struct sq_psn_search); + if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + psn_sz = sizeof(struct sq_msn_search); + psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + qplib_qp->sq.max_wqe : + ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / + sizeof(struct bnxt_qplib_sge)); + if (BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + psn_nume = roundup_pow_of_two(psn_nume); + + bytes += (psn_nume * psn_sz); + } + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get_compat(rdev, context, udata, ureq.qpsva, bytes, + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(umem)) { + dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n", + __func__, PTR_ERR(umem)); + return PTR_ERR(umem); + } + + qp->sumem = umem; + /* pgsize and pgshft were initialize already. */ + sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); + sginfo->npages = ib_umem_num_pages_compat(umem); + qplib_qp->qp_handle = ureq.qp_handle; + + if (!qp->qplib_qp.srq) { + sginfo = &qplib_qp->rq.sginfo; + bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get_compat(rdev, + context, udata, ureq.qprva, bytes, + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(umem)) { + dev_err(rdev_to_dev(rdev), + "%s: ib_umem_get failed ret =%ld\n", + __func__, PTR_ERR(umem)); + goto rqfail; + } + qp->rumem = umem; + /* pgsize and pgshft were initialize already. */ + sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); + sginfo->npages = ib_umem_num_pages_compat(umem); + } + + qplib_qp->dpi = &cntx->dpi; + qplib_qp->is_user = true; + + return 0; +rqfail: + ib_umem_release(qp->sumem); + qp->sumem = NULL; + qplib_qp->sq.sginfo.sghead = NULL; + qplib_qp->sq.sginfo.nmap = 0; + + return PTR_ERR(umem); +} + +static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah(struct bnxt_re_pd *pd, + struct bnxt_qplib_res *qp1_res, + struct bnxt_qplib_qp *qp1_qp) +{ + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_ah *ah; + union ib_gid sgid; + int rc; + + ah = kzalloc(sizeof(*ah), GFP_KERNEL); + if (!ah) { + dev_err(rdev_to_dev(rdev), "Allocate Address Handle failed!\n"); + return NULL; + } + memset(ah, 0, sizeof(*ah)); + ah->rdev = rdev; + ah->qplib_ah.pd = &pd->qplib_pd; + + rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); + if (rc) + goto fail; + + /* supply the dgid data same as sgid */ + memcpy(ah->qplib_ah.dgid.data, &sgid.raw, + sizeof(union ib_gid)); + ah->qplib_ah.sgid_index = 0; + + ah->qplib_ah.traffic_class = 0; + ah->qplib_ah.flow_label = 0; + ah->qplib_ah.hop_limit = 1; + ah->qplib_ah.sl = 0; + /* Have DMAC same as SMAC */ + ether_addr_copy(ah->qplib_ah.dmac, rdev->dev_addr); + dev_dbg(rdev_to_dev(rdev), "ah->qplib_ah.dmac = %x:%x:%x:%x:%x:%x\n", + ah->qplib_ah.dmac[0], ah->qplib_ah.dmac[1], ah->qplib_ah.dmac[2], + ah->qplib_ah.dmac[3], ah->qplib_ah.dmac[4], ah->qplib_ah.dmac[5]); + + rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, true); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Allocate HW AH for Shadow QP failed!\n"); + goto fail; + } + dev_dbg(rdev_to_dev(rdev), "AH ID = %d\n", ah->qplib_ah.id); + atomic_inc(&rdev->stats.rsors.ah_count); + + return ah; +fail: + kfree(ah); + return NULL; +} + +void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_qp *gsi_qp; + struct bnxt_re_ah *sah; + struct bnxt_re_pd *pd; + struct ib_pd *ib_pd; + int rc; + + if (!rdev) + return; + + sah = rdev->gsi_ctx.gsi_sah; + + dev_dbg(rdev_to_dev(rdev), "Updating the AH\n"); + if (sah) { + /* Check if the AH created with current mac address */ + if (!compare_ether_header(sah->qplib_ah.dmac, rdev->dev_addr)) { + dev_dbg(rdev_to_dev(rdev), + "Not modifying shadow AH during AH update\n"); + return; + } + + gsi_qp = rdev->gsi_ctx.gsi_qp; + ib_pd = gsi_qp->ib_qp.pd; + pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, + &sah->qplib_ah, false); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to destroy shadow AH during AH update\n"); + return; + } + atomic_dec(&rdev->stats.rsors.ah_count); + kfree(sah); + rdev->gsi_ctx.gsi_sah = NULL; + + sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, + &gsi_qp->qplib_qp); + if (!sah) { + dev_err(rdev_to_dev(rdev), + "Failed to update AH for ShadowQP\n"); + return; + } + rdev->gsi_ctx.gsi_sah = sah; + atomic_inc(&rdev->stats.rsors.ah_count); + } +} + +static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd, + struct bnxt_qplib_res *qp1_res, + struct bnxt_qplib_qp *qp1_qp) +{ + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_qp *qp; + int rc; + + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) { + dev_err(rdev_to_dev(rdev), "Allocate internal UD QP failed!\n"); + return NULL; + } + memset(qp, 0, sizeof(*qp)); + qp->rdev = rdev; + + /* Initialize the shadow QP structure from the QP1 values */ + ether_addr_copy(qp->qplib_qp.smac, rdev->dev_addr); + qp->qplib_qp.pd = &pd->qplib_pd; + qp->qplib_qp.qp_handle = (u64)&qp->qplib_qp; + qp->qplib_qp.type = IB_QPT_UD; + + qp->qplib_qp.max_inline_data = 0; + qp->qplib_qp.sig_type = true; + + /* Shadow QP SQ depth should be same as QP1 RQ depth */ + qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(0, 6); + qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; + qp->qplib_qp.sq.max_sge = 2; + /* Q full delta can be 1 since it is internal QP */ + qp->qplib_qp.sq.q_full_delta = 1; + qp->qplib_qp.sq.sginfo.pgsize = PAGE_SIZE; + qp->qplib_qp.sq.sginfo.pgshft = PAGE_SHIFT; + + qp->qplib_qp.scq = qp1_qp->scq; + qp->qplib_qp.rcq = qp1_qp->rcq; + + qp->qplib_qp.rq.wqe_size = _max_rwqe_sz(6); /* 128 Byte wqe size */ + qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; + qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; + qp->qplib_qp.rq.sginfo.pgsize = PAGE_SIZE; + qp->qplib_qp.rq.sginfo.pgshft = PAGE_SHIFT; + /* Q full delta can be 1 since it is internal QP */ + qp->qplib_qp.rq.q_full_delta = 1; + qp->qplib_qp.mtu = qp1_qp->mtu; + qp->qplib_qp.dpi = &rdev->dpi_privileged; + + rc = bnxt_qplib_alloc_hdr_buf(qp1_res, &qp->qplib_qp, 0, + BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6); + if (rc) + goto fail; + + rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "create HW QP failed!\n"); + goto qp_fail; + } + + dev_dbg(rdev_to_dev(rdev), "Created shadow QP with ID = %d\n", + qp->qplib_qp.id); + spin_lock_init(&qp->sq_lock); + INIT_LIST_HEAD(&qp->list); + mutex_lock(&rdev->qp_lock); + list_add_tail(&qp->list, &rdev->qp_list); + atomic_inc(&rdev->stats.rsors.qp_count); + mutex_unlock(&rdev->qp_lock); + return qp; +qp_fail: + bnxt_qplib_free_hdr_buf(qp1_res, &qp->qplib_qp); +fail: + kfree(qp); + return NULL; +} + +static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, + struct ib_qp_init_attr *init_attr, void *cntx) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_q *rq; + int entries; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + rq = &qplqp->rq; + dev_attr = rdev->dev_attr; + + if (init_attr->srq) { + struct bnxt_re_srq *srq; + + srq = to_bnxt_re(init_attr->srq, struct bnxt_re_srq, ibsrq); + if (!srq) { + dev_err(rdev_to_dev(rdev), "SRQ not found\n"); + return -EINVAL; + } + qplqp->srq = &srq->qplib_srq; + rq->max_wqe = 0; + } else { + rq->max_sge = init_attr->cap.max_recv_sge; + if (rq->max_sge > dev_attr->max_qp_sges) + rq->max_sge = dev_attr->max_qp_sges; + init_attr->cap.max_recv_sge = rq->max_sge; + rq->wqe_size = bnxt_re_get_rwqe_size(qplqp, rq->max_sge, + dev_attr->max_qp_sges); + + /* Allocate 1 more than what's provided so posting max doesn't + mean empty */ + entries = init_attr->cap.max_recv_wr + 1; + entries = bnxt_re_init_depth(entries, cntx); + rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); + rq->q_full_delta = 0; + rq->sginfo.pgsize = PAGE_SIZE; + rq->sginfo.pgshft = PAGE_SHIFT; + } + + return 0; +} + +static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + dev_attr = rdev->dev_attr; + + if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) + qplqp->rq.max_sge = dev_attr->max_qp_sges; +} + +static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, + struct ib_qp_init_attr *init_attr, + void *cntx) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_q *sq; + int diff = 0; + int entries; + int rc; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + sq = &qplqp->sq; + dev_attr = rdev->dev_attr; + + sq->max_sge = init_attr->cap.max_send_sge; + if (sq->max_sge > dev_attr->max_qp_sges) { + sq->max_sge = dev_attr->max_qp_sges; + init_attr->cap.max_send_sge = sq->max_sge; + } + rc = bnxt_re_setup_swqe_size(qp, init_attr); + if (rc) + return rc; + /* + * Change the SQ depth if user has requested minimum using + * configfs. Only supported for kernel consumers. Setting + * min_tx_depth to 4096 to handle iser SQ full condition + * in most of the newer OS distros + */ + entries = init_attr->cap.max_send_wr; + if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) { + /* + * If users specify any value greater than 1 use min_tx_depth + * provided by user for comparison. Else, compare it with the + * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly. + */ + if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth) + entries = rdev->min_tx_depth; + else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH) + entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH; + } + diff = bnxt_re_get_diff(cntx, rdev->chip_ctx); + entries = bnxt_re_init_depth(entries + diff + 1, cntx); + sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); + sq->q_full_delta = diff + 1; + /* + * Reserving one slot for Phantom WQE. Application can + * post one extra entry in this case. But allowing this to avoid + * unexpected Queue full condition + */ + sq->q_full_delta -= 1; /* becomes 0 for gen-p5 */ + sq->sginfo.pgsize = PAGE_SIZE; + sq->sginfo.pgshft = PAGE_SHIFT; + return 0; +} + +static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp, + struct ib_qp_init_attr *init_attr, + void *cntx) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + int entries; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + dev_attr = rdev->dev_attr; + + if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { + entries = init_attr->cap.max_send_wr + 1; + entries = bnxt_re_init_depth(entries, cntx); + qplqp->sq.max_wqe = min_t(u32, entries, + dev_attr->max_qp_wqes + 1); + qplqp->sq.q_full_delta = qplqp->sq.max_wqe - + init_attr->cap.max_send_wr; + qplqp->sq.max_sge++; /* Need one extra sge to put UD header */ + if (qplqp->sq.max_sge > dev_attr->max_qp_sges) + qplqp->sq.max_sge = dev_attr->max_qp_sges; + } +} + +static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, + struct ib_qp_init_attr *init_attr) +{ + struct bnxt_qplib_chip_ctx *chip_ctx; + struct bnxt_re_gsi_context *gsi_ctx; + int qptype; + + chip_ctx = rdev->chip_ctx; + gsi_ctx = &rdev->gsi_ctx; + + qptype = __from_ib_qp_type(init_attr->qp_type); + if (qptype == IB_QPT_MAX) { + dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported\n", + qptype); + qptype = -EINVAL; + goto out; + } + + if (_is_chip_gen_p5_p7(chip_ctx) && init_attr->qp_type == IB_QPT_GSI) { + /* For Thor always force UD mode. */ + qptype = CMDQ_CREATE_QP_TYPE_GSI; + gsi_ctx->gsi_qp_mode = BNXT_RE_GSI_MODE_UD; + } +out: + return qptype; +} + +static int bnxt_re_init_qp_wqe_mode(struct bnxt_re_dev *rdev) +{ + return rdev->chip_ctx->modes.wqe_mode; +} + +static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_re_ucontext *cntx = NULL; + struct ib_ucontext *context; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + struct bnxt_re_cq *cq; + int rc = 0, qptype; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + dev_attr = rdev->dev_attr; + + if (udata) { + context = pd->ibpd.uobject->context; + cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); + } + + /* Setup misc params */ + qplqp->is_user = false; + qplqp->pd = &pd->qplib_pd; + qplqp->qp_handle = (u64)qplqp; + qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? + true : false); + qptype = bnxt_re_init_qp_type(rdev, init_attr); + if (qptype < 0) { + rc = qptype; + goto out; + } + qplqp->type = (u8)qptype; + qplqp->wqe_mode = bnxt_re_init_qp_wqe_mode(rdev); + ether_addr_copy(qplqp->smac, rdev->dev_addr); + + if (init_attr->qp_type == IB_QPT_RC) { + qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom; + qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; + } + qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->if_mtu)); + qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */ + if (init_attr->create_flags) { + dev_dbg(rdev_to_dev(rdev), + "QP create flags 0x%x not supported\n", + init_attr->create_flags); + return -EOPNOTSUPP; + } + + /* Setup CQs */ + if (init_attr->send_cq) { + cq = to_bnxt_re(init_attr->send_cq, struct bnxt_re_cq, ibcq); + if (!cq) { + dev_err(rdev_to_dev(rdev), "Send CQ not found\n"); + rc = -EINVAL; + goto out; + } + qplqp->scq = &cq->qplib_cq; + qp->scq = cq; + } + + if (init_attr->recv_cq) { + cq = to_bnxt_re(init_attr->recv_cq, struct bnxt_re_cq, ibcq); + if (!cq) { + dev_err(rdev_to_dev(rdev), "Receive CQ not found\n"); + rc = -EINVAL; + goto out; + } + qplqp->rcq = &cq->qplib_cq; + qp->rcq = cq; + } + + /* Setup RQ/SRQ */ + rc = bnxt_re_init_rq_attr(qp, init_attr, cntx); + if (rc) + goto out; + if (init_attr->qp_type == IB_QPT_GSI) + bnxt_re_adjust_gsi_rq_attr(qp); + + /* Setup SQ */ + rc = bnxt_re_init_sq_attr(qp, init_attr, cntx); + if (rc) + goto out; + if (init_attr->qp_type == IB_QPT_GSI) + bnxt_re_adjust_gsi_sq_attr(qp, init_attr, cntx); + + if (udata) /* This will update DPI and qp_handle */ + rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); +out: + return rc; +} + +static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp, + struct bnxt_re_pd *pd) +{ + struct bnxt_re_sqp_entries *sqp_tbl = NULL; + struct bnxt_re_dev *rdev; + struct bnxt_re_qp *sqp; + struct bnxt_re_ah *sah; + int rc = 0; + + rdev = qp->rdev; + /* Create a shadow QP to handle the QP1 traffic */ + sqp_tbl = kzalloc(sizeof(*sqp_tbl) * BNXT_RE_MAX_GSI_SQP_ENTRIES, + GFP_KERNEL); + if (!sqp_tbl) + return -ENOMEM; + rdev->gsi_ctx.sqp_tbl = sqp_tbl; + + sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp); + if (!sqp) { + rc = -ENODEV; + dev_err(rdev_to_dev(rdev), + "Failed to create Shadow QP for QP1\n"); + goto out; + } + rdev->gsi_ctx.gsi_sqp = sqp; + + sqp->rcq = qp->rcq; + sqp->scq = qp->scq; + sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, + &qp->qplib_qp); + if (!sah) { + bnxt_qplib_destroy_qp(&rdev->qplib_res, + &sqp->qplib_qp); + rc = -ENODEV; + dev_err(rdev_to_dev(rdev), + "Failed to create AH entry for ShadowQP\n"); + goto out; + } + rdev->gsi_ctx.gsi_sah = sah; + + return 0; +out: + kfree(sqp_tbl); + return rc; +} + +static int __get_rq_hdr_buf_size(u8 gsi_mode) +{ + return (gsi_mode == BNXT_RE_GSI_MODE_ALL) ? + BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 : + BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE; +} + +static int __get_sq_hdr_buf_size(u8 gsi_mode) +{ + return (gsi_mode != BNXT_RE_GSI_MODE_ROCE_V1) ? + BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 : + BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE; +} + +static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd) +{ + struct bnxt_qplib_qp *qplqp; + struct bnxt_qplib_res *res; + struct bnxt_re_dev *rdev; + u32 sstep, rstep; + u8 gsi_mode; + int rc = 0; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + res = &rdev->qplib_res; + gsi_mode = rdev->gsi_ctx.gsi_qp_mode; + + rstep = __get_rq_hdr_buf_size(gsi_mode); + sstep = __get_sq_hdr_buf_size(gsi_mode); + rc = bnxt_qplib_alloc_hdr_buf(res, qplqp, sstep, rstep); + if (rc) + goto out; + + rc = bnxt_qplib_create_qp1(res, qplqp); + if (rc) { + dev_err(rdev_to_dev(rdev), "create HW QP1 failed!\n"); + goto out; + } + + if (gsi_mode == BNXT_RE_GSI_MODE_ALL) + rc = bnxt_re_create_shadow_gsi(qp, pd); +out: + return rc; +} + +static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev, + struct ib_qp_init_attr *init_attr, + struct bnxt_qplib_dev_attr *dev_attr) +{ + bool rc = true; + int ilsize; + + ilsize = ALIGN(init_attr->cap.max_inline_data, sizeof(struct sq_sge)); + if ((init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || + (init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || + (init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || + (init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || + (ilsize > dev_attr->max_inline_data)) { + dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded! " + "0x%x/0x%x 0x%x/0x%x 0x%x/0x%x " + "0x%x/0x%x 0x%x/0x%x\n", + init_attr->cap.max_send_wr, dev_attr->max_qp_wqes, + init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes, + init_attr->cap.max_send_sge, dev_attr->max_qp_sges, + init_attr->cap.max_recv_sge, dev_attr->max_qp_sges, + init_attr->cap.max_inline_data, + dev_attr->max_inline_data); + rc = false; + } + return rc; +} + +static inline struct +bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in, + struct bnxt_re_dev *rdev) +{ + struct bnxt_re_qp *qp; + + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) + dev_err(rdev_to_dev(rdev), "Allocate QP failed!\n"); + return qp; +} + +struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in, + struct ib_qp_init_attr *qp_init_attr, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd; + struct ib_pd *ib_pd = qp_in; + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_re_dev *rdev; + u32 active_qps, tmp_qps; + struct bnxt_re_qp *qp; + int rc; + + pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + rdev = pd->rdev; + dev_attr = rdev->dev_attr; + if (rdev->mod_exit) { + rc = -EIO; + dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); + goto exit; + } + + if (atomic_read(&rdev->stats.rsors.qp_count) >= dev_attr->max_qp) { + dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded(QPs Alloc'd %u of max %u)\n", + atomic_read(&rdev->stats.rsors.qp_count), dev_attr->max_qp); + rc = -EINVAL; + goto exit; + } + + rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); + if (!rc) { + rc = -EINVAL; + goto exit; + } + qp = __get_qp_from_qp_in(qp_in, rdev); + if (!qp) { + rc = -ENOMEM; + goto exit; + } + qp->rdev = rdev; + + rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata); + if (rc) + goto fail; + + if (qp_init_attr->qp_type == IB_QPT_GSI && + !_is_chip_gen_p5_p7(rdev->chip_ctx)) { + rc = bnxt_re_create_gsi_qp(qp, pd); + if (rc == -ENODEV) + goto qp_destroy; + if (rc) + goto fail; + } else { + rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "create HW QP failed!\n"); + goto free_umem; + } + + if (udata) { + struct bnxt_re_qp_resp resp; + + resp.qpid = qp->qplib_qp.id; + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) + goto qp_destroy; + } + } + + qp->ib_qp.qp_num = qp->qplib_qp.id; + if (qp_init_attr->qp_type == IB_QPT_GSI) + rdev->gsi_ctx.gsi_qp = qp; + spin_lock_init(&qp->sq_lock); + spin_lock_init(&qp->rq_lock); + INIT_LIST_HEAD(&qp->list); + mutex_lock(&rdev->qp_lock); + list_add_tail(&qp->list, &rdev->qp_list); + mutex_unlock(&rdev->qp_lock); + atomic_inc(&rdev->stats.rsors.qp_count); + active_qps = atomic_read(&rdev->stats.rsors.qp_count); + if (active_qps > atomic_read(&rdev->stats.rsors.max_qp_count)) + atomic_set(&rdev->stats.rsors.max_qp_count, active_qps); + + bnxt_re_dump_debug_stats(rdev, active_qps); + + /* Get the counters for RC QPs and UD QPs */ + if (qp_init_attr->qp_type == IB_QPT_RC) { + tmp_qps = atomic_inc_return(&rdev->stats.rsors.rc_qp_count); + if (tmp_qps > atomic_read(&rdev->stats.rsors.max_rc_qp_count)) + atomic_set(&rdev->stats.rsors.max_rc_qp_count, tmp_qps); + } else if (qp_init_attr->qp_type == IB_QPT_UD) { + tmp_qps = atomic_inc_return(&rdev->stats.rsors.ud_qp_count); + if (tmp_qps > atomic_read(&rdev->stats.rsors.max_ud_qp_count)) + atomic_set(&rdev->stats.rsors.max_ud_qp_count, tmp_qps); + } + + return &qp->ib_qp; + +qp_destroy: + bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); +free_umem: + if (udata) { + if (qp->rumem && !IS_ERR(qp->rumem)) + ib_umem_release(qp->rumem); + if (qp->sumem && !IS_ERR(qp->sumem)) + ib_umem_release(qp->sumem); + } +fail: + kfree(qp); +exit: + return ERR_PTR(rc); +} + +static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp1_qp, + int qp_attr_mask) +{ + struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp; + int rc = 0; + + if (qp_attr_mask & IB_QP_STATE) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; + qp->qplib_qp.state = qp1_qp->qplib_qp.state; + } + if (qp_attr_mask & IB_QP_PKEY_INDEX) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; + qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; + } + + if (qp_attr_mask & IB_QP_QKEY) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; + /* Using a Random QKEY */ + qp->qplib_qp.qkey = BNXT_RE_QP_RANDOM_QKEY; + } + if (qp_attr_mask & IB_QP_SQ_PSN) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; + qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; + } + + rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) + dev_err(rdev_to_dev(rdev), "Modify Shadow QP for QP1 failed\n"); + return rc; +} + +static u32 ipv4_from_gid(u8 *gid) +{ + return (gid[15] << 24 | gid[14] << 16 | gid[13] << 8 | gid[12]); +} + +static u16 get_source_port(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp) +{ + u8 ip_off, data[48], smac[ETH_ALEN]; + u16 crc = 0, buf_len = 0, i; + u8 addr_len; + u32 qpn; + + if (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6) { + addr_len = 6; + ip_off = 10; + } else { + addr_len = 4; + ip_off = 12; + } + + memcpy(smac, qp->qplib_qp.smac, ETH_ALEN); + + memset(data, 0, 48); + memcpy(data, qp->qplib_qp.ah.dmac, ETH_ALEN); + buf_len += ETH_ALEN; + + memcpy(data + buf_len, smac, ETH_ALEN); + buf_len += ETH_ALEN; + + memcpy(data + buf_len, qp->qplib_qp.ah.dgid.data + ip_off, addr_len); + buf_len += addr_len; + + memcpy(data + buf_len, qp->qp_info_entry.sgid.raw + ip_off, addr_len); + buf_len += addr_len; + + qpn = htonl(qp->qplib_qp.dest_qpn); + memcpy(data + buf_len, (u8 *)&qpn + 1, 3); + buf_len += 3; + + for (i = 0; i < buf_len; i++) + crc = crc16(crc, (data + i), 1); + + return crc; +} + +static void bnxt_re_update_qp_info(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp) +{ + u16 type; + + type = __from_hw_to_ib_qp_type(qp->qplib_qp.type); + + /* User-space can extract ip address with sgid_index. */ + if (ipv6_addr_v4mapped((struct in6_addr *)&qp->qplib_qp.ah.dgid)) { + qp->qp_info_entry.s_ip.ipv4_addr = ipv4_from_gid(qp->qp_info_entry.sgid.raw); + qp->qp_info_entry.d_ip.ipv4_addr = ipv4_from_gid(qp->qplib_qp.ah.dgid.data); + } else { + memcpy(&qp->qp_info_entry.s_ip.ipv6_addr, qp->qp_info_entry.sgid.raw, + sizeof(qp->qp_info_entry.s_ip.ipv6_addr)); + memcpy(&qp->qp_info_entry.d_ip.ipv6_addr, qp->qplib_qp.ah.dgid.data, + sizeof(qp->qp_info_entry.d_ip.ipv6_addr)); + } + + if (type == IB_QPT_RC && + (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 || + qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6)) { + qp->qp_info_entry.s_port = get_source_port(rdev, qp); + } + qp->qp_info_entry.d_port = BNXT_RE_QP_DEST_PORT; +} + +static void bnxt_qplib_manage_flush_qp(struct bnxt_re_qp *qp) +{ + struct bnxt_qplib_q *rq, *sq; + unsigned long flags; + + if (qp->sumem) + return; + + if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { + rq = &qp->qplib_qp.rq; + sq = &qp->qplib_qp.sq; + + dev_dbg(rdev_to_dev(qp->rdev), + "Move QP = %p to flush list\n", qp); + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_add_flush_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + + if (sq->hwq.prod != sq->hwq.cons) + bnxt_re_handle_cqn(&qp->scq->qplib_cq); + + if (qp->rcq && (qp->rcq != qp->scq) && + (rq->hwq.prod != rq->hwq.cons)) + bnxt_re_handle_cqn(&qp->rcq->qplib_cq); + } + + if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { + dev_dbg(rdev_to_dev(qp->rdev), + "Move QP = %p out of flush list\n", qp); + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_clean_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + } +} + +bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, + enum ib_qp_state next_state, + enum ib_qp_type type, + enum ib_qp_attr_mask mask) +{ + return (ib_modify_qp_is_ok(cur_state, next_state, + type, mask)); +} + +int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_udata *udata) +{ + enum ib_qp_state curr_qp_state, new_qp_state; + struct bnxt_re_modify_qp_ex_resp resp = {}; + struct bnxt_re_modify_qp_ex_req ureq = {}; + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_ppp *ppp = NULL; + struct bnxt_re_dev *rdev; + struct bnxt_re_qp *qp; + struct ib_gid_attr *sgid_attr; + struct ib_gid_attr gid_attr; + union ib_gid sgid, *gid_ptr = NULL; + u8 nw_type; + int rc, entries, status; + bool is_copy_to_udata = false; + bool is_qpmtu_high = false; + + qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); + rdev = qp->rdev; + dev_attr = rdev->dev_attr; + + qp->qplib_qp.modify_flags = 0; + ppp = &qp->qplib_qp.ppp; + if (qp_attr_mask & IB_QP_STATE) { + curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); + new_qp_state = qp_attr->qp_state; + if (!ib_modify_qp_is_ok_compat(curr_qp_state, new_qp_state, + ib_qp->qp_type, qp_attr_mask)) { + dev_err(rdev_to_dev(rdev),"invalid attribute mask=0x%x" + " specified for qpn=0x%x of type=0x%x" + " current_qp_state=0x%x, new_qp_state=0x%x\n", + qp_attr_mask, ib_qp->qp_num, ib_qp->qp_type, + curr_qp_state, new_qp_state); + return -EINVAL; + } + dev_dbg(rdev_to_dev(rdev), "%s:%d INFO attribute mask=0x%x qpn=0x%x " + "of type=0x%x current_qp_state=0x%x, new_qp_state=0x%x\n", + __func__, __LINE__, qp_attr_mask, ib_qp->qp_num, + ib_qp->qp_type, curr_qp_state, new_qp_state); + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; + qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); + + if (udata && curr_qp_state == IB_QPS_RESET && + new_qp_state == IB_QPS_INIT) { + if (!ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { + if (ureq.comp_mask & + BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK) { + ppp->req = BNXT_QPLIB_PPP_REQ; + ppp->dpi = ureq.dpi; + } + } + } + } + if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; + qp->qplib_qp.en_sqd_async_notify = true; + } + if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; + qp->qplib_qp.access = + __from_ib_access_flags(qp_attr->qp_access_flags); + /* LOCAL_WRITE access must be set to allow RC receive */ + qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; + qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE; + qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ; + } + if (qp_attr_mask & IB_QP_PKEY_INDEX) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; + qp->qplib_qp.pkey_index = qp_attr->pkey_index; + } + if (qp_attr_mask & IB_QP_QKEY) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; + qp->qplib_qp.qkey = qp_attr->qkey; + } + if (qp_attr_mask & IB_QP_AV) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | + CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | + CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | + CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | + CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; + memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw, + sizeof(qp->qplib_qp.ah.dgid.data)); + qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label; + qp->qplib_qp.ah.sgid_index = _get_sgid_index(rdev, + qp_attr->ah_attr.grh.sgid_index); + qp->qplib_qp.ah.host_sgid_index = qp_attr->ah_attr.grh.sgid_index; + qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit; + qp->qplib_qp.ah.traffic_class = + qp_attr->ah_attr.grh.traffic_class; + qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl; + ether_addr_copy(qp->qplib_qp.ah.dmac, ROCE_DMAC(&qp_attr->ah_attr)); + sgid_attr = &gid_attr; + status = bnxt_re_get_cached_gid(&rdev->ibdev, 1, + qp_attr->ah_attr.grh.sgid_index, + &sgid, &sgid_attr, + &qp_attr->ah_attr.grh, NULL); + if (!status) + if_rele(sgid_attr->ndev); + gid_ptr = &sgid; + if (sgid_attr->ndev) { + memcpy(qp->qplib_qp.smac, rdev->dev_addr, + ETH_ALEN); + nw_type = bnxt_re_gid_to_network_type(sgid_attr, &sgid); + dev_dbg(rdev_to_dev(rdev), + "Connection using the nw_type %d\n", nw_type); + switch (nw_type) { + case RDMA_NETWORK_IPV4: + qp->qplib_qp.nw_type = + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; + break; + case RDMA_NETWORK_IPV6: + qp->qplib_qp.nw_type = + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; + break; + default: + qp->qplib_qp.nw_type = + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; + break; + } + } + memcpy(&qp->qp_info_entry.sgid, gid_ptr, sizeof(qp->qp_info_entry.sgid)); + } + + /* MTU settings allowed only during INIT -> RTR */ + if (qp_attr->qp_state == IB_QPS_RTR) { + bnxt_re_init_qpmtu(qp, rdev->netdev->if_mtu, qp_attr_mask, qp_attr, + &is_qpmtu_high); + if (udata && !ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { + if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK) { + resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK; + resp.path_mtu = qp->qplib_qp.mtu; + is_copy_to_udata = true; + } else if (is_qpmtu_high) { + dev_err(rdev_to_dev(rdev), "qp %#x invalid mtu\n", + qp->qplib_qp.id); + return -EINVAL; + } + } + } + + if (qp_attr_mask & IB_QP_TIMEOUT) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; + qp->qplib_qp.timeout = qp_attr->timeout; + } + if (qp_attr_mask & IB_QP_RETRY_CNT) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; + qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; + } + if (qp_attr_mask & IB_QP_RNR_RETRY) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; + qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; + } + if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; + qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; + } + if (qp_attr_mask & IB_QP_RQ_PSN) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; + qp->qplib_qp.rq.psn = qp_attr->rq_psn; + } + if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; + /* Cap the max_rd_atomic to device max */ + if (qp_attr->max_rd_atomic > dev_attr->max_qp_rd_atom) + dev_dbg(rdev_to_dev(rdev), + "max_rd_atomic requested %d is > device max %d\n", + qp_attr->max_rd_atomic, + dev_attr->max_qp_rd_atom); + qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, + dev_attr->max_qp_rd_atom); + } + if (qp_attr_mask & IB_QP_SQ_PSN) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; + qp->qplib_qp.sq.psn = qp_attr->sq_psn; + } + if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { + if (qp_attr->max_dest_rd_atomic > + dev_attr->max_qp_init_rd_atom) { + dev_err(rdev_to_dev(rdev), + "max_dest_rd_atomic requested %d is > device max %d\n", + qp_attr->max_dest_rd_atomic, + dev_attr->max_qp_init_rd_atom); + return -EINVAL; + } + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; + qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; + } + if (qp_attr_mask & IB_QP_CAP) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | + CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; + if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || + (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || + (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || + (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || + (qp_attr->cap.max_inline_data >= + dev_attr->max_inline_data)) { + dev_err(rdev_to_dev(rdev), + "Create QP failed - max exceeded\n"); + return -EINVAL; + } + entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); + if (entries > dev_attr->max_qp_wqes) + entries = dev_attr->max_qp_wqes; + entries = min_t(u32, entries, dev_attr->max_qp_wqes); + qp->qplib_qp.sq.max_wqe = entries; + qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - + qp_attr->cap.max_send_wr; + /* + * Reserving one slot for Phantom WQE. Some application can + * post one extra entry in this case. Allowing this to avoid + * unexpected Queue full condition + */ + qp->qplib_qp.sq.q_full_delta -= 1; + qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; + if (qp->qplib_qp.rq.max_wqe) { + entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); + if (entries > dev_attr->max_qp_wqes) + entries = dev_attr->max_qp_wqes; + qp->qplib_qp.rq.max_wqe = entries; + qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - + qp_attr->cap.max_recv_wr; + qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; + } else { + /* SRQ was used prior, just ignore the RQ caps */ + } + } + if (qp_attr_mask & IB_QP_DEST_QPN) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; + qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; + } + + rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "Modify HW QP failed!\n"); + return rc; + } + if (qp_attr_mask & IB_QP_STATE) + bnxt_qplib_manage_flush_qp(qp); + if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK && + ppp->st_idx_en & CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED) { + resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN; + resp.ppp_st_idx = ppp->st_idx_en >> + BNXT_QPLIB_PPP_ST_IDX_SHIFT; + is_copy_to_udata = true; + } + + if (is_copy_to_udata) { + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) + return rc; + } + + if (ib_qp->qp_type == IB_QPT_GSI && + rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && + rdev->gsi_ctx.gsi_sqp) + rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); + /* + * Update info when qp_info_info + */ + bnxt_re_update_qp_info(rdev, qp); + return rc; +} + +int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) +{ + struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_re_dev *rdev = qp->rdev; + struct bnxt_qplib_qp *qplib_qp; + int rc; + + qplib_qp = kcalloc(1, sizeof(*qplib_qp), GFP_KERNEL); + if (!qplib_qp) + return -ENOMEM; + + qplib_qp->id = qp->qplib_qp.id; + qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; + + rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "Query HW QP (0x%x) failed! rc = %d\n", + qplib_qp->id, rc); + goto free_mem; + } + qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); + qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state); + qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; + qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); + qp_attr->pkey_index = qplib_qp->pkey_index; + qp_attr->qkey = qplib_qp->qkey; + memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp->ah.dgid.data, + sizeof(qplib_qp->ah.dgid.data)); + qp_attr->ah_attr.grh.flow_label = qplib_qp->ah.flow_label; + qp_attr->ah_attr.grh.sgid_index = qplib_qp->ah.host_sgid_index; + qp_attr->ah_attr.grh.hop_limit = qplib_qp->ah.hop_limit; + qp_attr->ah_attr.grh.traffic_class = qplib_qp->ah.traffic_class; + qp_attr->ah_attr.sl = qplib_qp->ah.sl; + ether_addr_copy(ROCE_DMAC(&qp_attr->ah_attr), qplib_qp->ah.dmac); + qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); + qp_attr->timeout = qplib_qp->timeout; + qp_attr->retry_cnt = qplib_qp->retry_cnt; + qp_attr->rnr_retry = qplib_qp->rnr_retry; + qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; + qp_attr->rq_psn = qplib_qp->rq.psn; + qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; + qp_attr->sq_psn = qplib_qp->sq.psn; + qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; + qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : + IB_SIGNAL_REQ_WR; + qp_attr->dest_qp_num = qplib_qp->dest_qpn; + + qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; + qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; + qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; + qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; + qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; + qp_init_attr->cap = qp_attr->cap; + +free_mem: + kfree(qplib_qp); + return rc; +} + +/* Builders */ + +/* For Raw, the application is responsible to build the entire packet */ +static void bnxt_re_build_raw_send(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + switch (wr->send_flags) { + case IB_SEND_IP_CSUM: + wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; + break; + default: + /* Pad HW RoCE iCRC */ + wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; + break; + } +} + +/* For QP1, the driver must build the entire RoCE (v1/v2) packet hdr + * as according to the sgid and AV + */ +static int bnxt_re_build_qp1_send(struct bnxt_re_qp *qp, const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe, int payload_size) +{ + struct bnxt_re_ah *ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, + ibah); + struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; + struct bnxt_qplib_sge sge; + int i, rc = 0; + union ib_gid sgid; + u16 vlan_id; + u8 *ptmac; + void *buf; + + memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); + + /* Get sgid */ + rc = bnxt_re_query_gid(&qp->rdev->ibdev, 1, qplib_ah->sgid_index, &sgid); + if (rc) + return rc; + + /* ETH */ + qp->qp1_hdr.eth_present = 1; + ptmac = ah->qplib_ah.dmac; + memcpy(qp->qp1_hdr.eth.dmac_h, ptmac, 4); + ptmac += 4; + memcpy(qp->qp1_hdr.eth.dmac_l, ptmac, 2); + + ptmac = qp->qplib_qp.smac; + memcpy(qp->qp1_hdr.eth.smac_h, ptmac, 2); + ptmac += 2; + memcpy(qp->qp1_hdr.eth.smac_l, ptmac, 4); + + qp->qp1_hdr.eth.type = cpu_to_be16(BNXT_QPLIB_ETHTYPE_ROCEV1); + + /* For vlan, check the sgid for vlan existence */ + vlan_id = rdma_get_vlan_id(&sgid); + if (vlan_id && vlan_id < 0x1000) { + qp->qp1_hdr.vlan_present = 1; + qp->qp1_hdr.eth.type = cpu_to_be16(ETH_P_8021Q); + } + /* GRH */ + qp->qp1_hdr.grh_present = 1; + qp->qp1_hdr.grh.ip_version = 6; + qp->qp1_hdr.grh.payload_length = + cpu_to_be16((IB_BTH_BYTES + IB_DETH_BYTES + payload_size + 7) + & ~3); + qp->qp1_hdr.grh.next_header = 0x1b; + memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid)); + memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, + sizeof(sgid)); + + /* BTH */ + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; + qp->qp1_hdr.immediate_present = 1; + } else { + qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; + } + if (wr->send_flags & IB_SEND_SOLICITED) + qp->qp1_hdr.bth.solicited_event = 1; + qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; + /* P_key for QP1 is for all members */ + qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); + qp->qp1_hdr.bth.destination_qpn = IB_QP1; + qp->qp1_hdr.bth.ack_req = 0; + qp->send_psn++; + qp->send_psn &= BTH_PSN_MASK; + qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); + /* DETH */ + /* Use the priviledged Q_Key for QP1 */ + qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); + qp->qp1_hdr.deth.source_qpn = IB_QP1; + + /* Pack the QP1 to the transmit buffer */ + buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); + if (!buf) { + dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); + rc = -ENOMEM; + } + for (i = wqe->num_sge; i; i--) { + wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; + wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; + wqe->sg_list[i].size = wqe->sg_list[i - 1].size; + } + wqe->sg_list[0].addr = sge.addr; + wqe->sg_list[0].lkey = sge.lkey; + wqe->sg_list[0].size = sge.size; + wqe->num_sge++; + + return rc; +} + +static int bnxt_re_build_gsi_send(struct bnxt_re_qp *qp, + const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_dev *rdev; + int rc, indx, len = 0; + + rdev = qp->rdev; + + /* Mode UD is applicable to Gen P5 only */ + if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) + return 0; + + for (indx = 0; indx < wr->num_sge; indx++) { + wqe->sg_list[indx].addr = wr->sg_list[indx].addr; + wqe->sg_list[indx].lkey = wr->sg_list[indx].lkey; + wqe->sg_list[indx].size = wr->sg_list[indx].length; + len += wr->sg_list[indx].length; + } + rc = bnxt_re_build_qp1_send(qp, wr, wqe, len); + wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; + + return rc; +} + +/* For the MAD layer, it only provides the recv SGE the size of + ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, + nor RoCE iCRC. The Cu+ solution must provide buffer for the entire + receive packet (334 bytes) with no VLAN and then copy the GRH + and the MAD datagram out to the provided SGE. +*/ + +static int bnxt_re_build_qp1_recv(struct bnxt_re_qp *qp, + const struct ib_recv_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_dev *rdev = qp->rdev; + struct bnxt_qplib_sge ref, sge; + u8 udp_hdr_size = 0; + u8 ip_hdr_size = 0; + int rc = 0; + int size; + + if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { + /* Create 5 SGEs as according to the following: + * Ethernet header (14) + * ib_grh (40) - as provided from the wr + * ib_bth + ib_deth + UDP(RoCE v2 only) (28) + * MAD (256) - as provided from the wr + * iCRC (4) + */ + + /* Set RoCE v2 header size and offsets */ + if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ROCE_V2_IPV4) + ip_hdr_size = 20; + if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_ROCE_V1) + udp_hdr_size = 8; + + /* Save the reference from ULP */ + ref.addr = wr->sg_list[0].addr; + ref.lkey = wr->sg_list[0].lkey; + ref.size = wr->sg_list[0].length; + + /* SGE 1 */ + size = sge.size; + wqe->sg_list[0].addr = sge.addr; + wqe->sg_list[0].lkey = sge.lkey; + wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE; + size -= wqe->sg_list[0].size; + if (size <= 0) { + dev_err(rdev_to_dev(qp->rdev),"QP1 rq buffer is empty!\n"); + rc = -ENOMEM; + goto done; + } + sge.size = (u32)size; + sge.addr += wqe->sg_list[0].size; + + /* SGE 2 */ + /* In case of RoCE v2 ipv4 lower 20 bytes should have IP hdr */ + wqe->sg_list[1].addr = ref.addr + ip_hdr_size; + wqe->sg_list[1].lkey = ref.lkey; + wqe->sg_list[1].size = sizeof(struct ib_grh) - ip_hdr_size; + ref.size -= wqe->sg_list[1].size; + if (ref.size <= 0) { + dev_err(rdev_to_dev(qp->rdev), + "QP1 ref buffer is empty!\n"); + rc = -ENOMEM; + goto done; + } + ref.addr += wqe->sg_list[1].size + ip_hdr_size; + + /* SGE 3 */ + wqe->sg_list[2].addr = sge.addr; + wqe->sg_list[2].lkey = sge.lkey; + wqe->sg_list[2].size = BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE + + udp_hdr_size; + size -= wqe->sg_list[2].size; + if (size <= 0) { + dev_err(rdev_to_dev(qp->rdev), + "QP1 rq buffer is empty!\n"); + rc = -ENOMEM; + goto done; + } + sge.size = (u32)size; + sge.addr += wqe->sg_list[2].size; + + /* SGE 4 */ + wqe->sg_list[3].addr = ref.addr; + wqe->sg_list[3].lkey = ref.lkey; + wqe->sg_list[3].size = ref.size; + ref.size -= wqe->sg_list[3].size; + if (ref.size) { + dev_err(rdev_to_dev(qp->rdev), + "QP1 ref buffer is incorrect!\n"); + rc = -ENOMEM; + goto done; + } + /* SGE 5 */ + wqe->sg_list[4].addr = sge.addr; + wqe->sg_list[4].lkey = sge.lkey; + wqe->sg_list[4].size = sge.size; + size -= wqe->sg_list[4].size; + if (size) { + dev_err(rdev_to_dev(qp->rdev), + "QP1 rq buffer is incorrect!\n"); + rc = -ENOMEM; + goto done; + } + sge.size = (u32)size; + wqe->num_sge = 5; + } else { + dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); + rc = -ENOMEM; + } +done: + return rc; +} + +static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, + const struct ib_recv_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_sqp_entries *sqp_entry; + struct bnxt_qplib_sge sge; + struct bnxt_re_dev *rdev; + u32 rq_prod_index; + int rc = 0; + + rdev = qp->rdev; + + rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); + + if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { + /* Create 1 SGE to receive the entire + * ethernet packet + */ + /* SGE 1 */ + wqe->sg_list[0].addr = sge.addr; + /* TODO check the lkey to be used */ + wqe->sg_list[0].lkey = sge.lkey; + wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; + if (sge.size < wqe->sg_list[0].size) { + dev_err(rdev_to_dev(qp->rdev), + "QP1 rq buffer is empty!\n"); + rc = -ENOMEM; + goto done; + } + + sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index]; + sqp_entry->sge.addr = wr->sg_list[0].addr; + sqp_entry->sge.lkey = wr->sg_list[0].lkey; + sqp_entry->sge.size = wr->sg_list[0].length; + /* Store the wrid for reporting completion */ + sqp_entry->wrid = wqe->wr_id; + /* change the wqe->wrid to table index */ + wqe->wr_id = rq_prod_index; + } +done: + return rc; +} + +static bool is_ud_qp(struct bnxt_re_qp *qp) +{ + return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD || + qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI); +} + +static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, + const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_ah *ah = NULL; + + if(is_ud_qp(qp)) { + ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, ibah); + wqe->send.q_key = ud_wr(wr)->remote_qkey; + wqe->send.dst_qp = ud_wr(wr)->remote_qpn; + wqe->send.avid = ah->qplib_ah.id; + } + switch (wr->opcode) { + case IB_WR_SEND: + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; + break; + case IB_WR_SEND_WITH_IMM: + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; + wqe->send.imm_data = wr->ex.imm_data; + break; + case IB_WR_SEND_WITH_INV: + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; + wqe->send.inv_key = wr->ex.invalidate_rkey; + break; + default: + dev_err(rdev_to_dev(qp->rdev), "%s Invalid opcode %d!\n", + __func__, wr->opcode); + return -EINVAL; + } + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_FENCE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + if (wr->send_flags & IB_SEND_INLINE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; + + return 0; +} + +static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + switch (wr->opcode) { + case IB_WR_RDMA_WRITE: + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; + break; + case IB_WR_RDMA_WRITE_WITH_IMM: + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; + wqe->rdma.imm_data = wr->ex.imm_data; + break; + case IB_WR_RDMA_READ: + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; + wqe->rdma.inv_key = wr->ex.invalidate_rkey; + break; + default: + return -EINVAL; + } + wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; + wqe->rdma.r_key = rdma_wr(wr)->rkey; + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_FENCE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + if (wr->send_flags & IB_SEND_INLINE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; + + return 0; +} + +static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + switch (wr->opcode) { + case IB_WR_ATOMIC_CMP_AND_SWP: + wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; + wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; + wqe->atomic.swap_data = atomic_wr(wr)->swap; + break; + case IB_WR_ATOMIC_FETCH_AND_ADD: + wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; + wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; + break; + default: + return -EINVAL; + } + wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; + wqe->atomic.r_key = atomic_wr(wr)->rkey; + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_FENCE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + return 0; +} + +static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; + wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_FENCE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + + return 0; +} + +static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_mr *mr = to_bnxt_re(wr->mr, struct bnxt_re_mr, ib_mr); + struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; + int reg_len, i, access = wr->access; + + if (mr->npages > qplib_frpl->max_pg_ptrs) { + dev_err_ratelimited(rdev_to_dev(mr->rdev), + " %s: failed npages %d > %d\n", __func__, + mr->npages, qplib_frpl->max_pg_ptrs); + return -EINVAL; + } + + wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; + wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; + wqe->frmr.levels = qplib_frpl->hwq.level; + wqe->frmr.page_list = mr->pages; + wqe->frmr.page_list_len = mr->npages; + wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; + + if (wr->wr.send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (access & IB_ACCESS_LOCAL_WRITE) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; + if (access & IB_ACCESS_REMOTE_READ) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; + if (access & IB_ACCESS_REMOTE_WRITE) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; + if (access & IB_ACCESS_REMOTE_ATOMIC) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; + if (access & IB_ACCESS_MW_BIND) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; + + /* TODO: OFED provides the rkey of the MR instead of the lkey */ + wqe->frmr.l_key = wr->key; + wqe->frmr.length = wr->mr->length; + wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K); + wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K); + wqe->frmr.va = wr->mr->iova; + reg_len = wqe->frmr.page_list_len * wr->mr->page_size; + + if (wqe->frmr.length > reg_len) { + dev_err_ratelimited(rdev_to_dev(mr->rdev), + "%s: bnxt_re_mr 0x%px len (%d > %d)\n", + __func__, (void *)mr, wqe->frmr.length, + reg_len); + + for (i = 0; i < mr->npages; i++) + dev_dbg(rdev_to_dev(mr->rdev), + "%s: build_reg_wqe page[%d] = 0x%llx\n", + __func__, i, mr->pages[i]); + + return -EINVAL; + } + + return 0; +} + +static void bnxt_re_set_sg_list(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + wqe->sg_list = (struct bnxt_qplib_sge *)wr->sg_list; + wqe->num_sge = wr->num_sge; +} + +static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) +{ + if ((qp->ib_qp.qp_type == IB_QPT_UD || qp->ib_qp.qp_type == IB_QPT_GSI || + qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && + qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { + int qp_attr_mask; + struct ib_qp_attr qp_attr; + + qp_attr_mask = IB_QP_STATE; + qp_attr.qp_state = IB_QPS_RTS; + bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); + qp->qplib_qp.wqe_cnt = 0; + } +} + +static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp, + const struct ib_send_wr *wr) +{ + struct bnxt_qplib_swqe wqe; + unsigned long flags; + int rc = 0; + + spin_lock_irqsave(&qp->sq_lock, flags); + while (wr) { + /* House keeping */ + memset(&wqe, 0, sizeof(wqe)); + /* Common */ + if (wr->num_sge > qp->qplib_qp.sq.max_sge) { + dev_err(rdev_to_dev(rdev), + "Limit exceeded for Send SGEs\n"); + rc = -EINVAL; + break; + } + + bnxt_re_set_sg_list(wr, &wqe); + wqe.wr_id = wr->wr_id; + wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; + rc = bnxt_re_build_send_wqe(qp, wr, &wqe); + if (rc) + break; + + rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); + if (rc) { + dev_err(rdev_to_dev(rdev), + "bad_wr seen with opcode = 0x%x rc = %d\n", + wr->opcode, rc); + break; + } + wr = wr->next; + } + bnxt_qplib_post_send_db(&qp->qplib_qp); + bnxt_ud_qp_hw_stall_workaround(qp); + spin_unlock_irqrestore(&qp->sq_lock, flags); + return rc; +} + +static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe) +{ + /* Need unconditional fence for non-wire memory opcode + * to work as expected. + */ + if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV || + wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR || + wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR || + wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; +} + +int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_qplib_sge sge[6]; + struct bnxt_qplib_swqe wqe; + struct bnxt_re_dev *rdev; + unsigned long flags; + int rc = 0; + + rdev = qp->rdev; + spin_lock_irqsave(&qp->sq_lock, flags); + while (wr) { + /* House keeping */ + memset(&wqe, 0, sizeof(wqe)); + /* Common */ + if (wr->num_sge > qp->qplib_qp.sq.max_sge) { + dev_err(rdev_to_dev(rdev), + "Limit exceeded for Send SGEs\n"); + rc = -EINVAL; + goto bad; + } + + bnxt_re_set_sg_list(wr, &wqe); + wqe.wr_id = wr->wr_id; + + switch (wr->opcode) { + case IB_WR_SEND: + case IB_WR_SEND_WITH_IMM: + if (ib_qp->qp_type == IB_QPT_GSI && + rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { + memset(sge, 0, sizeof(sge)); + wqe.sg_list = sge; + rc = bnxt_re_build_gsi_send(qp, wr, &wqe); + if (rc) + goto bad; + } else if (ib_qp->qp_type == IB_QPT_RAW_ETHERTYPE) { + bnxt_re_build_raw_send(wr, &wqe); + } + switch (wr->send_flags) { + case IB_SEND_IP_CSUM: + wqe.rawqp1.lflags |= + SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; + break; + default: + break; + } + fallthrough; + case IB_WR_SEND_WITH_INV: + rc = bnxt_re_build_send_wqe(qp, wr, &wqe); + break; + case IB_WR_RDMA_WRITE: + case IB_WR_RDMA_WRITE_WITH_IMM: + case IB_WR_RDMA_READ: + rc = bnxt_re_build_rdma_wqe(wr, &wqe); + break; + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + rc = bnxt_re_build_atomic_wqe(wr, &wqe); + break; + case IB_WR_RDMA_READ_WITH_INV: + dev_err(rdev_to_dev(rdev), + "RDMA Read with Invalidate is not supported\n"); + rc = -EINVAL; + goto bad; + case IB_WR_LOCAL_INV: + rc = bnxt_re_build_inv_wqe(wr, &wqe); + break; + case IB_WR_REG_MR: + rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); + break; + default: + /* Unsupported WRs */ + dev_err(rdev_to_dev(rdev), + "WR (0x%x) is not supported\n", wr->opcode); + rc = -EINVAL; + goto bad; + } + + if (likely(!rc)) { + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + bnxt_re_legacy_set_uc_fence(&wqe); + rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); + } +bad: + if (unlikely(rc)) { + dev_err(rdev_to_dev(rdev), + "bad_wr seen with opcode = 0x%x\n", wr->opcode); + *bad_wr = wr; + break; + } + wr = wr->next; + } + bnxt_qplib_post_send_db(&qp->qplib_qp); + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + bnxt_ud_qp_hw_stall_workaround(qp); + spin_unlock_irqrestore(&qp->sq_lock, flags); + + return rc; +} + +static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp, + struct ib_recv_wr *wr) +{ + struct bnxt_qplib_swqe wqe; + int rc = 0; + + /* rq lock can be pardoned here. */ + while (wr) { + /* House keeping */ + memset(&wqe, 0, sizeof(wqe)); + /* Common */ + if (wr->num_sge > qp->qplib_qp.rq.max_sge) { + dev_err(rdev_to_dev(rdev), + "Limit exceeded for Receive SGEs\n"); + rc = -EINVAL; + goto bad; + } + + wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; + wqe.num_sge = wr->num_sge; + wqe.wr_id = wr->wr_id; + wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; + rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); +bad: + if (rc) { + dev_err(rdev_to_dev(rdev), + "bad_wr seen with RQ post\n"); + break; + } + wr = wr->next; + } + bnxt_qplib_post_recv_db(&qp->qplib_qp); + return rc; +} + +static int bnxt_re_build_gsi_recv(struct bnxt_re_qp *qp, + const struct ib_recv_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_dev *rdev = qp->rdev; + int rc = 0; + + if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) + rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, wqe); + else + rc = bnxt_re_build_qp1_recv(qp, wr, wqe); + + return rc; +} + +int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_qplib_sge sge[6]; + struct bnxt_qplib_swqe wqe; + unsigned long flags; + u32 count = 0; + int rc = 0; + + spin_lock_irqsave(&qp->rq_lock, flags); + while (wr) { + memset(&wqe, 0, sizeof(wqe)); + if (wr->num_sge > qp->qplib_qp.rq.max_sge) { + dev_err(rdev_to_dev(qp->rdev), + "Limit exceeded for Receive SGEs\n"); + rc = -EINVAL; + goto bad; + } + wqe.num_sge = wr->num_sge; + wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; + wqe.wr_id = wr->wr_id; + wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; + + if (ib_qp->qp_type == IB_QPT_GSI && + qp->rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { + memset(sge, 0, sizeof(sge)); + wqe.sg_list = sge; + rc = bnxt_re_build_gsi_recv(qp, wr, &wqe); + if (rc) + goto bad; + } + rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); +bad: + if (rc) { + dev_err(rdev_to_dev(qp->rdev), + "bad_wr seen with RQ post\n"); + *bad_wr = wr; + break; + } + /* Ring DB if the RQEs posted reaches a threshold value */ + if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) { + bnxt_qplib_post_recv_db(&qp->qplib_qp); + count = 0; + } + wr = wr->next; + } + + if (count) + bnxt_qplib_post_recv_db(&qp->qplib_qp); + spin_unlock_irqrestore(&qp->rq_lock, flags); + + return rc; +} + +/* Completion Queues */ +void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +{ + struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); + struct bnxt_re_dev *rdev = cq->rdev; + int rc = 0; + + if (cq->uctx_cq_page) { + BNXT_RE_CQ_PAGE_LIST_DEL(cq->uctx, cq); + free_page((u64)cq->uctx_cq_page); + cq->uctx_cq_page = NULL; + } + if (cq->is_dbr_soft_cq && cq->uctx) { + void *dbr_page; + + if (cq->uctx->dbr_recov_cq) { + dbr_page = cq->uctx->dbr_recov_cq_page; + cq->uctx->dbr_recov_cq_page = NULL; + cq->uctx->dbr_recov_cq = NULL; + free_page((unsigned long)dbr_page); + } + goto end; + } + /* CQ getting destroyed. Set this state for cqn handler */ + spin_lock_bh(&cq->qplib_cq.compl_lock); + cq->qplib_cq.destroyed = true; + spin_unlock_bh(&cq->qplib_cq.compl_lock); + if (ib_cq->poll_ctx == IB_POLL_WORKQUEUE || + ib_cq->poll_ctx == IB_POLL_UNBOUND_WORKQUEUE) + cancel_work_sync(&ib_cq->work); + + rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + if (rc) + dev_err_ratelimited(rdev_to_dev(rdev), + "%s id = %d failed rc = %d\n", + __func__, cq->qplib_cq.id, rc); + + bnxt_re_put_nq(rdev, cq->qplib_cq.nq); + if (cq->umem && !IS_ERR(cq->umem)) + ib_umem_release(cq->umem); + + kfree(cq->cql); + atomic_dec(&rdev->stats.rsors.cq_count); +end: + return; +} + +static inline struct +bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in, + struct bnxt_re_dev *rdev) +{ + struct bnxt_re_cq *cq; + cq = container_of(cq_in, struct bnxt_re_cq, ibcq); + return cq; +} + +int bnxt_re_create_cq(struct ib_cq *cq_in, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_re_ucontext *uctx = NULL; + struct ib_ucontext *context = NULL; + struct bnxt_qplib_cq *qplcq; + struct bnxt_re_cq_req ureq; + struct bnxt_re_dev *rdev; + int rc, entries; + struct bnxt_re_cq *cq; + u32 max_active_cqs; + int cqe = attr->cqe; + + if (attr->flags) + return -EOPNOTSUPP; + + rdev = rdev_from_cq_in(cq_in); + if (rdev->mod_exit) { + rc = -EIO; + dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); + goto exit; + } + if (udata) { + uctx = rdma_udata_to_drv_context(udata, + struct bnxt_re_ucontext, + ibucontext); + context = &uctx->ibucontext; + } + dev_attr = rdev->dev_attr; + + if (atomic_read(&rdev->stats.rsors.cq_count) >= dev_attr->max_cq) { + dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQs)\n"); + rc = -EINVAL; + goto exit; + } + /* Validate CQ fields */ + if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { + dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQ_WQs)\n"); + rc = -EINVAL; + goto exit; + } + + cq = __get_cq_from_cq_in(cq_in, rdev); + if (!cq) { + rc = -ENOMEM; + goto exit; + } + cq->rdev = rdev; + cq->uctx = uctx; + qplcq = &cq->qplib_cq; + qplcq->cq_handle = (u64)qplcq; + /* + * Since CQ is for QP1 is shared with Shadow CQ, the size + * should be double the size. There is no way to identify + * whether this CQ is for GSI QP. So assuming that the first + * CQ created is for QP1 + */ + if (!udata && !rdev->gsi_ctx.first_cq_created && + rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) { + rdev->gsi_ctx.first_cq_created = true; + /* + * Total CQE required for the CQ = CQE for QP1 RQ + + * CQE for Shadow QP SQEs + CQE for Shadow QP RQEs. + * Max entries of shadow QP SQ and RQ = QP1 RQEs = cqe + */ + cqe *= 3; + } + + entries = bnxt_re_init_depth(cqe + 1, uctx); + if (entries > dev_attr->max_cq_wqes + 1) + entries = dev_attr->max_cq_wqes + 1; + + qplcq->sginfo.pgshft = PAGE_SHIFT; + qplcq->sginfo.pgsize = PAGE_SIZE; + if (udata) { + if (udata->inlen < sizeof(ureq)) + dev_warn(rdev_to_dev(rdev), + "Update the library ulen %d klen %d\n", + (unsigned int)udata->inlen, + (unsigned int)sizeof(ureq)); + + rc = ib_copy_from_udata(&ureq, udata, + min(udata->inlen, sizeof(ureq))); + if (rc) + goto fail; + + if (BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(ureq)) { + cq->is_dbr_soft_cq = true; + goto success; + } + + if (BNXT_RE_IS_DBR_RECOV_CQ(ureq)) { + void *dbr_page; + u32 *epoch; + + dbr_page = (void *)__get_free_page(GFP_KERNEL); + if (!dbr_page) { + dev_err(rdev_to_dev(rdev), + "DBR recov CQ page allocation failed!"); + rc = -ENOMEM; + goto fail; + } + + /* memset the epoch and epoch_ack to 0 */ + epoch = dbr_page; + epoch[0] = 0x0; + epoch[1] = 0x0; + + uctx->dbr_recov_cq = cq; + uctx->dbr_recov_cq_page = dbr_page; + + cq->is_dbr_soft_cq = true; + goto success; + } + + cq->umem = ib_umem_get_compat + (rdev, context, udata, ureq.cq_va, + entries * sizeof(struct cq_base), + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(cq->umem)) { + rc = PTR_ERR(cq->umem); + dev_err(rdev_to_dev(rdev), + "%s: ib_umem_get failed! rc = %d\n", + __func__, rc); + goto fail; + } + qplcq->sginfo.sghead = get_ib_umem_sgl(cq->umem, + &qplcq->sginfo.nmap); + qplcq->sginfo.npages = ib_umem_num_pages_compat(cq->umem); + if (!uctx->dpi.dbr) { + rc = bnxt_re_get_user_dpi(rdev, uctx); + if (rc) + goto c2fail; + } + qplcq->dpi = &uctx->dpi; + } else { + cq->max_cql = entries > MAX_CQL_PER_POLL ? MAX_CQL_PER_POLL : entries; + cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), + GFP_KERNEL); + if (!cq->cql) { + dev_err(rdev_to_dev(rdev), + "Allocate CQL for %d failed!\n", cq->max_cql); + rc = -ENOMEM; + goto fail; + } + qplcq->dpi = &rdev->dpi_privileged; + } + /* + * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a + * used for getting the NQ index. + */ + qplcq->max_wqe = entries; + qplcq->nq = bnxt_re_get_nq(rdev); + qplcq->cnq_hw_ring_id = qplcq->nq->ring_id; + + rc = bnxt_qplib_create_cq(&rdev->qplib_res, qplcq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Create HW CQ failed!\n"); + goto fail; + } + + INIT_LIST_HEAD(&cq->cq_list); + cq->ibcq.cqe = entries; + cq->cq_period = qplcq->period; + + atomic_inc(&rdev->stats.rsors.cq_count); + max_active_cqs = atomic_read(&rdev->stats.rsors.cq_count); + if (max_active_cqs > atomic_read(&rdev->stats.rsors.max_cq_count)) + atomic_set(&rdev->stats.rsors.max_cq_count, max_active_cqs); + spin_lock_init(&cq->cq_lock); + + if (udata) { + struct bnxt_re_cq_resp resp; + + resp.cqid = qplcq->id; + resp.tail = qplcq->hwq.cons; + resp.phase = qplcq->period; + resp.comp_mask = 0; + resp.dbr = (u64)uctx->dpi.umdbr; + resp.dpi = uctx->dpi.dpi; + resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO; + /* Copy only on a valid wcpdi */ + if (uctx->wcdpi.dpi) { + resp.wcdpi = uctx->wcdpi.dpi; + resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI; + } + + if (_is_chip_p7(rdev->chip_ctx)) { + cq->uctx_cq_page = (void *)__get_free_page(GFP_KERNEL); + + if (!cq->uctx_cq_page) { + dev_err(rdev_to_dev(rdev), + "CQ page allocation failed!\n"); + bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); + rc = -ENOMEM; + goto c2fail; + } + + resp.uctx_cq_page = (u64)cq->uctx_cq_page; + resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE; + } + + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) { + free_page((u64)cq->uctx_cq_page); + cq->uctx_cq_page = NULL; + bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); + goto c2fail; + } + + if (cq->uctx_cq_page) + BNXT_RE_CQ_PAGE_LIST_ADD(uctx, cq); + } + +success: + return 0; +c2fail: + if (udata && cq->umem && !IS_ERR(cq->umem)) + ib_umem_release(cq->umem); +fail: + if (cq) { + if (cq->cql) + kfree(cq->cql); + } +exit: + return rc; +} + +int bnxt_re_modify_cq(struct ib_cq *ib_cq, u16 cq_count, u16 cq_period) +{ + struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); + struct bnxt_re_dev *rdev = cq->rdev; + int rc; + + if ((cq->cq_count != cq_count) || (cq->cq_period != cq_period)) { + cq->qplib_cq.count = cq_count; + cq->qplib_cq.period = cq_period; + rc = bnxt_qplib_modify_cq(&rdev->qplib_res, &cq->qplib_cq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Modify HW CQ %#x failed!\n", + cq->qplib_cq.id); + return rc; + } + /* On success, update the shadow */ + cq->cq_count = cq_count; + cq->cq_period = cq_period; + } + return 0; +} + +static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq) +{ + struct bnxt_re_dev *rdev = cq->rdev; + + bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); + + cq->qplib_cq.max_wqe = cq->resize_cqe; + if (cq->resize_umem) { + ib_umem_release(cq->umem); + cq->umem = cq->resize_umem; + cq->resize_umem = NULL; + cq->resize_cqe = 0; + } +} + +int bnxt_re_resize_cq(struct ib_cq *ib_cq, int cqe, struct ib_udata *udata) +{ + struct bnxt_qplib_sg_info sginfo = {}; + struct bnxt_qplib_dpi *orig_dpi = NULL; + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_re_ucontext *uctx = NULL; + struct bnxt_re_resize_cq_req ureq; + struct ib_ucontext *context = NULL; + struct bnxt_re_dev *rdev; + struct bnxt_re_cq *cq; + int rc, entries; + + /* Don't allow more than one resize request at the same time. + * TODO: need a mutex here when we support kernel consumers of resize. + */ + cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); + rdev = cq->rdev; + dev_attr = rdev->dev_attr; + if (ib_cq->uobject) { + uctx = rdma_udata_to_drv_context(udata, + struct bnxt_re_ucontext, + ibucontext); + context = &uctx->ibucontext; + } + + if (cq->resize_umem) { + dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - Busy\n", + cq->qplib_cq.id); + return -EBUSY; + } + + /* Check the requested cq depth out of supported depth */ + if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { + dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - max exceeded\n", + cq->qplib_cq.id); + return -EINVAL; + } + + entries = bnxt_re_init_depth(cqe + 1, uctx); + entries = min_t(u32, (u32)entries, dev_attr->max_cq_wqes + 1); + + /* Check to see if the new requested size can be handled by already + * existing CQ + */ + if (entries == cq->ibcq.cqe) { + dev_info(rdev_to_dev(rdev), "CQ is already at size %d\n", cqe); + return 0; + } + + if (ib_cq->uobject && udata) { + if (udata->inlen < sizeof(ureq)) + dev_warn(rdev_to_dev(rdev), + "Update the library ulen %d klen %d\n", + (unsigned int)udata->inlen, + (unsigned int)sizeof(ureq)); + + rc = ib_copy_from_udata(&ureq, udata, + min(udata->inlen, sizeof(ureq))); + if (rc) + goto fail; + + dev_dbg(rdev_to_dev(rdev), "%s: va %p\n", __func__, + (void *)ureq.cq_va); + cq->resize_umem = ib_umem_get_compat + (rdev, + context, udata, ureq.cq_va, + entries * sizeof(struct cq_base), + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(cq->resize_umem)) { + rc = PTR_ERR(cq->resize_umem); + cq->resize_umem = NULL; + dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n", + __func__, rc); + goto fail; + } + cq->resize_cqe = entries; + dev_dbg(rdev_to_dev(rdev), "%s: ib_umem_get() success\n", + __func__); + memcpy(&sginfo, &cq->qplib_cq.sginfo, sizeof(sginfo)); + orig_dpi = cq->qplib_cq.dpi; + + cq->qplib_cq.sginfo.sghead = get_ib_umem_sgl(cq->resize_umem, + &cq->qplib_cq.sginfo.nmap); + cq->qplib_cq.sginfo.npages = + ib_umem_num_pages_compat(cq->resize_umem); + cq->qplib_cq.sginfo.pgsize = PAGE_SIZE; + cq->qplib_cq.sginfo.pgshft = PAGE_SHIFT; + cq->qplib_cq.dpi = &uctx->dpi; + } else { + /* TODO: kernel consumer */ + } + + rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries); + if (rc) { + dev_err(rdev_to_dev(rdev), "Resize HW CQ %#x failed!\n", + cq->qplib_cq.id); + goto fail; + } + + cq->ibcq.cqe = cq->resize_cqe; + /* For kernel consumers complete resize here. For uverbs consumers, + * we complete it in the context of ibv_poll_cq(). + */ + if (!cq->resize_umem) + bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); + + atomic_inc(&rdev->stats.rsors.resize_count); + return 0; + +fail: + if (cq->resize_umem) { + ib_umem_release(cq->resize_umem); + cq->resize_umem = NULL; + cq->resize_cqe = 0; + memcpy(&cq->qplib_cq.sginfo, &sginfo, sizeof(sginfo)); + cq->qplib_cq.dpi = orig_dpi; + } + return rc; +} + +static enum ib_wc_status __req_to_ib_wc_status(u8 qstatus) +{ + switch(qstatus) { + case CQ_REQ_STATUS_OK: + return IB_WC_SUCCESS; + case CQ_REQ_STATUS_BAD_RESPONSE_ERR: + return IB_WC_BAD_RESP_ERR; + case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: + return IB_WC_LOC_LEN_ERR; + case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: + return IB_WC_LOC_QP_OP_ERR; + case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: + return IB_WC_LOC_PROT_ERR; + case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: + return IB_WC_GENERAL_ERR; + case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: + return IB_WC_REM_INV_REQ_ERR; + case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: + return IB_WC_REM_ACCESS_ERR; + case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: + return IB_WC_REM_OP_ERR; + case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: + return IB_WC_RNR_RETRY_EXC_ERR; + case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: + return IB_WC_RETRY_EXC_ERR; + case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: + return IB_WC_WR_FLUSH_ERR; + default: + return IB_WC_GENERAL_ERR; + } + return 0; +} + +static enum ib_wc_status __rawqp1_to_ib_wc_status(u8 qstatus) +{ + switch(qstatus) { + case CQ_RES_RAWETH_QP1_STATUS_OK: + return IB_WC_SUCCESS; + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: + return IB_WC_LOC_ACCESS_ERR; + case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: + return IB_WC_LOC_LEN_ERR; + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: + return IB_WC_LOC_PROT_ERR; + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: + return IB_WC_LOC_QP_OP_ERR; + case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: + return IB_WC_GENERAL_ERR; + case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: + return IB_WC_WR_FLUSH_ERR; + case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: + return IB_WC_WR_FLUSH_ERR; + default: + return IB_WC_GENERAL_ERR; + } +} + +static enum ib_wc_status __rc_to_ib_wc_status(u8 qstatus) +{ + switch(qstatus) { + case CQ_RES_RC_STATUS_OK: + return IB_WC_SUCCESS; + case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: + return IB_WC_LOC_ACCESS_ERR; + case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: + return IB_WC_LOC_LEN_ERR; + case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: + return IB_WC_LOC_PROT_ERR; + case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: + return IB_WC_LOC_QP_OP_ERR; + case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: + return IB_WC_GENERAL_ERR; + case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: + return IB_WC_REM_INV_REQ_ERR; + case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: + return IB_WC_WR_FLUSH_ERR; + case CQ_RES_RC_STATUS_HW_FLUSH_ERR: + return IB_WC_WR_FLUSH_ERR; + default: + return IB_WC_GENERAL_ERR; + } +} + +static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) +{ + switch (cqe->type) { + case BNXT_QPLIB_SWQE_TYPE_SEND: + wc->opcode = IB_WC_SEND; + break; + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: + wc->opcode = IB_WC_SEND; + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: + wc->opcode = IB_WC_SEND; + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + break; + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: + wc->opcode = IB_WC_RDMA_WRITE; + break; + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RDMA_WRITE; + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: + wc->opcode = IB_WC_RDMA_READ; + break; + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: + wc->opcode = IB_WC_COMP_SWAP; + break; + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: + wc->opcode = IB_WC_FETCH_ADD; + break; + case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: + wc->opcode = IB_WC_LOCAL_INV; + break; + case BNXT_QPLIB_SWQE_TYPE_REG_MR: + wc->opcode = IB_WC_REG_MR; + break; + default: + wc->opcode = IB_WC_SEND; + break; + } + + wc->status = __req_to_ib_wc_status(cqe->status); +} + +static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, u16 raweth_qp1_flags2) +{ + bool is_ipv6 = false, is_ipv4 = false; + + /* raweth_qp1_flags Bit 9-6 indicates itype */ + + if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) + != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) + return -1; + + if (raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && + raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { + /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ + (raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? + (is_ipv6 = true) : (is_ipv4 = true); + return ((is_ipv6) ? + BNXT_RE_ROCEV2_IPV6_PACKET : + BNXT_RE_ROCEV2_IPV4_PACKET); + } else { + return BNXT_RE_ROCE_V1_PACKET; + } +} + +static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, + void *rq_hdr_buf) +{ + u8 *tmp_buf = NULL; + struct ethhdr *eth_hdr; + u16 eth_type; + bool rc = false; + + tmp_buf = (u8 *)rq_hdr_buf; + /* + * If dest mac is not same as I/F mac, this could be a + * loopback address or multicast address, check whether + * it is a loopback packet + */ + if (!ether_addr_equal(tmp_buf, rdev->dev_addr)) { + tmp_buf += 4; + /* Check the ether type */ + eth_hdr = (struct ethhdr *)tmp_buf; + eth_type = ntohs(eth_hdr->h_proto); + switch (eth_type) { + case BNXT_QPLIB_ETHTYPE_ROCEV1: + rc = true; + break; + default: + break; + } + } + + return rc; +} + +static bool bnxt_re_is_vlan_in_packet(struct bnxt_re_dev *rdev, + void *rq_hdr_buf, + struct bnxt_qplib_cqe *cqe) +{ + struct vlan_hdr *vlan_hdr; + struct ethhdr *eth_hdr; + u8 *tmp_buf = NULL; + u16 eth_type; + + tmp_buf = (u8 *)rq_hdr_buf; + /* Check the ether type */ + eth_hdr = (struct ethhdr *)tmp_buf; + eth_type = ntohs(eth_hdr->h_proto); + if (eth_type == ETH_P_8021Q) { + tmp_buf += sizeof(struct ethhdr); + vlan_hdr = (struct vlan_hdr *)tmp_buf; + cqe->raweth_qp1_metadata = + ntohs(vlan_hdr->h_vlan_TCI) | + (eth_type << + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); + cqe->raweth_qp1_flags2 |= + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN; + return true; + } + + return false; +} + +static int bnxt_re_process_raw_qp_packet_receive(struct bnxt_re_qp *gsi_qp, + struct bnxt_qplib_cqe *cqe) +{ + struct bnxt_re_sqp_entries *sqp_entry = NULL; + struct bnxt_qplib_hdrbuf *hdr_buf; + dma_addr_t shrq_hdr_buf_map; + struct ib_sge s_sge[2] = {}; + struct ib_sge r_sge[2] = {}; + struct ib_recv_wr rwr = {}; + struct bnxt_re_ah *gsi_sah; + struct bnxt_re_qp *gsi_sqp; + dma_addr_t rq_hdr_buf_map; + struct bnxt_re_dev *rdev; + struct ib_send_wr *swr; + u32 skip_bytes = 0; + void *rq_hdr_buf; + int pkt_type = 0; + u32 offset = 0; + u32 tbl_idx; + int rc; + struct ib_ud_wr udwr = {}; + + swr = &udwr.wr; + rdev = gsi_qp->rdev; + gsi_sqp = rdev->gsi_ctx.gsi_sqp; + tbl_idx = cqe->wr_id; + + hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf; + rq_hdr_buf = (u8 *) hdr_buf->va + tbl_idx * hdr_buf->step; + rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, + tbl_idx); + /* Shadow QP header buffer */ + shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_sqp->qplib_qp, + tbl_idx); + sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; + + /* Find packet type from the cqe */ + pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, + cqe->raweth_qp1_flags2); + if (pkt_type < 0) { + dev_err(rdev_to_dev(rdev), "Not handling this packet\n"); + return -EINVAL; + } + + /* Adjust the offset for the user buffer and post in the rq */ + + if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) + offset = 20; + + /* + * QP1 loopback packet has 4 bytes of internal header before + * ether header. Skip these four bytes. + */ + if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) + skip_bytes = 4; + + if (bnxt_re_is_vlan_in_packet(rdev, rq_hdr_buf, cqe)) + skip_bytes += VLAN_HLEN; + + /* Store this cqe */ + memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); + sqp_entry->qp1_qp = gsi_qp; + + /* First send SGE . Skip the ether header*/ + s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE + + skip_bytes; + s_sge[0].lkey = 0xFFFFFFFF; + s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : + BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; + + /* Second Send SGE */ + s_sge[1].addr = s_sge[0].addr + s_sge[0].length + + BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; + if (pkt_type != BNXT_RE_ROCE_V1_PACKET) + s_sge[1].addr += 8; + s_sge[1].lkey = 0xFFFFFFFF; + s_sge[1].length = 256; + + /* First recv SGE */ + r_sge[0].addr = shrq_hdr_buf_map; + r_sge[0].lkey = 0xFFFFFFFF; + r_sge[0].length = 40; + + r_sge[1].addr = sqp_entry->sge.addr + offset; + r_sge[1].lkey = sqp_entry->sge.lkey; + r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; + + /* Create receive work request */ + rwr.num_sge = 2; + rwr.sg_list = r_sge; + rwr.wr_id = tbl_idx; + rwr.next = NULL; + + rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to post Rx buffers to shadow QP\n"); + return -ENOMEM; + } + + swr->num_sge = 2; + swr->sg_list = s_sge; + swr->wr_id = tbl_idx; + swr->opcode = IB_WR_SEND; + swr->next = NULL; + + gsi_sah = rdev->gsi_ctx.gsi_sah; + udwr.ah = &gsi_sah->ibah; + udwr.remote_qpn = gsi_sqp->qplib_qp.id; + udwr.remote_qkey = gsi_sqp->qplib_qp.qkey; + /* post data received in the send queue */ + rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr); + + return rc; +} + +static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + wc->opcode = IB_WC_RECV; + wc->status = __rawqp1_to_ib_wc_status(cqe->status); + wc->wc_flags |= IB_WC_GRH; +} + +static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + wc->opcode = IB_WC_RECV; + wc->status = __rc_to_ib_wc_status(cqe->status); + + if (cqe->flags & CQ_RES_RC_FLAGS_IMM) + wc->wc_flags |= IB_WC_WITH_IMM; + if (cqe->flags & CQ_RES_RC_FLAGS_INV) + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == + (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; +} + +/* Returns TRUE if pkt has valid VLAN and if VLAN id is non-zero */ +static bool bnxt_re_is_nonzero_vlanid_pkt(struct bnxt_qplib_cqe *orig_cqe, + u16 *vid, u8 *sl) +{ + u32 metadata; + u16 tpid; + bool ret = false; + metadata = orig_cqe->raweth_qp1_metadata; + if (orig_cqe->raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) { + tpid = ((metadata & + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >> + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); + if (tpid == ETH_P_8021Q) { + *vid = metadata & + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK; + *sl = (metadata & + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >> + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT; + ret = !!(*vid); + } + } + + return ret; +} + +static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp, + struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + u32 tbl_idx; + struct bnxt_re_dev *rdev = gsi_sqp->rdev; + struct bnxt_re_qp *gsi_qp = NULL; + struct bnxt_qplib_cqe *orig_cqe = NULL; + struct bnxt_re_sqp_entries *sqp_entry = NULL; + int nw_type; + u16 vlan_id; + u8 sl; + + tbl_idx = cqe->wr_id; + + sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; + gsi_qp = sqp_entry->qp1_qp; + orig_cqe = &sqp_entry->cqe; + + wc->wr_id = sqp_entry->wrid; + wc->byte_len = orig_cqe->length; + wc->qp = &gsi_qp->ib_qp; + + wc->ex.imm_data = orig_cqe->immdata; + wc->src_qp = orig_cqe->src_qp; + memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); + if (bnxt_re_is_nonzero_vlanid_pkt(orig_cqe, &vlan_id, &sl)) { + if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { + wc->sl = sl; + wc->vlan_id = vlan_id; + wc->wc_flags |= IB_WC_WITH_VLAN; + } + } + wc->port_num = 1; + wc->vendor_err = orig_cqe->status; + + wc->opcode = IB_WC_RECV; + wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); + wc->wc_flags |= IB_WC_GRH; + + nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, + orig_cqe->raweth_qp1_flags2); + if(nw_type >= 0) + dev_dbg(rdev_to_dev(rdev), "%s nw_type = %d\n", __func__, nw_type); +} + +static void bnxt_re_process_res_ud_wc(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp, struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + u16 vlan_id = 0; + + wc->opcode = IB_WC_RECV; + wc->status = __rc_to_ib_wc_status(cqe->status); + if (cqe->flags & CQ_RES_UD_FLAGS_IMM) + wc->wc_flags |= IB_WC_WITH_IMM; + if (cqe->flags & CQ_RES_RC_FLAGS_INV) + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + /* report only on GSI QP for Thor */ + if (rdev->gsi_ctx.gsi_qp->qplib_qp.id == qp->qplib_qp.id && + rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) { + wc->wc_flags |= IB_WC_GRH; + memcpy(wc->smac, cqe->smac, ETH_ALEN); + wc->wc_flags |= IB_WC_WITH_SMAC; + if (_is_cqe_v2_supported(rdev->dev_attr->dev_cap_flags)) { + if (cqe->flags & CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK) { + if (cqe->cfa_meta & + BNXT_QPLIB_CQE_CFA_META1_VALID) + vlan_id = (cqe->cfa_meta & 0xFFF); + } + } else if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) { + vlan_id = (cqe->cfa_meta & 0xFFF); + } + /* Mark only if vlan_id is non zero */ + if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { + wc->vlan_id = vlan_id; + wc->wc_flags |= IB_WC_WITH_VLAN; + } + } +} + +static int bnxt_re_legacy_send_phantom_wqe(struct bnxt_re_qp *qp) +{ + struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; + unsigned long flags; + int rc = 0; + + spin_lock_irqsave(&qp->sq_lock, flags); + + rc = bnxt_re_legacy_bind_fence_mw(lib_qp); + if (!rc) { + lib_qp->sq.phantom_wqe_cnt++; + dev_dbg(&lib_qp->sq.hwq.pdev->dev, + "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", + lib_qp->id, lib_qp->sq.hwq.prod, + HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), + lib_qp->sq.phantom_wqe_cnt); + } + + spin_unlock_irqrestore(&qp->sq_lock, flags); + return rc; +} + +int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) +{ + struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); + struct bnxt_re_dev *rdev = cq->rdev; + struct bnxt_re_qp *qp; + struct bnxt_qplib_cqe *cqe; + int i, ncqe, budget, init_budget; + struct bnxt_qplib_q *sq; + struct bnxt_qplib_qp *lib_qp; + u32 tbl_idx; + struct bnxt_re_sqp_entries *sqp_entry = NULL; + unsigned long flags; + u8 gsi_mode; + + /* + * DB recovery CQ; only process the door bell pacing alert from + * the user lib + */ + if (cq->is_dbr_soft_cq) { + bnxt_re_pacing_alert(rdev); + return 0; + } + + /* User CQ; the only processing we do is to + * complete any pending CQ resize operation. + */ + if (cq->umem) { + if (cq->resize_umem) + bnxt_re_resize_cq_complete(cq); + return 0; + } + + spin_lock_irqsave(&cq->cq_lock, flags); + + budget = min_t(u32, num_entries, cq->max_cql); + init_budget = budget; + if (!cq->cql) { + dev_err(rdev_to_dev(rdev), "POLL CQ no CQL to use\n"); + goto exit; + } + cqe = &cq->cql[0]; + gsi_mode = rdev->gsi_ctx.gsi_qp_mode; + while (budget) { + lib_qp = NULL; + ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); + if (lib_qp) { + sq = &lib_qp->sq; + if (sq->legacy_send_phantom == true) { + qp = container_of(lib_qp, struct bnxt_re_qp, qplib_qp); + if (bnxt_re_legacy_send_phantom_wqe(qp) == -ENOMEM) + dev_err(rdev_to_dev(rdev), + "Phantom failed! Scheduled to send again\n"); + else + sq->legacy_send_phantom = false; + } + } + if (ncqe < budget) + ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, + cqe + ncqe, + budget - ncqe); + + if (!ncqe) + break; + + for (i = 0; i < ncqe; i++, cqe++) { + /* Transcribe each qplib_wqe back to ib_wc */ + memset(wc, 0, sizeof(*wc)); + + wc->wr_id = cqe->wr_id; + wc->byte_len = cqe->length; + qp = to_bnxt_re((struct bnxt_qplib_qp *)cqe->qp_handle, + struct bnxt_re_qp, qplib_qp); + if (!qp) { + dev_err(rdev_to_dev(rdev), + "POLL CQ bad QP handle\n"); + continue; + } + wc->qp = &qp->ib_qp; + wc->ex.imm_data = cqe->immdata; + wc->src_qp = cqe->src_qp; + memcpy(wc->smac, cqe->smac, ETH_ALEN); + wc->port_num = 1; + wc->vendor_err = cqe->status; + + switch(cqe->opcode) { + case CQ_BASE_CQE_TYPE_REQ: + if (gsi_mode == BNXT_RE_GSI_MODE_ALL && + qp->qplib_qp.id == + rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { + /* Handle this completion with + * the stored completion */ + dev_dbg(rdev_to_dev(rdev), + "Skipping this UD Send CQ\n"); + memset(wc, 0, sizeof(*wc)); + continue; + } + bnxt_re_process_req_wc(wc, cqe); + break; + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: + if (gsi_mode == BNXT_RE_GSI_MODE_ALL) { + if (!cqe->status) { + int rc = 0; + rc = bnxt_re_process_raw_qp_packet_receive(qp, cqe); + if (!rc) { + memset(wc, 0, + sizeof(*wc)); + continue; + } + cqe->status = -1; + } + /* Errors need not be looped back. + * But change the wr_id to the one + * stored in the table + */ + tbl_idx = cqe->wr_id; + sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; + wc->wr_id = sqp_entry->wrid; + } + + bnxt_re_process_res_rawqp1_wc(wc, cqe); + break; + case CQ_BASE_CQE_TYPE_RES_RC: + bnxt_re_process_res_rc_wc(wc, cqe); + break; + case CQ_BASE_CQE_TYPE_RES_UD: + if (gsi_mode == BNXT_RE_GSI_MODE_ALL && + qp->qplib_qp.id == + rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { + /* Handle this completion with + * the stored completion + */ + dev_dbg(rdev_to_dev(rdev), + "Handling the UD receive CQ\n"); + if (cqe->status) { + /* TODO handle this completion as a failure in + * loopback porocedure + */ + continue; + } else { + bnxt_re_process_res_shadow_qp_wc(qp, wc, cqe); + break; + } + } + bnxt_re_process_res_ud_wc(rdev, qp, wc, cqe); + break; + default: + dev_err(rdev_to_dev(cq->rdev), + "POLL CQ type 0x%x not handled, skip!\n", + cqe->opcode); + continue; + } + wc++; + budget--; + } + } +exit: + spin_unlock_irqrestore(&cq->cq_lock, flags); + return init_budget - budget; +} + +int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, + enum ib_cq_notify_flags ib_cqn_flags) +{ + struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); + int type = 0, rc = 0; + unsigned long flags; + + spin_lock_irqsave(&cq->cq_lock, flags); + /* Trigger on the very next completion */ + if (ib_cqn_flags & IB_CQ_NEXT_COMP) + type = DBC_DBC_TYPE_CQ_ARMALL; + /* Trigger on the next solicited completion */ + else if (ib_cqn_flags & IB_CQ_SOLICITED) + type = DBC_DBC_TYPE_CQ_ARMSE; + + bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); + + /* Poll to see if there are missed events */ + if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) && + !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) + rc = 1; + + spin_unlock_irqrestore(&cq->cq_lock, flags); + + return rc; +} + +/* Memory Regions */ +struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) +{ + struct bnxt_qplib_mrinfo mrinfo; + struct bnxt_re_dev *rdev; + struct bnxt_re_mr *mr; + struct bnxt_re_pd *pd; + u32 max_mr_count; + u64 pbl = 0; + int rc; + + memset(&mrinfo, 0, sizeof(mrinfo)); + pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + rdev = pd->rdev; + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + dev_err(rdev_to_dev(rdev), + "Allocate memory for DMA MR failed!\n"); + return ERR_PTR(-ENOMEM); + } + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; + + /* Allocate and register 0 as the address */ + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Allocate DMA MR failed!\n"); + goto fail; + } + mr->qplib_mr.total_size = -1; /* Infinite length */ + mrinfo.ptes = &pbl; + mrinfo.sg.npages = 0; + mrinfo.sg.pgsize = PAGE_SIZE; + mrinfo.sg.pgshft = PAGE_SHIFT; + mrinfo.sg.pgsize = PAGE_SIZE; + mrinfo.mrw = &mr->qplib_mr; + mrinfo.is_dma = true; + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); + if (rc) { + dev_err(rdev_to_dev(rdev), "Register DMA MR failed!\n"); + goto fail_mr; + } + mr->ib_mr.lkey = mr->qplib_mr.lkey; + if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | + IB_ACCESS_REMOTE_ATOMIC)) + mr->ib_mr.rkey = mr->ib_mr.lkey; + atomic_inc(&rdev->stats.rsors.mr_count); + max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); + if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) + atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); + + return &mr->ib_mr; + +fail_mr: + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); +fail: + kfree(mr); + return ERR_PTR(rc); +} + +int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) +{ + struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); + struct bnxt_re_dev *rdev = mr->rdev; + int rc = 0; + + rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) + dev_err(rdev_to_dev(rdev), "Dereg MR failed (%d): rc - %#x\n", + mr->qplib_mr.lkey, rc); + + if (mr->pages) { + bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, + &mr->qplib_frpl); + kfree(mr->pages); + mr->npages = 0; + mr->pages = NULL; + } + if (!IS_ERR(mr->ib_umem) && mr->ib_umem) { + mr->is_invalcb_active = false; + bnxt_re_peer_mem_release(mr->ib_umem); + } + kfree(mr); + atomic_dec(&rdev->stats.rsors.mr_count); + return 0; +} + +static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) +{ + struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); + + if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) + return -ENOMEM; + + mr->pages[mr->npages++] = addr; + dev_dbg(NULL, "%s: ibdev %p Set MR pages[%d] = 0x%lx\n", + ROCE_DRV_MODULE_NAME, ib_mr->device, mr->npages - 1, + mr->pages[mr->npages - 1]); + return 0; +} + +int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset) +{ + struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); + + mr->npages = 0; + return ib_sg_to_pages(ib_mr, sg, sg_nents, + sg_offset, bnxt_re_set_page); +} + +struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, + u32 max_num_sg, struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_mr *mr; + u32 max_mr_count; + int rc; + + dev_dbg(rdev_to_dev(rdev), "Alloc MR\n"); + if (type != IB_MR_TYPE_MEM_REG) { + dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported\n", type); + return ERR_PTR(-EINVAL); + } + if (max_num_sg > MAX_PBL_LVL_1_PGS) { + dev_dbg(rdev_to_dev(rdev), "Max SG exceeded\n"); + return ERR_PTR(-EINVAL); + } + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + dev_err(rdev_to_dev(rdev), "Allocate MR mem failed!\n"); + return ERR_PTR(-ENOMEM); + } + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; + + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n"); + goto fail; + } + mr->ib_mr.lkey = mr->qplib_mr.lkey; + mr->ib_mr.rkey = mr->ib_mr.lkey; + mr->pages = kzalloc(sizeof(u64) * max_num_sg, GFP_KERNEL); + if (!mr->pages) { + dev_err(rdev_to_dev(rdev), + "Allocate MR page list mem failed!\n"); + rc = -ENOMEM; + goto fail_mr; + } + rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, + &mr->qplib_frpl, max_num_sg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Allocate HW Fast reg page list failed!\n"); + goto free_page; + } + dev_dbg(rdev_to_dev(rdev), "Alloc MR pages = 0x%p\n", mr->pages); + + atomic_inc(&rdev->stats.rsors.mr_count); + max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); + if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) + atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); + return &mr->ib_mr; + +free_page: + kfree(mr->pages); +fail_mr: + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); +fail: + kfree(mr); + return ERR_PTR(rc); +} + +/* Memory Windows */ +struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_mw *mw; + u32 max_mw_count; + int rc; + + mw = kzalloc(sizeof(*mw), GFP_KERNEL); + if (!mw) { + dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n"); + rc = -ENOMEM; + goto exit; + } + mw->rdev = rdev; + mw->qplib_mw.pd = &pd->qplib_pd; + + mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? + CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 : + CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B); + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); + if (rc) { + dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n"); + goto fail; + } + mw->ib_mw.rkey = mw->qplib_mw.rkey; + atomic_inc(&rdev->stats.rsors.mw_count); + max_mw_count = atomic_read(&rdev->stats.rsors.mw_count); + if (max_mw_count > atomic_read(&rdev->stats.rsors.max_mw_count)) + atomic_set(&rdev->stats.rsors.max_mw_count, max_mw_count); + + return &mw->ib_mw; +fail: + kfree(mw); +exit: + return ERR_PTR(rc); +} + +int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) +{ + struct bnxt_re_mw *mw = to_bnxt_re(ib_mw, struct bnxt_re_mw, ib_mw); + struct bnxt_re_dev *rdev = mw->rdev; + int rc; + + rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); + if (rc) { + dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc); + return rc; + } + + kfree(mw); + atomic_dec(&rdev->stats.rsors.mw_count); + return rc; +} + +static int bnxt_re_page_size_ok(int page_shift) +{ + switch (page_shift) { + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G: + return 1; + default: + return 0; + } +} + +static int bnxt_re_get_page_shift(struct ib_umem *umem, + u64 va, u64 st, u64 cmask) +{ + int pgshft; + + pgshft = ilog2(umem->page_size); + + return pgshft; +} + +static int bnxt_re_get_num_pages(struct ib_umem *umem, u64 start, u64 length, int page_shift) +{ + int npages = 0; + + if (page_shift == PAGE_SHIFT) { + npages = ib_umem_num_pages_compat(umem); + } else { + npages = ALIGN(length, BIT(page_shift)) / BIT(page_shift); + if (start % BIT(page_shift)) + npages++; + } + return npages; +} + +/* uverbs */ +struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_qplib_mrinfo mrinfo; + int umem_pgs, page_shift, rc; + struct bnxt_re_mr *mr; + struct ib_umem *umem; + u32 max_mr_count; + int npages; + + dev_dbg(rdev_to_dev(rdev), "Reg user MR\n"); + + if (bnxt_re_get_total_mr_mw_count(rdev) >= rdev->dev_attr->max_mr) + return ERR_PTR(-ENOMEM); + + if (rdev->mod_exit) { + dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); + return ERR_PTR(-EIO); + } + memset(&mrinfo, 0, sizeof(mrinfo)); + if (length > BNXT_RE_MAX_MR_SIZE) { + dev_err(rdev_to_dev(rdev), "Requested MR Size: %lu " + "> Max supported: %ld\n", length, BNXT_RE_MAX_MR_SIZE); + return ERR_PTR(-ENOMEM); + } + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n"); + return ERR_PTR (-ENOMEM); + } + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; + + if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Alloc MR failed!\n"); + goto fail; + } + /* The fixed portion of the rkey is the same as the lkey */ + mr->ib_mr.rkey = mr->qplib_mr.rkey; + } + + umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, + udata, start, length, + mr_access_flags, 0); + if (IS_ERR(umem)) { + rc = PTR_ERR(umem); + dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n", + __func__, rc); + goto free_mr; + } + mr->ib_umem = umem; + + mr->qplib_mr.va = virt_addr; + umem_pgs = ib_umem_num_pages_compat(umem); + if (!umem_pgs) { + dev_err(rdev_to_dev(rdev), "umem is invalid!\n"); + rc = -EINVAL; + goto free_umem; + } + mr->qplib_mr.total_size = length; + page_shift = bnxt_re_get_page_shift(umem, virt_addr, start, + rdev->dev_attr->page_size_cap); + if (!bnxt_re_page_size_ok(page_shift)) { + dev_err(rdev_to_dev(rdev), "umem page size unsupported!\n"); + rc = -EFAULT; + goto free_umem; + } + npages = bnxt_re_get_num_pages(umem, start, length, page_shift); + + /* Map umem buf ptrs to the PBL */ + mrinfo.sg.npages = npages; + mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap); + mrinfo.sg.pgshft = page_shift; + mrinfo.sg.pgsize = BIT(page_shift); + + mrinfo.mrw = &mr->qplib_mr; + + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); + if (rc) { + dev_err(rdev_to_dev(rdev), "Reg user MR failed!\n"); + goto free_umem; + } + + mr->ib_mr.lkey = mr->ib_mr.rkey = mr->qplib_mr.lkey; + atomic_inc(&rdev->stats.rsors.mr_count); + max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); + if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) + atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); + + return &mr->ib_mr; + +free_umem: + bnxt_re_peer_mem_release(mr->ib_umem); +free_mr: + if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); +fail: + kfree(mr); + return ERR_PTR(rc); +} + +int +bnxt_re_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, + struct ib_pd *ib_pd, struct ib_udata *udata) +{ + struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + int umem_pgs = 0, page_shift = PAGE_SHIFT, rc; + struct bnxt_re_dev *rdev = mr->rdev; + struct bnxt_qplib_mrinfo mrinfo; + struct ib_umem *umem; + u32 npages; + + /* TODO: Must decipher what to modify based on the flags */ + memset(&mrinfo, 0, sizeof(mrinfo)); + if (flags & IB_MR_REREG_TRANS) { + umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, + udata, start, length, + mr_access_flags, 0); + if (IS_ERR(umem)) { + rc = PTR_ERR(umem); + dev_err(rdev_to_dev(rdev), + "%s: ib_umem_get failed! ret = %d\n", + __func__, rc); + goto fail; + } + mr->ib_umem = umem; + + mr->qplib_mr.va = virt_addr; + umem_pgs = ib_umem_num_pages_compat(umem); + if (!umem_pgs) { + dev_err(rdev_to_dev(rdev), "umem is invalid!\n"); + rc = -EINVAL; + goto fail_free_umem; + } + mr->qplib_mr.total_size = length; + page_shift = bnxt_re_get_page_shift(umem, virt_addr, start, + rdev->dev_attr->page_size_cap); + if (!bnxt_re_page_size_ok(page_shift)) { + dev_err(rdev_to_dev(rdev), + "umem page size unsupported!\n"); + rc = -EFAULT; + goto fail_free_umem; + } + npages = bnxt_re_get_num_pages(umem, start, length, page_shift); + /* Map umem buf ptrs to the PBL */ + mrinfo.sg.npages = npages; + mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap); + mrinfo.sg.pgshft = page_shift; + mrinfo.sg.pgsize = BIT(page_shift); + } + + mrinfo.mrw = &mr->qplib_mr; + if (flags & IB_MR_REREG_PD) + mr->qplib_mr.pd = &pd->qplib_pd; + + if (flags & IB_MR_REREG_ACCESS) + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); + + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); + if (rc) { + dev_err(rdev_to_dev(rdev), "Rereg user MR failed!\n"); + goto fail_free_umem; + } + mr->ib_mr.rkey = mr->qplib_mr.rkey; + + return 0; + +fail_free_umem: + bnxt_re_peer_mem_release(mr->ib_umem); +fail: + return rc; +} + +static int bnxt_re_check_abi_version(struct bnxt_re_dev *rdev) +{ + struct ib_device *ibdev = &rdev->ibdev; + u32 uverbs_abi_ver; + + uverbs_abi_ver = GET_UVERBS_ABI_VERSION(ibdev); + dev_dbg(rdev_to_dev(rdev), "ABI version requested %d\n", + uverbs_abi_ver); + if (uverbs_abi_ver != BNXT_RE_ABI_VERSION) { + dev_dbg(rdev_to_dev(rdev), " is different from the device %d \n", + BNXT_RE_ABI_VERSION); + return -EPERM; + } + return 0; +} + +int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in, + struct ib_udata *udata) +{ + struct ib_ucontext *ctx = uctx_in; + struct ib_device *ibdev = ctx->device; + struct bnxt_re_ucontext *uctx = + container_of(ctx, struct bnxt_re_ucontext, ibucontext); + + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; + struct bnxt_re_uctx_resp resp = {}; + struct bnxt_re_uctx_req ureq = {}; + struct bnxt_qplib_chip_ctx *cctx; + u32 chip_met_rev_num; + bool genp5 = false; + int rc; + + cctx = rdev->chip_ctx; + rc = bnxt_re_check_abi_version(rdev); + if (rc) + goto fail; + + uctx->rdev = rdev; + uctx->shpg = (void *)__get_free_page(GFP_KERNEL); + if (!uctx->shpg) { + dev_err(rdev_to_dev(rdev), "shared memory allocation failed!\n"); + rc = -ENOMEM; + goto fail; + } + spin_lock_init(&uctx->sh_lock); + if (BNXT_RE_ABI_VERSION >= 4) { + chip_met_rev_num = cctx->chip_num; + chip_met_rev_num |= ((u32)cctx->chip_rev & 0xFF) << + BNXT_RE_CHIP_ID0_CHIP_REV_SFT; + chip_met_rev_num |= ((u32)cctx->chip_metal & 0xFF) << + BNXT_RE_CHIP_ID0_CHIP_MET_SFT; + resp.chip_id0 = chip_met_rev_num; + resp.chip_id1 = 0; /* future extension of chip info */ + } + + if (BNXT_RE_ABI_VERSION != 4) { + /*Temp, Use idr_alloc instead*/ + resp.dev_id = rdev->en_dev->pdev->devfn; + resp.max_qp = rdev->qplib_res.hctx->qp_ctx.max; + } + + genp5 = _is_chip_gen_p5_p7(cctx); + if (BNXT_RE_ABI_VERSION > 5) { + resp.modes = genp5 ? cctx->modes.wqe_mode : 0; + if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED; + } + + resp.pg_size = PAGE_SIZE; + resp.cqe_sz = sizeof(struct cq_base); + resp.max_cqd = dev_attr->max_cq_wqes; + if (genp5 && cctx->modes.db_push) { + resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED; + if (_is_chip_p7(cctx) && + !(dev_attr->dev_cap_flags & + CREQ_QUERY_FUNC_RESP_SB_PINGPONG_PUSH_MODE)) + resp.comp_mask &= + ~BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED; + } + + resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED; + + if (rdev->dbr_pacing) + resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED; + + if (rdev->dbr_drop_recov && rdev->user_dbr_drop_recov) + resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED; + + if (udata->inlen >= sizeof(ureq)) { + rc = ib_copy_from_udata(&ureq, udata, + min(udata->inlen, sizeof(ureq))); + if (rc) + goto cfail; + if (bnxt_re_init_pow2_flag(&ureq, &resp)) + dev_warn(rdev_to_dev(rdev), + "Enabled roundup logic. Library bug?\n"); + if (bnxt_re_init_rsvd_wqe_flag(&ureq, &resp, genp5)) + dev_warn(rdev_to_dev(rdev), + "Rsvd wqe in use! Try the updated library.\n"); + } else { + dev_warn(rdev_to_dev(rdev), + "Enabled roundup logic. Update the library!\n"); + resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED; + + dev_warn(rdev_to_dev(rdev), + "Rsvd wqe in use. Update the library!\n"); + resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; + } + + uctx->cmask = (uint64_t)resp.comp_mask; + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) + goto cfail; + + INIT_LIST_HEAD(&uctx->cq_list); + mutex_init(&uctx->cq_lock); + + return 0; +cfail: + free_page((u64)uctx->shpg); + uctx->shpg = NULL; +fail: + return rc; +} + +void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) +{ + struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx, + struct bnxt_re_ucontext, + ibucontext); + struct bnxt_re_dev *rdev = uctx->rdev; + int rc = 0; + + if (uctx->shpg) + free_page((u64)uctx->shpg); + + if (uctx->dpi.dbr) { + /* Free DPI only if this is the first PD allocated by the + * application and mark the context dpi as NULL + */ + if (_is_chip_gen_p5_p7(rdev->chip_ctx) && uctx->wcdpi.dbr) { + rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, + &uctx->wcdpi); + if (rc) + dev_err(rdev_to_dev(rdev), + "dealloc push dp failed\n"); + uctx->wcdpi.dbr = NULL; + } + + rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, + &uctx->dpi); + if (rc) + dev_err(rdev_to_dev(rdev), "Deallocte HW DPI failed!\n"); + /* Don't fail, continue*/ + uctx->dpi.dbr = NULL; + } + return; +} + +static struct bnxt_re_cq *is_bnxt_re_cq_page(struct bnxt_re_ucontext *uctx, + u64 pg_off) +{ + struct bnxt_re_cq *cq = NULL, *tmp_cq; + + if (!_is_chip_p7(uctx->rdev->chip_ctx)) + return NULL; + + mutex_lock(&uctx->cq_lock); + list_for_each_entry(tmp_cq, &uctx->cq_list, cq_list) { + if (((u64)tmp_cq->uctx_cq_page >> PAGE_SHIFT) == pg_off) { + cq = tmp_cq; + break; + } + } + mutex_unlock(&uctx->cq_lock); + return cq; +} + +/* Helper function to mmap the virtual memory from user app */ +int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) +{ + struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx, + struct bnxt_re_ucontext, + ibucontext); + struct bnxt_re_dev *rdev = uctx->rdev; + struct bnxt_re_cq *cq = NULL; + int rc = 0; + u64 pfn; + + switch (vma->vm_pgoff) { + case BNXT_RE_MAP_SH_PAGE: + pfn = vtophys(uctx->shpg) >> PAGE_SHIFT; + return rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); + dev_dbg(rdev_to_dev(rdev), "%s:%d uctx->shpg 0x%lx, vtophys(uctx->shpg) 0x%lx, pfn = 0x%lx \n", + __func__, __LINE__, (u64) uctx->shpg, vtophys(uctx->shpg), pfn); + if (rc) { + dev_err(rdev_to_dev(rdev), "Shared page mapping failed!\n"); + rc = -EAGAIN; + } + return rc; + case BNXT_RE_MAP_WC: + vma->vm_page_prot = + pgprot_writecombine(vma->vm_page_prot); + pfn = (uctx->wcdpi.umdbr >> PAGE_SHIFT); + if (!pfn) + return -EFAULT; + break; + case BNXT_RE_DBR_PAGE: + /* Driver doesn't expect write access request */ + if (vma->vm_flags & VM_WRITE) + return -EFAULT; + + pfn = vtophys(rdev->dbr_page) >> PAGE_SHIFT; + if (!pfn) + return -EFAULT; + break; + case BNXT_RE_MAP_DB_RECOVERY_PAGE: + pfn = vtophys(uctx->dbr_recov_cq_page) >> PAGE_SHIFT; + if (!pfn) + return -EFAULT; + break; + default: + cq = is_bnxt_re_cq_page(uctx, vma->vm_pgoff); + if (cq) { + pfn = vtophys((void *)cq->uctx_cq_page) >> PAGE_SHIFT; + rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); + if (rc) { + dev_err(rdev_to_dev(rdev), + "CQ page mapping failed!\n"); + rc = -EAGAIN; + } + goto out; + } else { + vma->vm_page_prot = + pgprot_noncached(vma->vm_page_prot); + pfn = vma->vm_pgoff; + } + break; + } + + rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); + if (rc) { + dev_err(rdev_to_dev(rdev), "DPI mapping failed!\n"); + return -EAGAIN; + } + rc = __bnxt_re_set_vma_data(uctx, vma); +out: + return rc; +} + +int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, + const struct ib_wc *wc, const struct ib_grh *grh, + const struct ib_mad_hdr *in_mad, size_t in_mad_size, + struct ib_mad_hdr *out_mad, size_t *out_mad_size, + u16 *out_mad_pkey_index) +{ + return IB_MAD_RESULT_SUCCESS; +} + +void bnxt_re_disassociate_ucntx(struct ib_ucontext *ib_uctx) +{ +} diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.h b/sys/dev/bnxt/bnxt_re/ib_verbs.h new file mode 100644 index 00000000000..cb9f7974e92 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/ib_verbs.h @@ -0,0 +1,632 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: IB Verbs interpreter (header) + */ + +#ifndef __BNXT_RE_IB_VERBS_H__ +#define __BNXT_RE_IB_VERBS_H__ + +#include +#include "bnxt_re-abi.h" +#include "qplib_res.h" +#include "qplib_fp.h" + +struct bnxt_re_dev; + +#define BNXT_RE_ROCE_V2_UDP_SPORT 0x8CD1 +#define BNXT_RE_QP_RANDOM_QKEY 0x81818181 + +#ifndef IB_MTU_8192 +#define IB_MTU_8192 8192 +#endif + +#ifndef SPEED_1000 +#define SPEED_1000 1000 +#endif + +#ifndef SPEED_10000 +#define SPEED_10000 10000 +#endif + +#ifndef SPEED_20000 +#define SPEED_20000 20000 +#endif + +#ifndef SPEED_25000 +#define SPEED_25000 25000 +#endif + +#ifndef SPEED_40000 +#define SPEED_40000 40000 +#endif + +#ifndef SPEED_50000 +#define SPEED_50000 50000 +#endif + +#ifndef SPEED_100000 +#define SPEED_100000 100000 +#endif + +#ifndef SPEED_200000 +#define SPEED_200000 200000 +#endif + +#ifndef IB_SPEED_HDR +#define IB_SPEED_HDR 64 +#endif + +#define RDMA_NETWORK_IPV4 1 +#define RDMA_NETWORK_IPV6 2 + +#define ROCE_DMAC(x) (x)->dmac + +#define dma_rmb() rmb() + +#define compat_ib_alloc_device(size) ib_alloc_device(size); + +#define rdev_from_cq_in(cq_in) to_bnxt_re_dev(cq_in->device, ibdev) + +#define GET_UVERBS_ABI_VERSION(ibdev) (ibdev->uverbs_abi_ver) + +#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB 0x1cUL + +#define IB_POLL_UNBOUND_WORKQUEUE IB_POLL_WORKQUEUE + +#define BNXT_RE_LEGACY_FENCE_BYTES 64 +#define BNXT_RE_LEGACY_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_LEGACY_FENCE_BYTES, PAGE_SIZE) + + +static inline struct +bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in, + struct bnxt_re_dev *rdev); +static inline struct +bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in, + struct bnxt_re_dev *rdev); + +static inline bool +bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, u16 vlan_id); + +#define bnxt_re_compat_qfwstr(void) \ + bnxt_re_query_fw_str(struct ib_device *ibdev, \ + char *str, size_t str_len) + +static inline +struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap); + +struct bnxt_re_gid_ctx { + u32 idx; + u32 refcnt; +}; + +struct bnxt_re_legacy_fence_data { + u32 size; + void *va; + dma_addr_t dma_addr; + struct bnxt_re_mr *mr; + struct ib_mw *mw; + struct bnxt_qplib_swqe bind_wqe; + u32 bind_rkey; +}; + +struct bnxt_re_pd { + struct ib_pd ibpd; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_pd qplib_pd; + struct bnxt_re_legacy_fence_data fence; +}; + +struct bnxt_re_ah { + struct ib_ah ibah; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_ah qplib_ah; +}; + +struct bnxt_re_srq { + struct ib_srq ibsrq; + struct bnxt_re_dev *rdev; + u32 srq_limit; + struct bnxt_qplib_srq qplib_srq; + struct ib_umem *umem; + spinlock_t lock; +}; + +union ip_addr { + u32 ipv4_addr; + u8 ipv6_addr[16]; +}; + +struct bnxt_re_qp_info_entry { + union ib_gid sgid; + union ib_gid dgid; + union ip_addr s_ip; + union ip_addr d_ip; + u16 s_port; +#define BNXT_RE_QP_DEST_PORT 4791 + u16 d_port; +}; + +struct bnxt_re_qp { + struct ib_qp ib_qp; + struct list_head list; + struct bnxt_re_dev *rdev; + spinlock_t sq_lock; + spinlock_t rq_lock; + struct bnxt_qplib_qp qplib_qp; + struct ib_umem *sumem; + struct ib_umem *rumem; + /* QP1 */ + u32 send_psn; + struct ib_ud_header qp1_hdr; + struct bnxt_re_cq *scq; + struct bnxt_re_cq *rcq; + struct dentry *qp_info_pdev_dentry; + struct bnxt_re_qp_info_entry qp_info_entry; + void *qp_data; +}; + +struct bnxt_re_cq { + struct ib_cq ibcq; + struct list_head cq_list; + struct bnxt_re_dev *rdev; + struct bnxt_re_ucontext *uctx; + spinlock_t cq_lock; + u16 cq_count; + u16 cq_period; + struct bnxt_qplib_cq qplib_cq; + struct bnxt_qplib_cqe *cql; +#define MAX_CQL_PER_POLL 1024 + u32 max_cql; + struct ib_umem *umem; + struct ib_umem *resize_umem; + struct ib_ucontext *context; + int resize_cqe; + /* list of cq per uctx. Used only for Thor-2 */ + void *uctx_cq_page; + void *dbr_recov_cq_page; + bool is_dbr_soft_cq; +}; + +struct bnxt_re_mr { + struct bnxt_re_dev *rdev; + struct ib_mr ib_mr; + struct ib_umem *ib_umem; + struct bnxt_qplib_mrw qplib_mr; + u32 npages; + u64 *pages; + struct bnxt_qplib_frpl qplib_frpl; + bool is_invalcb_active; +}; + +struct bnxt_re_frpl { + struct bnxt_re_dev *rdev; + struct bnxt_qplib_frpl qplib_frpl; + u64 *page_list; +}; + +struct bnxt_re_mw { + struct bnxt_re_dev *rdev; + struct ib_mw ib_mw; + struct bnxt_qplib_mrw qplib_mw; +}; + +struct bnxt_re_ucontext { + struct ib_ucontext ibucontext; + struct bnxt_re_dev *rdev; + struct list_head cq_list; + struct bnxt_qplib_dpi dpi; + struct bnxt_qplib_dpi wcdpi; + void *shpg; + spinlock_t sh_lock; + uint64_t cmask; + struct mutex cq_lock; /* Protect cq list */ + void *dbr_recov_cq_page; + struct bnxt_re_cq *dbr_recov_cq; +}; + +struct bnxt_re_ah_info { + union ib_gid sgid; + struct ib_gid_attr sgid_attr; + u16 vlan_tag; + u8 nw_type; +}; + +struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev, + u8 port_num); + +int bnxt_re_query_device(struct ib_device *ibdev, + struct ib_device_attr *ib_attr, + struct ib_udata *udata); +int bnxt_re_modify_device(struct ib_device *ibdev, + int device_modify_mask, + struct ib_device_modify *device_modify); +int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, + struct ib_port_attr *port_attr); +int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, + int port_modify_mask, + struct ib_port_modify *port_modify); +int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable); +void bnxt_re_compat_qfwstr(void); +int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, + u16 index, u16 *pkey); +int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, + unsigned int index, void **context); +int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, + unsigned int index, const union ib_gid *gid, + const struct ib_gid_attr *attr, void **context); +int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, + int index, union ib_gid *gid); +enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, + u8 port_num); +int bnxt_re_alloc_pd(struct ib_pd *pd_in, struct ib_udata *udata); +void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata); + +int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr, + u32 flags, struct ib_udata *udata); + +int bnxt_re_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); +int bnxt_re_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); + +void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags); +int bnxt_re_create_srq(struct ib_srq *srq_in, + struct ib_srq_init_attr *srq_init_attr, + struct ib_udata *udata); +int bnxt_re_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr, + enum ib_srq_attr_mask srq_attr_mask, + struct ib_udata *udata); +int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); +void bnxt_re_destroy_srq(struct ib_srq *ib_srq, + struct ib_udata *udata); +int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); +struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in, + struct ib_qp_init_attr *qp_init_attr, + struct ib_udata *udata); +int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_udata *udata); +int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); +int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata); +int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr); +int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); +int bnxt_re_create_cq(struct ib_cq *cq_in, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata); +void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); +int bnxt_re_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); +int bnxt_re_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata); +int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc); +int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); +struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags); +int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset); +struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, + u32 max_num_sg, struct ib_udata *udata); +int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); +struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, + struct ib_udata *udata); +int bnxt_re_dealloc_mw(struct ib_mw *mw); +struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, + struct ib_udata *udata); +int +bnxt_re_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, struct ib_pd *pd, + struct ib_udata *udata); +int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in, + struct ib_udata *udata); +void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx); +int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma); +int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, + const struct ib_wc *wc, const struct ib_grh *grh, + const struct ib_mad_hdr *in_mad, size_t in_mad_size, + struct ib_mad_hdr *out_mad, size_t *out_mad_size, + u16 *out_mad_pkey_index); +unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp); +void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags); +void bnxt_re_disassociate_ucntx(struct ib_ucontext *ibcontext); +static inline int __bnxt_re_set_vma_data(void *bnxt_re_uctx, + struct vm_area_struct *vma); +void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev); +void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq); +static inline int +bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index, + union ib_gid *sgid, struct ib_gid_attr **sgid_attr, + struct ib_global_route *grh, struct ib_ah *ah); +static inline enum rdma_network_type +bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr, + union ib_gid *sgid); +static inline +struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev, + struct ib_ucontext *ucontext, + struct ib_udata *udata, + unsigned long addr, + size_t size, int access, int dmasync); +static inline +struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev, + struct ib_ucontext *ucontext, + struct ib_udata *udata, + unsigned long addr, + size_t size, int access, int dmasync); +static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem); +static inline void bnxt_re_peer_mem_release(struct ib_umem *umem); +void bnxt_re_resolve_dmac_task(struct work_struct *work); + +static inline enum ib_qp_type __from_hw_to_ib_qp_type(u8 type) +{ + switch (type) { + case CMDQ_CREATE_QP1_TYPE_GSI: + case CMDQ_CREATE_QP_TYPE_GSI: + return IB_QPT_GSI; + case CMDQ_CREATE_QP_TYPE_RC: + return IB_QPT_RC; + case CMDQ_CREATE_QP_TYPE_UD: + return IB_QPT_UD; + case CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE: + return IB_QPT_RAW_ETHERTYPE; + default: + return IB_QPT_MAX; + } +} + +static inline u8 __from_ib_qp_state(enum ib_qp_state state) +{ + switch (state) { + case IB_QPS_RESET: + return CMDQ_MODIFY_QP_NEW_STATE_RESET; + case IB_QPS_INIT: + return CMDQ_MODIFY_QP_NEW_STATE_INIT; + case IB_QPS_RTR: + return CMDQ_MODIFY_QP_NEW_STATE_RTR; + case IB_QPS_RTS: + return CMDQ_MODIFY_QP_NEW_STATE_RTS; + case IB_QPS_SQD: + return CMDQ_MODIFY_QP_NEW_STATE_SQD; + case IB_QPS_SQE: + return CMDQ_MODIFY_QP_NEW_STATE_SQE; + case IB_QPS_ERR: + default: + return CMDQ_MODIFY_QP_NEW_STATE_ERR; + } +} + +static inline u32 __from_ib_mtu(enum ib_mtu mtu) +{ + switch (mtu) { + case IB_MTU_256: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; + case IB_MTU_512: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; + case IB_MTU_1024: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; + case IB_MTU_2048: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; + case IB_MTU_4096: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; + default: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; + } +} + +static inline enum ib_mtu __to_ib_mtu(u32 mtu) +{ + switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { + case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: + return IB_MTU_256; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: + return IB_MTU_512; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: + return IB_MTU_1024; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: + return IB_MTU_2048; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: + return IB_MTU_4096; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_8192: + return IB_MTU_8192; + default: + return IB_MTU_2048; + } +} + +static inline enum ib_qp_state __to_ib_qp_state(u8 state) +{ + switch (state) { + case CMDQ_MODIFY_QP_NEW_STATE_RESET: + return IB_QPS_RESET; + case CMDQ_MODIFY_QP_NEW_STATE_INIT: + return IB_QPS_INIT; + case CMDQ_MODIFY_QP_NEW_STATE_RTR: + return IB_QPS_RTR; + case CMDQ_MODIFY_QP_NEW_STATE_RTS: + return IB_QPS_RTS; + case CMDQ_MODIFY_QP_NEW_STATE_SQD: + return IB_QPS_SQD; + case CMDQ_MODIFY_QP_NEW_STATE_SQE: + return IB_QPS_SQE; + case CMDQ_MODIFY_QP_NEW_STATE_ERR: + default: + return IB_QPS_ERR; + } +} + +static inline int bnxt_re_init_pow2_flag(struct bnxt_re_uctx_req *req, + struct bnxt_re_uctx_resp *resp) +{ + resp->comp_mask |= BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED; + if (!(req->comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT)) { + resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED; + return -EINVAL; + } + return 0; +} + +static inline u32 bnxt_re_init_depth(u32 ent, struct bnxt_re_ucontext *uctx) +{ + return uctx ? (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED) ? + ent : roundup_pow_of_two(ent) : ent; +} + +static inline int bnxt_re_init_rsvd_wqe_flag(struct bnxt_re_uctx_req *req, + struct bnxt_re_uctx_resp *resp, + bool genp5) +{ + resp->comp_mask |= BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; + if (!(req->comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE)) { + resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; + return -EINVAL; + } else if (!genp5) { + resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; + } + return 0; +} + +static inline u32 bnxt_re_get_diff(struct bnxt_re_ucontext *uctx, + struct bnxt_qplib_chip_ctx *cctx) +{ + if (!uctx) { + /* return res-wqe only for gen p4 for user resource */ + return _is_chip_gen_p5_p7(cctx) ? 0 : BNXT_QPLIB_RESERVED_QP_WRS; + } else if (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED) { + return 0; + } + /* old lib */ + return BNXT_QPLIB_RESERVED_QP_WRS; +} + +static inline void bnxt_re_init_qpmtu(struct bnxt_re_qp *qp, int mtu, + int mask, struct ib_qp_attr *qp_attr, + bool *is_qpmtu_high) +{ + int qpmtu, qpmtu_int; + int ifmtu, ifmtu_int; + + ifmtu = iboe_get_mtu(mtu); + ifmtu_int = ib_mtu_enum_to_int(ifmtu); + qpmtu = ifmtu; + qpmtu_int = ifmtu_int; + if (mask & IB_QP_PATH_MTU) { + qpmtu = qp_attr->path_mtu; + qpmtu_int = ib_mtu_enum_to_int(qpmtu); + if (qpmtu_int > ifmtu_int) { + /* Trim the QP path mtu to interface mtu and update + * the new mtu to user qp for retransmission psn + * calculations. + */ + qpmtu = ifmtu; + qpmtu_int = ifmtu_int; + *is_qpmtu_high = true; + } + } + qp->qplib_qp.path_mtu = __from_ib_mtu(qpmtu); + qp->qplib_qp.mtu = qpmtu_int; + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; +} + +inline unsigned long compare_ether_header(void *a, void *b) +{ + u32 *a32 = (u32 *)((u8 *)a + 2); + u32 *b32 = (u32 *)((u8 *)b + 2); + + return (*(u16 *)a ^ *(u16 *)b) | (a32[0] ^ b32[0]) | + (a32[1] ^ b32[1]) | (a32[2] ^ b32[2]); +} + +struct vlan_hdr { + __be16 h_vlan_TCI; + __be16 h_vlan_encapsulated_proto; +}; + +inline uint16_t +crc16(uint16_t crc, const void *buffer, unsigned int len) +{ + const unsigned char *cp = buffer; + /* CRC table for the CRC-16. The poly is 0x8005 (x16 + x15 + x2 + 1). */ + static uint16_t const crc16_table[256] = { + 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241, + 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440, + 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40, + 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841, + 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40, + 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41, + 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641, + 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040, + 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240, + 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441, + 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41, + 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840, + 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41, + 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40, + 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640, + 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041, + 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240, + 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441, + 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41, + 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840, + 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41, + 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40, + 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640, + 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041, + 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241, + 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440, + 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40, + 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841, + 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40, + 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41, + 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641, + 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040 + }; + + while (len--) + crc = (((crc >> 8) & 0xffU) ^ + crc16_table[(crc ^ *cp++) & 0xffU]) & 0x0000ffffU; + return crc; +} + +static inline int __bnxt_re_set_vma_data(void *bnxt_re_uctx, + struct vm_area_struct *vma) +{ + return 0; +} + +static inline bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, + u16 vlan_id) +{ + bool ret = true; + /* + * Check if the vlan is configured in the host. + * If not configured, it can be a transparent + * VLAN. So dont report the vlan id. + */ + return ret; +} + +#endif diff --git a/sys/dev/bnxt/bnxt_re/main.c b/sys/dev/bnxt/bnxt_re/main.c new file mode 100644 index 00000000000..e6c6f754ea4 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/main.c @@ -0,0 +1,4467 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Main component of the bnxt_re driver + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bnxt_re.h" +#include "ib_verbs.h" +#include "bnxt_re-abi.h" +#include "bnxt.h" + +static char drv_version[] = + "Broadcom NetXtreme-C/E RoCE Driver " ROCE_DRV_MODULE_NAME \ + " v" ROCE_DRV_MODULE_VERSION " (" ROCE_DRV_MODULE_RELDATE ")\n"; + +#define BNXT_RE_DESC "Broadcom NetXtreme RoCE" +#define BNXT_ADEV_NAME "if_bnxt" + +MODULE_DESCRIPTION("Broadcom NetXtreme-C/E RoCE Driver"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DEPEND(bnxt_re, linuxkpi, 1, 1, 1); +MODULE_DEPEND(bnxt_re, ibcore, 1, 1, 1); +MODULE_DEPEND(bnxt_re, if_bnxt, 1, 1, 1); +MODULE_VERSION(bnxt_re, 1); + + +DEFINE_MUTEX(bnxt_re_mutex); /* mutex lock for driver */ + +static unsigned int restrict_mrs = 0; +module_param(restrict_mrs, uint, 0); +MODULE_PARM_DESC(restrict_mrs, " Restrict the no. of MRs 0 = 256K , 1 = 64K"); + +unsigned int restrict_stats = 0; +module_param(restrict_stats, uint, 0); +MODULE_PARM_DESC(restrict_stats, "Restrict stats query frequency to ethtool coalesce value. Disabled by default"); + +unsigned int enable_fc = 1; +module_param(enable_fc, uint, 0); +MODULE_PARM_DESC(enable_fc, "Enable default PFC, CC,ETS during driver load. 1 - fc enable, 0 - fc disable - Default is 1"); + +unsigned int min_tx_depth = 1; +module_param(min_tx_depth, uint, 0); +MODULE_PARM_DESC(min_tx_depth, "Minimum TX depth - Default is 1"); + +static uint8_t max_msix_vec[BNXT_RE_MAX_DEVICES] = {0}; +static unsigned int max_msix_vec_argc; +module_param_array(max_msix_vec, byte, &max_msix_vec_argc, 0444); +MODULE_PARM_DESC(max_msix_vec, "Max MSI-x vectors per PF (2 - 64) - Default is 64"); + +unsigned int cmdq_shadow_qd = RCFW_CMD_NON_BLOCKING_SHADOW_QD; +module_param_named(cmdq_shadow_qd, cmdq_shadow_qd, uint, 0644); +MODULE_PARM_DESC(cmdq_shadow_qd, "Perf Stat Debug: Shadow QD Range (1-64) - Default is 64"); + + +/* globals */ +struct list_head bnxt_re_dev_list = LINUX_LIST_HEAD_INIT(bnxt_re_dev_list); +static int bnxt_re_probe_count; + +DEFINE_MUTEX(bnxt_re_dev_lock); +static u32 gmod_exit; +static u32 gadd_dev_inprogress; + +static void bnxt_re_task(struct work_struct *work_task); +static struct workqueue_struct *bnxt_re_wq; +static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev); +static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, + u32 *offset); +static int bnxt_re_ib_init(struct bnxt_re_dev *rdev); +static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev); +void _bnxt_re_remove(struct auxiliary_device *adev); +void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32); +u32 readl_fbsd(struct bnxt_softc *bp, u32, u8); +static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev); + +int bnxt_re_register_netdevice_notifier(struct notifier_block *nb) +{ + int rc; + rc = register_netdevice_notifier(nb); + return rc; +} + +int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb) +{ + int rc; + rc = unregister_netdevice_notifier(nb); + return rc; +} + +void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev) +{ + ibdev->dma_device = &rdev->en_dev->pdev->dev; +} + +void bnxt_re_init_resolve_wq(struct bnxt_re_dev *rdev) +{ + rdev->resolve_wq = create_singlethread_workqueue("bnxt_re_resolve_wq"); + INIT_LIST_HEAD(&rdev->mac_wq_list); +} + +void bnxt_re_uninit_resolve_wq(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_resolve_dmac_work *tmp_work = NULL, *tmp_st; + if (!rdev->resolve_wq) + return; + flush_workqueue(rdev->resolve_wq); + list_for_each_entry_safe(tmp_work, tmp_st, &rdev->mac_wq_list, list) { + list_del(&tmp_work->list); + kfree(tmp_work); + } + destroy_workqueue(rdev->resolve_wq); + rdev->resolve_wq = NULL; +} + +u32 readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx) +{ + + if (bar_idx) + return bus_space_read_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off); + else + return bus_space_read_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off); +} + +void writel_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx, u32 val) +{ + if (bar_idx) + bus_space_write_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off, htole32(val)); + else + bus_space_write_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off, htole32(val)); +} + +static void bnxt_re_update_fifo_occup_slabs(struct bnxt_re_dev *rdev, + u32 fifo_occup) +{ + if (fifo_occup > rdev->dbg_stats->dbq.fifo_occup_water_mark) + rdev->dbg_stats->dbq.fifo_occup_water_mark = fifo_occup; + + if (fifo_occup > 8 * rdev->pacing_algo_th) + rdev->dbg_stats->dbq.fifo_occup_slab_4++; + else if (fifo_occup > 4 * rdev->pacing_algo_th) + rdev->dbg_stats->dbq.fifo_occup_slab_3++; + else if (fifo_occup > 2 * rdev->pacing_algo_th) + rdev->dbg_stats->dbq.fifo_occup_slab_2++; + else if (fifo_occup > rdev->pacing_algo_th) + rdev->dbg_stats->dbq.fifo_occup_slab_1++; +} + +static void bnxt_re_update_do_pacing_slabs(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; + + if (pacing_data->do_pacing > rdev->dbg_stats->dbq.do_pacing_water_mark) + rdev->dbg_stats->dbq.do_pacing_water_mark = pacing_data->do_pacing; + + if (pacing_data->do_pacing > 16 * rdev->dbr_def_do_pacing) + rdev->dbg_stats->dbq.do_pacing_slab_5++; + else if (pacing_data->do_pacing > 8 * rdev->dbr_def_do_pacing) + rdev->dbg_stats->dbq.do_pacing_slab_4++; + else if (pacing_data->do_pacing > 4 * rdev->dbr_def_do_pacing) + rdev->dbg_stats->dbq.do_pacing_slab_3++; + else if (pacing_data->do_pacing > 2 * rdev->dbr_def_do_pacing) + rdev->dbg_stats->dbq.do_pacing_slab_2++; + else if (pacing_data->do_pacing > rdev->dbr_def_do_pacing) + rdev->dbg_stats->dbq.do_pacing_slab_1++; +} + +static bool bnxt_re_is_qp1_qp(struct bnxt_re_qp *qp) +{ + return qp->ib_qp.qp_type == IB_QPT_GSI; +} + +static struct bnxt_re_qp *bnxt_re_get_qp1_qp(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_qp *qp; + + mutex_lock(&rdev->qp_lock); + list_for_each_entry(qp, &rdev->qp_list, list) { + if (bnxt_re_is_qp1_qp(qp)) { + mutex_unlock(&rdev->qp_lock); + return qp; + } + } + mutex_unlock(&rdev->qp_lock); + return NULL; +} + +/* Set the maximum number of each resource that the driver actually wants + * to allocate. This may be up to the maximum number the firmware has + * reserved for the function. The driver may choose to allocate fewer + * resources than the firmware maximum. + */ +static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_max_res dev_res = {}; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_qplib_dev_attr *attr; + struct bnxt_qplib_ctx *hctx; + int i; + + attr = rdev->dev_attr; + hctx = rdev->qplib_res.hctx; + cctx = rdev->chip_ctx; + + bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, false); + if (!_is_chip_gen_p5_p7(cctx)) { + hctx->qp_ctx.max = min_t(u32, dev_res.max_qp, attr->max_qp); + hctx->mrw_ctx.max = min_t(u32, dev_res.max_mr, attr->max_mr); + /* To accommodate 16k MRs and 16k AHs, + * driver has to allocate 32k backing store memory + */ + hctx->mrw_ctx.max *= 2; + hctx->srq_ctx.max = min_t(u32, dev_res.max_srq, attr->max_srq); + hctx->cq_ctx.max = min_t(u32, dev_res.max_cq, attr->max_cq); + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) + hctx->tqm_ctx.qcount[i] = attr->tqm_alloc_reqs[i]; + } else { + hctx->qp_ctx.max = attr->max_qp ? attr->max_qp : dev_res.max_qp; + hctx->mrw_ctx.max = attr->max_mr ? attr->max_mr : dev_res.max_mr; + hctx->srq_ctx.max = attr->max_srq ? attr->max_srq : dev_res.max_srq; + hctx->cq_ctx.max = attr->max_cq ? attr->max_cq : dev_res.max_cq; + } +} + +static void bnxt_re_limit_vf_res(struct bnxt_re_dev *rdev, + struct bnxt_qplib_vf_res *vf_res, + u32 num_vf) +{ + struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; + struct bnxt_qplib_max_res dev_res = {}; + + bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, true); + vf_res->max_qp = dev_res.max_qp / num_vf; + vf_res->max_srq = dev_res.max_srq / num_vf; + vf_res->max_cq = dev_res.max_cq / num_vf; + /* + * MR and AH shares the same backing store, the value specified + * for max_mrw is split into half by the FW for MR and AH + */ + vf_res->max_mrw = dev_res.max_mr * 2 / num_vf; + vf_res->max_gid = BNXT_RE_MAX_GID_PER_VF; +} + +static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_ctx *hctx; + + hctx = rdev->qplib_res.hctx; + memset(&hctx->vf_res, 0, sizeof(struct bnxt_qplib_vf_res)); + bnxt_re_limit_pf_res(rdev); + + if (rdev->num_vfs) + bnxt_re_limit_vf_res(rdev, &hctx->vf_res, rdev->num_vfs); +} + +static void bnxt_re_dettach_irq(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_rcfw *rcfw = NULL; + struct bnxt_qplib_nq *nq; + int indx; + + rcfw = &rdev->rcfw; + for (indx = 0; indx < rdev->nqr.max_init; indx++) { + nq = &rdev->nqr.nq[indx]; + mutex_lock(&nq->lock); + bnxt_qplib_nq_stop_irq(nq, false); + mutex_unlock(&nq->lock); + } + + bnxt_qplib_rcfw_stop_irq(rcfw, false); +} + +static void bnxt_re_detach_err_device(struct bnxt_re_dev *rdev) +{ + /* Free the MSIx vectors only so that L2 can proceed with MSIx disable */ + bnxt_re_dettach_irq(rdev); + + /* Set the state as detached to prevent sending any more commands */ + set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); + set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); + wake_up_all(&rdev->rcfw.cmdq.waitq); +} + +#define MAX_DSCP_PRI_TUPLE 64 + +struct bnxt_re_dcb_work { + struct work_struct work; + struct bnxt_re_dev *rdev; + struct hwrm_async_event_cmpl cmpl; +}; + +static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev) +{ + rdev->dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq"); +} + +static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev) +{ + if (!rdev->dcb_wq) + return; + flush_workqueue(rdev->dcb_wq); + destroy_workqueue(rdev->dcb_wq); + rdev->dcb_wq = NULL; +} + +static void bnxt_re_init_aer_wq(struct bnxt_re_dev *rdev) +{ + rdev->aer_wq = create_singlethread_workqueue("bnxt_re_aer_wq"); +} + +static void bnxt_re_uninit_aer_wq(struct bnxt_re_dev *rdev) +{ + if (!rdev->aer_wq) + return; + flush_workqueue(rdev->aer_wq); + destroy_workqueue(rdev->aer_wq); + rdev->aer_wq = NULL; +} + +static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_qp *qp; + + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + return 0; + + qp = bnxt_re_get_qp1_qp(rdev); + if (!qp) + return 0; + + qp->qplib_qp.modify_flags = CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP; + qp->qplib_qp.tos_dscp = rdev->cc_param.qp1_tos_dscp; + + return bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); +} + +static void bnxt_re_reconfigure_dscp(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_cc_param *cc_param; + struct bnxt_re_tc_rec *tc_rec; + bool update_cc = false; + u8 dscp_user; + int rc; + + cc_param = &rdev->cc_param; + tc_rec = &rdev->tc_rec[0]; + + if (!(cc_param->roce_dscp_user || cc_param->cnp_dscp_user)) + return; + + if (cc_param->cnp_dscp_user) { + dscp_user = (cc_param->cnp_dscp_user & 0x3f); + if ((tc_rec->cnp_dscp_bv & (1ul << dscp_user)) && + (cc_param->alt_tos_dscp != dscp_user)) { + cc_param->alt_tos_dscp = dscp_user; + cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP; + update_cc = true; + } + } + + if (cc_param->roce_dscp_user) { + dscp_user = (cc_param->roce_dscp_user & 0x3f); + if ((tc_rec->roce_dscp_bv & (1ul << dscp_user)) && + (cc_param->tos_dscp != dscp_user)) { + cc_param->tos_dscp = dscp_user; + cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; + update_cc = true; + } + } + + if (update_cc) { + rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); + if (rc) + dev_err(rdev_to_dev(rdev), "Failed to apply cc settings\n"); + } +} + +static void bnxt_re_dcb_wq_task(struct work_struct *work) +{ + struct bnxt_qplib_cc_param *cc_param; + struct bnxt_re_tc_rec *tc_rec; + struct bnxt_re_dev *rdev; + struct bnxt_re_dcb_work *dcb_work = + container_of(work, struct bnxt_re_dcb_work, work); + int rc; + + rdev = dcb_work->rdev; + if (!rdev) + goto exit; + + mutex_lock(&rdev->cc_lock); + + cc_param = &rdev->cc_param; + rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, cc_param); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to query ccparam rc:%d", rc); + goto fail; + } + tc_rec = &rdev->tc_rec[0]; + /* + * Upon the receival of DCB Async event: + * If roce_dscp or cnp_dscp or both (which user configured using configfs) + * is in the list, re-program the value using modify_roce_cc command + */ + bnxt_re_reconfigure_dscp(rdev); + + cc_param->roce_pri = tc_rec->roce_prio; + if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { + cc_param->qp1_tos_dscp = cc_param->tos_dscp; + rc = bnxt_re_update_qp1_tos_dscp(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1 rc:%d", + __func__, rc); + goto fail; + } + } + +fail: + mutex_unlock(&rdev->cc_lock); +exit: + kfree(dcb_work); +} + +static int bnxt_re_hwrm_dbr_pacing_broadcast_event(struct bnxt_re_dev *rdev) +{ + struct hwrm_func_dbr_pacing_broadcast_event_output resp = {0}; + struct hwrm_func_dbr_pacing_broadcast_event_input req = {0}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + int rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_DBR_PACING_BROADCAST_EVENT, -1, -1); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_dbg(rdev_to_dev(rdev), + "Failed to send dbr pacing broadcast event rc:%d", rc); + return rc; + } + return 0; +} + +static int bnxt_re_hwrm_dbr_pacing_nqlist_query(struct bnxt_re_dev *rdev) +{ + struct hwrm_func_dbr_pacing_nqlist_query_output resp = {0}; + struct hwrm_func_dbr_pacing_nqlist_query_input req = {0}; + struct bnxt_dbq_nq_list *nq_list = &rdev->nq_list; + struct bnxt_en_dev *en_dev = rdev->en_dev; + bool primary_found = false; + struct bnxt_fw_msg fw_msg; + struct bnxt_qplib_nq *nq; + int rc, i, j = 1; + u16 *nql_ptr; + + nq = &rdev->nqr.nq[0]; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_DBR_PACING_NQLIST_QUERY, -1, -1); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to send dbr pacing nq list query rc:%d", rc); + return rc; + } + nq_list->num_nql_entries = le32_to_cpu(resp.num_nqs); + nql_ptr = &resp.nq_ring_id0; + /* populate the nq_list of the primary function with list received + * from FW. Fill the NQ IDs of secondary functions from index 1 to + * num_nql_entries - 1. Fill the nq_list->nq_id[0] with the + * nq_id of the primary pf + */ + for (i = 0; i < nq_list->num_nql_entries; i++) { + u16 nq_id = *nql_ptr; + + dev_dbg(rdev_to_dev(rdev), + "nq_list->nq_id[%d] = %d\n", i, nq_id); + if (nq_id != nq->ring_id) { + nq_list->nq_id[j] = nq_id; + j++; + } else { + primary_found = true; + nq_list->nq_id[0] = nq->ring_id; + } + nql_ptr++; + } + if (primary_found) + bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); + + return 0; +} + +static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; + u32 read_val, fifo_occup; + bool first_read = true; + + /* loop shouldn't run infintely as the occupancy usually goes + * below pacing algo threshold as soon as pacing kicks in. + */ + while (1) { + read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_db_fifo_reg_off, 0); + fifo_occup = pacing_data->fifo_max_depth - + ((read_val & pacing_data->fifo_room_mask) >> + pacing_data->fifo_room_shift); + /* Fifo occupancy cannot be greater the MAX FIFO depth */ + if (fifo_occup > pacing_data->fifo_max_depth) + break; + + if (first_read) { + bnxt_re_update_fifo_occup_slabs(rdev, fifo_occup); + first_read = false; + } + if (fifo_occup < pacing_data->pacing_th) + break; + } +} + +static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; + + pacing_data->do_pacing = rdev->dbr_def_do_pacing; + pacing_data->pacing_th = rdev->pacing_algo_th; + pacing_data->alarm_th = + pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); +} + +#define CAG_RING_MASK 0x7FF +#define CAG_RING_SHIFT 17 +#define WATERMARK_MASK 0xFFF +#define WATERMARK_SHIFT 0 + +static bool bnxt_re_check_if_dbq_intr_triggered(struct bnxt_re_dev *rdev) +{ + u32 read_val; + int j; + + for (j = 0; j < 10; j++) { + read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0); + dev_dbg(rdev_to_dev(rdev), "AEQ ARM status = 0x%x\n", + read_val); + if (!read_val) + return true; + } + return false; +} + +int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, u16 nq_id, u32 throttle) +{ + u32 cag_ring_water_mark = 0, read_val; + u32 throttle_val; + + /* Convert throttle percentage to value */ + throttle_val = (rdev->qplib_res.pacing_data->fifo_max_depth * throttle) / 100; + + if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { + cag_ring_water_mark = (nq_id & CAG_RING_MASK) << CAG_RING_SHIFT | + (throttle_val & WATERMARK_MASK); + writel_fbsd(rdev->en_dev->softc, rdev->dbr_throttling_reg_off, 0, cag_ring_water_mark); + read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_throttling_reg_off, 0); + dev_dbg(rdev_to_dev(rdev), + "%s: dbr_throttling_reg_off read_val = 0x%x\n", + __func__, read_val); + if (read_val != cag_ring_water_mark) { + dev_dbg(rdev_to_dev(rdev), + "nq_id = %d write_val=0x%x read_val=0x%x\n", + nq_id, cag_ring_water_mark, read_val); + return 1; + } + } + writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0, 1); + return 0; +} + +static void bnxt_re_set_dbq_throttling_for_non_primary(struct bnxt_re_dev *rdev) +{ + struct bnxt_dbq_nq_list *nq_list; + struct bnxt_qplib_nq *nq; + int i; + + nq_list = &rdev->nq_list; + /* Run a loop for other Active functions if this is primary function */ + if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { + dev_dbg(rdev_to_dev(rdev), "%s: nq_list->num_nql_entries= %d\n", + __func__, nq_list->num_nql_entries); + nq = &rdev->nqr.nq[0]; + for (i = nq_list->num_nql_entries - 1; i > 0; i--) { + u16 nq_id = nq_list->nq_id[i]; + if (nq) + dev_dbg(rdev_to_dev(rdev), + "%s: nq_id = %d cur_fn_ring_id = %d\n", + __func__, nq_id, nq->ring_id); + if (bnxt_re_set_dbq_throttling_reg + (rdev, nq_id, 0)) + break; + bnxt_re_check_if_dbq_intr_triggered(rdev); + } + } +} + +static void bnxt_re_handle_dbr_nq_pacing_notification(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_nq *nq; + int rc = 0; + + nq = &rdev->nqr.nq[0]; + + /* Query the NQ list*/ + rc = bnxt_re_hwrm_dbr_pacing_nqlist_query(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to Query NQ list rc= %d", rc); + return; + } + /*Configure GRC access for Throttling and aeq_arm register */ + writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28, 0, + rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); + + rdev->dbr_throttling_reg_off = + (rdev->chip_ctx->dbr_throttling_reg & + BNXT_GRC_OFFSET_MASK) + 0x8000; + rdev->dbr_aeq_arm_reg_off = + (rdev->chip_ctx->dbr_aeq_arm_reg & + BNXT_GRC_OFFSET_MASK) + 0x8000; + + bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, rdev->dbq_watermark); +} + +static void bnxt_re_dbq_wq_task(struct work_struct *work) +{ + struct bnxt_re_dbq_work *dbq_work = + container_of(work, struct bnxt_re_dbq_work, work); + struct bnxt_re_dev *rdev; + + rdev = dbq_work->rdev; + + if (!rdev) + goto exit; + switch (dbq_work->event) { + case BNXT_RE_DBQ_EVENT_SCHED: + dev_dbg(rdev_to_dev(rdev), "%s: Handle DBQ Pacing event\n", + __func__); + if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) + bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); + else + bnxt_re_pacing_alert(rdev); + break; + case BNXT_RE_DBR_PACING_EVENT: + dev_dbg(rdev_to_dev(rdev), "%s: Sched interrupt/pacing worker\n", + __func__); + if (_is_chip_p7(rdev->chip_ctx)) + bnxt_re_pacing_alert(rdev); + else if (!rdev->chip_ctx->modes.dbr_pacing_v0) + bnxt_re_hwrm_dbr_pacing_qcfg(rdev); + break; + case BNXT_RE_DBR_NQ_PACING_NOTIFICATION: + bnxt_re_handle_dbr_nq_pacing_notification(rdev); + /* Issue a broadcast event to notify other functions + * that primary changed + */ + bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); + break; + } +exit: + kfree(dbq_work); +} + +static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *cmpl) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); + struct bnxt_re_dcb_work *dcb_work; + struct bnxt_re_dbq_work *dbq_work; + struct bnxt_re_dev *rdev; + u16 event_id; + u32 data1; + u32 data2 = 0; + + if (!cmpl) { + pr_err("Async event, bad completion\n"); + return; + } + + if (!en_info || !en_info->en_dev) { + pr_err("Async event, bad en_info or en_dev\n"); + return; + } + rdev = en_info->rdev; + + event_id = le16_to_cpu(cmpl->event_id); + data1 = le32_to_cpu(cmpl->event_data1); + data2 = le32_to_cpu(cmpl->event_data2); + + if (!rdev || !rdev_to_dev(rdev)) { + dev_dbg(NULL, "Async event, bad rdev or netdev\n"); + return; + } + + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) || + !test_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { + dev_dbg(NULL, "Async event, device already detached\n"); + return; + } + if (data2 >= 0) + dev_dbg(rdev_to_dev(rdev), "Async event_id = %d data1 = %d data2 = %d", + event_id, data1, data2); + + switch (event_id) { + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE: + /* Not handling the event in older FWs */ + if (!is_qport_service_type_supported(rdev)) + break; + if (!rdev->dcb_wq) + break; + dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC); + if (!dcb_work) + break; + + dcb_work->rdev = rdev; + memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl)); + INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task); + queue_work(rdev->dcb_wq, &dcb_work->work); + break; + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: + if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { + /* Set rcfw flag to control commands send to Bono */ + set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); + /* Set bnxt_re flag to control commands send via L2 driver */ + set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); + wake_up_all(&rdev->rcfw.cmdq.waitq); + } + break; + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD: + if (!rdev->dbr_pacing) + break; + dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); + if (!dbq_work) + goto unlock; + dbq_work->rdev = rdev; + dbq_work->event = BNXT_RE_DBR_PACING_EVENT; + INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); + queue_work(rdev->dbq_wq, &dbq_work->work); + rdev->dbr_sw_stats->dbq_int_recv++; + break; + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE: + if (!rdev->dbr_pacing) + break; + + dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); + if (!dbq_work) + goto unlock; + dbq_work->rdev = rdev; + dbq_work->event = BNXT_RE_DBR_NQ_PACING_NOTIFICATION; + INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); + queue_work(rdev->dbq_wq, &dbq_work->work); + break; + + default: + break; + } +unlock: + return; +} + +static void bnxt_re_db_fifo_check(struct work_struct *work) +{ + struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, + dbq_fifo_check_work); + struct bnxt_qplib_db_pacing_data *pacing_data; + u32 pacing_save; + + if (!mutex_trylock(&rdev->dbq_lock)) + return; + pacing_data = rdev->qplib_res.pacing_data; + pacing_save = rdev->do_pacing_save; + __wait_for_fifo_occupancy_below_th(rdev); + cancel_delayed_work_sync(&rdev->dbq_pacing_work); + if (rdev->dbr_recovery_on) + goto recovery_on; + if (pacing_save > rdev->dbr_def_do_pacing) { + /* Double the do_pacing value during the congestion */ + pacing_save = pacing_save << 1; + } else { + /* + * when a new congestion is detected increase the do_pacing + * by 8 times. And also increase the pacing_th by 4 times. The + * reason to increase pacing_th is to give more space for the + * queue to oscillate down without getting empty, but also more + * room for the queue to increase without causing another alarm. + */ + pacing_save = pacing_save << 3; + pacing_data->pacing_th = rdev->pacing_algo_th * 4; + } + + if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING) + pacing_save = BNXT_RE_MAX_DBR_DO_PACING; + + pacing_data->do_pacing = pacing_save; + rdev->do_pacing_save = pacing_data->do_pacing; + pacing_data->alarm_th = + pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); +recovery_on: + schedule_delayed_work(&rdev->dbq_pacing_work, + msecs_to_jiffies(rdev->dbq_pacing_time)); + rdev->dbr_sw_stats->dbq_pacing_alerts++; + mutex_unlock(&rdev->dbq_lock); +} + +static void bnxt_re_pacing_timer_exp(struct work_struct *work) +{ + struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, + dbq_pacing_work.work); + struct bnxt_qplib_db_pacing_data *pacing_data; + u32 read_val, fifo_occup; + struct bnxt_qplib_nq *nq; + + if (!mutex_trylock(&rdev->dbq_lock)) + return; + + pacing_data = rdev->qplib_res.pacing_data; + read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_db_fifo_reg_off, 0); + fifo_occup = pacing_data->fifo_max_depth - + ((read_val & pacing_data->fifo_room_mask) >> + pacing_data->fifo_room_shift); + + if (fifo_occup > pacing_data->pacing_th) + goto restart_timer; + + /* + * Instead of immediately going back to the default do_pacing + * reduce it by 1/8 times and restart the timer. + */ + pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3); + pacing_data->do_pacing = max_t(u32, rdev->dbr_def_do_pacing, pacing_data->do_pacing); + /* + * If the fifo_occup is less than the interrupt enable threshold + * enable the interrupt on the primary PF. + */ + if (rdev->dbq_int_disable && fifo_occup < rdev->pacing_en_int_th) { + if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { + if (!rdev->chip_ctx->modes.dbr_pacing_v0) { + nq = &rdev->nqr.nq[0]; + bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, + rdev->dbq_watermark); + rdev->dbr_sw_stats->dbq_int_en++; + rdev->dbq_int_disable = false; + } + } + } + if (pacing_data->do_pacing <= rdev->dbr_def_do_pacing) { + bnxt_re_set_default_pacing_data(rdev); + rdev->dbr_sw_stats->dbq_pacing_complete++; + goto dbq_unlock; + } +restart_timer: + schedule_delayed_work(&rdev->dbq_pacing_work, + msecs_to_jiffies(rdev->dbq_pacing_time)); + bnxt_re_update_do_pacing_slabs(rdev); + rdev->dbr_sw_stats->dbq_pacing_resched++; +dbq_unlock: + rdev->do_pacing_save = pacing_data->do_pacing; + mutex_unlock(&rdev->dbq_lock); +} + +void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data; + + if (!rdev->dbr_pacing) + return; + mutex_lock(&rdev->dbq_lock); + pacing_data = rdev->qplib_res.pacing_data; + + /* + * Increase the alarm_th to max so that other user lib instances do not + * keep alerting the driver. + */ + pacing_data->alarm_th = pacing_data->fifo_max_depth; + pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING; + cancel_work_sync(&rdev->dbq_fifo_check_work); + schedule_work(&rdev->dbq_fifo_check_work); + mutex_unlock(&rdev->dbq_lock); +} + +void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res) +{ + struct bnxt_re_dbq_work *dbq_work; + struct bnxt_re_dev *rdev; + + rdev = container_of(res, struct bnxt_re_dev, qplib_res); + + atomic_set(&rdev->dbq_intr_running, 1); + + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + goto exit; + /* Run the loop to send dbq event to other functions + * for newer FW + */ + if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && + !rdev->chip_ctx->modes.dbr_pacing_v0) + bnxt_re_set_dbq_throttling_for_non_primary(rdev); + + dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); + if (!dbq_work) + goto exit; + dbq_work->rdev = rdev; + dbq_work->event = BNXT_RE_DBQ_EVENT_SCHED; + INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); + queue_work(rdev->dbq_wq, &dbq_work->work); + rdev->dbr_sw_stats->dbq_int_recv++; + rdev->dbq_int_disable = true; +exit: + atomic_set(&rdev->dbq_intr_running, 0); +} + +static void bnxt_re_free_msix(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + int rc; + + rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); + if (rc) + dev_err(rdev_to_dev(rdev), "netdev %p free_msix failed! rc = 0x%x", + rdev->netdev, rc); +} + +static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + int rc = 0, num_msix_want, num_msix_got; + struct bnxt_msix_entry *entry; + + /* + * Request MSIx based on the function type. This is + * a temporory solution to enable max VFs when NPAR is + * enabled. + * TODO - change the scheme with an adapter specific check + * as the latest adapters can support more NQs. For now + * this change satisfy all adapter versions. + */ + + if (rdev->is_virtfn) + num_msix_want = BNXT_RE_MAX_MSIX_VF; + else if (BNXT_EN_NPAR(en_dev)) + num_msix_want = BNXT_RE_MAX_MSIX_NPAR_PF; + else if (_is_chip_gen_p5_p7(rdev->chip_ctx)) + num_msix_want = rdev->num_msix_requested ?: BNXT_RE_MAX_MSIX_GEN_P5_PF; + else + num_msix_want = BNXT_RE_MAX_MSIX_PF; + + /* + * Since MSIX vectors are used for both NQs and CREQ, we should try to + * allocate num_online_cpus + 1 by taking into account the CREQ. This + * leaves the number of MSIX vectors for NQs match the number of CPUs + * and allows the system to be fully utilized + */ + num_msix_want = min_t(u32, num_msix_want, num_online_cpus() + 1); + num_msix_want = min_t(u32, num_msix_want, BNXT_RE_MAX_MSIX); + num_msix_want = max_t(u32, num_msix_want, BNXT_RE_MIN_MSIX); + + entry = rdev->nqr.msix_entries; + + num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, + entry, num_msix_want); + if (num_msix_got < BNXT_RE_MIN_MSIX) { + rc = -EINVAL; + goto done; + } + if (num_msix_got != num_msix_want) + dev_warn(rdev_to_dev(rdev), + "bnxt_request_msix: wanted %d vectors, got %d\n", + num_msix_want, num_msix_got); + + rdev->nqr.num_msix = num_msix_got; + return 0; +done: + if (num_msix_got) + bnxt_re_free_msix(rdev); + return rc; +} + +static int __wait_for_ib_unregister(struct bnxt_re_dev *rdev, + struct bnxt_re_en_dev_info *en_info) +{ + u64 timeout = 0; + u32 cur_prod = 0, cur_cons = 0; + int retry = 0, rc = 0, ret = 0; + + cur_prod = rdev->rcfw.cmdq.hwq.prod; + cur_cons = rdev->rcfw.cmdq.hwq.cons; + timeout = msecs_to_jiffies(BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS); + retry = BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY; + /* During module exit, increase timeout ten-fold to 100 mins to wait + * as long as possible for ib_unregister() to complete + */ + if (rdev->mod_exit) + retry *= 10; + do { + /* + * Since the caller of this function invokes with bnxt_re_mutex held, + * release it to avoid holding a lock while in wait / sleep mode. + */ + mutex_unlock(&bnxt_re_mutex); + rc = wait_event_timeout(en_info->waitq, + en_info->ib_uninit_done, + timeout); + mutex_lock(&bnxt_re_mutex); + + if (!bnxt_re_is_rdev_valid(rdev)) + break; + + if (rc) + break; + + if (!RCFW_NO_FW_ACCESS(&rdev->rcfw)) { + /* No need to check for cmdq stall during module exit, + * wait for ib unregister to complete + */ + if (!rdev->mod_exit) + ret = __check_cmdq_stall(&rdev->rcfw, &cur_prod, &cur_cons); + if (ret || en_info->ib_uninit_done) + break; + } + } while (retry--); + + return rc; +} + +static int bnxt_re_handle_start(struct auxiliary_device *adev) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); + struct bnxt_re_dev *rdev = NULL; + struct ifnet *real_dev; + struct bnxt_en_dev *en_dev; + struct ifnet *netdev; + int rc = 0; + + if (!en_info || !en_info->en_dev) { + pr_err("Start, bad en_info or en_dev\n"); + return -EINVAL; + } + netdev = en_info->en_dev->net; + if (en_info->rdev) { + dev_info(rdev_to_dev(en_info->rdev), + "%s: Device is already added adev %p rdev: %p\n", + __func__, adev, en_info->rdev); + return 0; + } + + en_dev = en_info->en_dev; + real_dev = rdma_vlan_dev_real_dev(netdev); + if (!real_dev) + real_dev = netdev; + rc = bnxt_re_add_device(&rdev, real_dev, + en_info->gsi_mode, + BNXT_RE_POST_RECOVERY_INIT, + en_info->wqe_mode, + en_info->num_msix_requested, adev); + if (rc) { + /* Add device failed. Unregister the device. + * This has to be done explicitly as + * bnxt_re_stop would not have unregistered + */ + rtnl_lock(); + en_dev->en_ops->bnxt_unregister_device(en_dev, BNXT_ROCE_ULP); + rtnl_unlock(); + mutex_lock(&bnxt_re_dev_lock); + gadd_dev_inprogress--; + mutex_unlock(&bnxt_re_dev_lock); + return rc; + } + rdev->adev = adev; + rtnl_lock(); + bnxt_re_get_link_speed(rdev); + rtnl_unlock(); + rc = bnxt_re_ib_init(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed ib_init\n"); + return rc; + } + bnxt_re_ib_init_2(rdev); + + return rc; +} + +static void bnxt_re_stop(void *handle) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); + struct ifnet *netdev; + struct bnxt_re_dev *rdev; + struct bnxt_en_dev *en_dev; + int rc = 0; + + rtnl_unlock(); + mutex_lock(&bnxt_re_mutex); + if (!en_info || !en_info->en_dev) { + pr_err("Stop, bad en_info or en_dev\n"); + goto exit; + } + netdev = en_info->en_dev->net; + rdev = en_info->rdev; + if (!rdev) + goto exit; + + if (!bnxt_re_is_rdev_valid(rdev)) + goto exit; + + /* + * Check if fw has undergone reset or is in a fatal condition. + * If so, set flags so that no further commands are sent down to FW + */ + en_dev = rdev->en_dev; + if (en_dev->en_state & BNXT_STATE_FW_FATAL_COND || + en_dev->en_state & BNXT_STATE_FW_RESET_DET) { + /* Set rcfw flag to control commands send to Bono */ + set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); + /* Set bnxt_re flag to control commands send via L2 driver */ + set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); + wake_up_all(&rdev->rcfw.cmdq.waitq); + } + + if (test_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags)) + goto exit; + set_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags); + + en_info->wqe_mode = rdev->chip_ctx->modes.wqe_mode; + en_info->gsi_mode = rdev->gsi_ctx.gsi_qp_mode; + en_info->num_msix_requested = rdev->num_msix_requested; + en_info->ib_uninit_done = false; + + if (rdev->dbr_pacing) + bnxt_re_set_pacing_dev_state(rdev); + + dev_info(rdev_to_dev(rdev), "%s: L2 driver notified to stop." + "Attempting to stop and Dispatching event " + "to inform the stack\n", __func__); + init_waitqueue_head(&en_info->waitq); + /* Schedule a work item to handle IB UNINIT for recovery */ + bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, + NULL, netdev, rdev->adev); + rc = __wait_for_ib_unregister(rdev, en_info); + if (!bnxt_re_is_rdev_valid(rdev)) + goto exit; + if (!rc) { + dev_info(rdev_to_dev(rdev), "%s: Attempt to stop failed\n", + __func__); + bnxt_re_detach_err_device(rdev); + goto exit; + } + bnxt_re_remove_device(rdev, BNXT_RE_PRE_RECOVERY_REMOVE, rdev->adev); +exit: + mutex_unlock(&bnxt_re_mutex); + /* Take rtnl_lock before return, bnxt_re_stop is called with rtnl_lock */ + rtnl_lock(); + + return; +} + +static void bnxt_re_start(void *handle) +{ + rtnl_unlock(); + mutex_lock(&bnxt_re_mutex); + if (bnxt_re_handle_start((struct auxiliary_device *)handle)) + pr_err("Failed to start RoCE device"); + mutex_unlock(&bnxt_re_mutex); + /* Take rtnl_lock before return, bnxt_re_start is called with rtnl_lock */ + rtnl_lock(); + return; +} + +static void bnxt_re_shutdown(void *p) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(p); + struct bnxt_re_dev *rdev; + + if (!en_info) { + pr_err("Shutdown, bad en_info\n"); + return; + } + rtnl_unlock(); + mutex_lock(&bnxt_re_mutex); + rdev = en_info->rdev; + if (!rdev || !bnxt_re_is_rdev_valid(rdev)) + goto exit; + + /* rtnl_lock held by L2 before coming here */ + bnxt_re_stopqps_and_ib_uninit(rdev); + bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, rdev->adev); +exit: + mutex_unlock(&bnxt_re_mutex); + rtnl_lock(); + return; +} + +static void bnxt_re_stop_irq(void *handle) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); + struct bnxt_qplib_rcfw *rcfw = NULL; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_nq *nq; + int indx; + + if (!en_info) { + pr_err("Stop irq, bad en_info\n"); + return; + } + rdev = en_info->rdev; + + if (!rdev) + return; + + rcfw = &rdev->rcfw; + for (indx = 0; indx < rdev->nqr.max_init; indx++) { + nq = &rdev->nqr.nq[indx]; + mutex_lock(&nq->lock); + bnxt_qplib_nq_stop_irq(nq, false); + mutex_unlock(&nq->lock); + } + + if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) + bnxt_qplib_rcfw_stop_irq(rcfw, false); +} + +static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); + struct bnxt_msix_entry *msix_ent = NULL; + struct bnxt_qplib_rcfw *rcfw = NULL; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_nq *nq; + int indx, rc, vec; + + if (!en_info) { + pr_err("Start irq, bad en_info\n"); + return; + } + rdev = en_info->rdev; + if (!rdev) + return; + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return; + msix_ent = rdev->nqr.msix_entries; + rcfw = &rdev->rcfw; + + if (!ent) { + /* Not setting the f/w timeout bit in rcfw. + * During the driver unload the first command + * to f/w will timeout and that will set the + * timeout bit. + */ + dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n"); + return; + } + + /* Vectors may change after restart, so update with new vectors + * in device structure. + */ + for (indx = 0; indx < rdev->nqr.num_msix; indx++) + rdev->nqr.msix_entries[indx].vector = ent[indx].vector; + + if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) { + rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector, + false); + if (rc) { + dev_warn(rdev_to_dev(rdev), + "Failed to reinit CREQ\n"); + return; + } + } + for (indx = 0 ; indx < rdev->nqr.max_init; indx++) { + nq = &rdev->nqr.nq[indx]; + vec = indx + 1; + rc = bnxt_qplib_nq_start_irq(nq, indx, msix_ent[vec].vector, + false); + if (rc) { + dev_warn(rdev_to_dev(rdev), + "Failed to reinit NQ index %d\n", indx); + return; + } + } +} + +/* + * Except for ulp_async_notifier, the remaining ulp_ops + * below are called with rtnl_lock held + */ +static struct bnxt_ulp_ops bnxt_re_ulp_ops = { + .ulp_async_notifier = bnxt_re_async_notifier, + .ulp_stop = bnxt_re_stop, + .ulp_start = bnxt_re_start, + .ulp_shutdown = bnxt_re_shutdown, + .ulp_irq_stop = bnxt_re_stop_irq, + .ulp_irq_restart = bnxt_re_start_irq, +}; + +static inline const char *bnxt_re_netevent(unsigned long event) +{ + BNXT_RE_NETDEV_EVENT(event, NETDEV_UP); + BNXT_RE_NETDEV_EVENT(event, NETDEV_DOWN); + BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGE); + BNXT_RE_NETDEV_EVENT(event, NETDEV_REGISTER); + BNXT_RE_NETDEV_EVENT(event, NETDEV_UNREGISTER); + BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGEADDR); + return "Unknown"; +} + +/* RoCE -> Net driver */ + +/* Driver registration routines used to let the networking driver (bnxt_en) + * to know that the RoCE driver is now installed */ +static void bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + int rc; + + rtnl_lock(); + rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, + BNXT_ROCE_ULP); + rtnl_unlock(); + if (rc) + dev_err(rdev_to_dev(rdev), "netdev %p unregister failed! rc = 0x%x", + rdev->en_dev->net, rc); + + clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); +} + +static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + int rc = 0; + + rtnl_lock(); + rc = en_dev->en_ops->bnxt_register_device(en_dev, + BNXT_ROCE_ULP, + &bnxt_re_ulp_ops, + rdev->adev); + rtnl_unlock(); + if (rc) { + dev_err(rdev_to_dev(rdev), "netdev %p register failed! rc = 0x%x", + rdev->netdev, rc); + return rc; + } + + return rc; +} + +static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_en_dev *en_dev; + struct bnxt_qplib_res *res; + u32 l2db_len = 0; + u32 offset = 0; + u32 barlen; + int rc; + + res = &rdev->qplib_res; + en_dev = rdev->en_dev; + cctx = rdev->chip_ctx; + + /* Issue qcfg */ + rc = bnxt_re_hwrm_qcfg(rdev, &l2db_len, &offset); + if (rc) + dev_info(rdev_to_dev(rdev), + "Couldn't get DB bar size, Low latency framework is disabled\n"); + /* set register offsets for both UC and WC */ + if (_is_chip_p7(cctx)) + res->dpi_tbl.ucreg.offset = offset; + else + res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET : + BNXT_QPLIB_DBR_PF_DB_OFFSET; + res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset; + + /* If WC mapping is disabled by L2 driver then en_dev->l2_db_size + * is equal to the DB-Bar actual size. This indicates that L2 + * is mapping entire bar as UC-. RoCE driver can't enable WC mapping + * in such cases and DB-push will be disabled. + */ + barlen = pci_resource_len(res->pdev, RCFW_DBR_PCI_BAR_REGION); + if (cctx->modes.db_push && l2db_len && en_dev->l2_db_size != barlen) { + res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; + dev_info(rdev_to_dev(rdev), + "Low latency framework is enabled\n"); + } + + return; +} + +static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode) +{ + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_en_dev *en_dev; + + en_dev = rdev->en_dev; + cctx = rdev->chip_ctx; + cctx->modes.wqe_mode = _is_chip_gen_p5_p7(rdev->chip_ctx) ? + mode : BNXT_QPLIB_WQE_MODE_STATIC; + cctx->modes.te_bypass = false; + if (bnxt_re_hwrm_qcaps(rdev)) + dev_err(rdev_to_dev(rdev), + "Failed to query hwrm qcaps\n"); + /* + * TODO: Need a better mechanism for spreading of the + * 512 extended PPP pages in the presence of VF and + * NPAR, until then not enabling push + */ + if (_is_chip_p7(rdev->chip_ctx) && cctx->modes.db_push) { + if (rdev->is_virtfn || BNXT_EN_NPAR(en_dev)) + cctx->modes.db_push = false; + } + + rdev->roce_mode = en_dev->flags & BNXT_EN_FLAG_ROCE_CAP; + dev_dbg(rdev_to_dev(rdev), + "RoCE is supported on the device - caps:0x%x", + rdev->roce_mode); + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + rdev->roce_mode = BNXT_RE_FLAG_ROCEV2_CAP; + cctx->hw_stats_size = en_dev->hw_ring_stats_size; +} + +static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_chip_ctx *chip_ctx; + struct bnxt_qplib_res *res; + + if (!rdev->chip_ctx) + return; + + res = &rdev->qplib_res; + bnxt_qplib_unmap_db_bar(res); + + kfree(res->hctx); + res->rcfw = NULL; + kfree(rdev->dev_attr); + rdev->dev_attr = NULL; + + chip_ctx = rdev->chip_ctx; + rdev->chip_ctx = NULL; + res->cctx = NULL; + res->hctx = NULL; + res->pdev = NULL; + res->netdev = NULL; + kfree(chip_ctx); +} + +static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode) +{ + struct bnxt_qplib_chip_ctx *chip_ctx; + struct bnxt_en_dev *en_dev; + int rc; + + en_dev = rdev->en_dev; + /* Supply pci device to qplib */ + rdev->qplib_res.pdev = en_dev->pdev; + rdev->qplib_res.netdev = rdev->netdev; + rdev->qplib_res.en_dev = en_dev; + + chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL); + if (!chip_ctx) + return -ENOMEM; + rdev->chip_ctx = chip_ctx; + rdev->qplib_res.cctx = chip_ctx; + rc = bnxt_re_query_hwrm_intf_version(rdev); + if (rc) + goto fail; + rdev->dev_attr = kzalloc(sizeof(*rdev->dev_attr), GFP_KERNEL); + if (!rdev->dev_attr) { + rc = -ENOMEM; + goto fail; + } + rdev->qplib_res.dattr = rdev->dev_attr; + rdev->qplib_res.rcfw = &rdev->rcfw; + rdev->qplib_res.is_vf = rdev->is_virtfn; + + rdev->qplib_res.hctx = kzalloc(sizeof(*rdev->qplib_res.hctx), + GFP_KERNEL); + if (!rdev->qplib_res.hctx) { + rc = -ENOMEM; + goto fail; + } + bnxt_re_set_drv_mode(rdev, wqe_mode); + + bnxt_re_set_db_offset(rdev); + rc = bnxt_qplib_map_db_bar(&rdev->qplib_res); + if (rc) + goto fail; + + rc = bnxt_qplib_enable_atomic_ops_to_root(en_dev->pdev); + if (rc) + dev_dbg(rdev_to_dev(rdev), + "platform doesn't support global atomics"); + + return 0; +fail: + kfree(rdev->chip_ctx); + rdev->chip_ctx = NULL; + + kfree(rdev->dev_attr); + rdev->dev_attr = NULL; + + kfree(rdev->qplib_res.hctx); + rdev->qplib_res.hctx = NULL; + return rc; +} + +static u16 bnxt_re_get_rtype(struct bnxt_re_dev *rdev) { + return _is_chip_gen_p5_p7(rdev->chip_ctx) ? + HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ : + HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL; +} + +static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id) +{ + int rc = -EINVAL; + struct hwrm_ring_free_input req = {0}; + struct hwrm_ring_free_output resp; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + + if (!en_dev) + return rc; + + /* To avoid unnecessary error messages during recovery. + * HW is anyway in error state. So dont send down the command */ + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return 0; + + /* allocation had failed, no need to issue hwrm */ + if (fw_ring_id == 0xffff) + return 0; + + memset(&fw_msg, 0, sizeof(fw_msg)); + + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); + req.ring_type = bnxt_re_get_rtype(rdev); + req.ring_id = cpu_to_le16(fw_ring_id); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to free HW ring with rc = 0x%x", rc); + return rc; + } + dev_dbg(rdev_to_dev(rdev), "HW ring freed with id = 0x%x\n", + fw_ring_id); + + return rc; +} + +static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, + struct bnxt_re_ring_attr *ring_attr, + u16 *fw_ring_id) +{ + int rc = -EINVAL; + struct hwrm_ring_alloc_input req = {0}; + struct hwrm_ring_alloc_output resp; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + + if (!en_dev) + return rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); + req.flags = cpu_to_le16(ring_attr->flags); + req.enables = 0; + req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]); + if (ring_attr->pages > 1) { + /* Page size is in log2 units */ + req.page_size = BNXT_PAGE_SHIFT; + req.page_tbl_depth = 1; + } else { + req.page_size = 4; + req.page_tbl_depth = 0; + } + + req.fbo = 0; + /* Association of ring index with doorbell index and MSIX number */ + req.logical_id = cpu_to_le16(ring_attr->lrid); + req.length = cpu_to_le32(ring_attr->depth + 1); + req.ring_type = ring_attr->type; + req.int_mode = ring_attr->mode; + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to allocate HW ring with rc = 0x%x", rc); + return rc; + } + *fw_ring_id = le16_to_cpu(resp.ring_id); + dev_dbg(rdev_to_dev(rdev), + "HW ring allocated with id = 0x%x at slot 0x%x", + resp.ring_id, ring_attr->lrid); + + return rc; +} + +static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, + u32 fw_stats_ctx_id, u16 tid) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_stat_ctx_free_input req = {0}; + struct hwrm_stat_ctx_free_output resp; + struct bnxt_fw_msg fw_msg; + int rc = -EINVAL; + + if (!en_dev) + return rc; + + /* To avoid unnecessary error messages during recovery. + * HW is anyway in error state. So dont send down the command */ + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return 0; + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, tid); + req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to free HW stats ctx with rc = 0x%x", rc); + return rc; + } + dev_dbg(rdev_to_dev(rdev), + "HW stats ctx freed with id = 0x%x", fw_stats_ctx_id); + + return rc; +} + +static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, u16 tid) +{ + struct hwrm_stat_ctx_alloc_output resp = {}; + struct hwrm_stat_ctx_alloc_input req = {}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_qplib_stats *stat; + struct bnxt_qplib_ctx *hctx; + struct bnxt_fw_msg fw_msg; + int rc = 0; + + hctx = rdev->qplib_res.hctx; + stat = (tid == 0xffff) ? &hctx->stats : &hctx->stats2; + stat->fw_id = INVALID_STATS_CTX_ID; + + if (!en_dev) + return -EINVAL; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_STAT_CTX_ALLOC, -1, tid); + req.update_period_ms = cpu_to_le32(1000); + req.stats_dma_length = rdev->chip_ctx->hw_stats_size; + req.stats_dma_addr = cpu_to_le64(stat->dma_map); + req.stat_ctx_flags = HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE; + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to allocate HW stats ctx, rc = 0x%x", rc); + return rc; + } + stat->fw_id = le32_to_cpu(resp.stat_ctx_id); + dev_dbg(rdev_to_dev(rdev), "HW stats ctx allocated with id = 0x%x", + stat->fw_id); + + return rc; +} + +static void bnxt_re_net_unregister_async_event(struct bnxt_re_dev *rdev) +{ + const struct bnxt_en_ops *en_ops; + + if (rdev->is_virtfn || + test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return; + + memset(rdev->event_bitmap, 0, sizeof(rdev->event_bitmap)); + en_ops = rdev->en_dev->en_ops; + if (en_ops->bnxt_register_fw_async_events + (rdev->en_dev, BNXT_ROCE_ULP, + (unsigned long *)rdev->event_bitmap, + HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) + dev_err(rdev_to_dev(rdev), + "Failed to unregister async event"); +} + +static void bnxt_re_net_register_async_event(struct bnxt_re_dev *rdev) +{ + const struct bnxt_en_ops *en_ops; + + if (rdev->is_virtfn) + return; + + rdev->event_bitmap[0] |= + BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE) | + BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY); + + rdev->event_bitmap[2] |= + BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT - 64); + rdev->event_bitmap[2] |= + BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD - 64) | + BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE - 64); + en_ops = rdev->en_dev->en_ops; + if (en_ops->bnxt_register_fw_async_events + (rdev->en_dev, BNXT_ROCE_ULP, + (unsigned long *)rdev->event_bitmap, + HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) + dev_err(rdev_to_dev(rdev), + "Failed to reg Async event"); +} + +static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_ver_get_output resp = {0}; + struct hwrm_ver_get_input req = {0}; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_fw_msg fw_msg; + int rc = 0; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_VER_GET, -1, -1); + req.hwrm_intf_maj = HWRM_VERSION_MAJOR; + req.hwrm_intf_min = HWRM_VERSION_MINOR; + req.hwrm_intf_upd = HWRM_VERSION_UPDATE; + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to query HW version, rc = 0x%x", rc); + return rc; + } + cctx = rdev->chip_ctx; + cctx->hwrm_intf_ver = (u64) le16_to_cpu(resp.hwrm_intf_major) << 48 | + (u64) le16_to_cpu(resp.hwrm_intf_minor) << 32 | + (u64) le16_to_cpu(resp.hwrm_intf_build) << 16 | + le16_to_cpu(resp.hwrm_intf_patch); + + cctx->hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout); + + if (!cctx->hwrm_cmd_max_timeout) + cctx->hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT; + + cctx->chip_num = le16_to_cpu(resp.chip_num); + cctx->chip_rev = resp.chip_rev; + cctx->chip_metal = resp.chip_metal; + return 0; +} + +/* Query device config using common hwrm */ +static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, + u32 *offset) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_func_qcfg_output resp = {0}; + struct hwrm_func_qcfg_input req = {0}; + struct bnxt_fw_msg fw_msg; + int rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_QCFG, -1, -1); + req.fid = cpu_to_le16(0xffff); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to query config, rc = %#x", rc); + return rc; + } + + *db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024); + *offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024); + return 0; +} + +/* Query function capabilities using common hwrm */ +int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_func_qcaps_output resp = {0}; + struct hwrm_func_qcaps_input req = {0}; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_fw_msg fw_msg; + u8 push_enable = false; + int rc; + + cctx = rdev->chip_ctx; + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_QCAPS, -1, -1); + req.fid = cpu_to_le16(0xffff); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to query capabilities, rc = %#x", rc); + return rc; + } + if (_is_chip_p7(rdev->chip_ctx)) + push_enable = + (resp.flags_ext & + HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PPP_PUSH_MODE_SUPPORTED) ? + true : false; + else + push_enable = + (resp.flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WCB_PUSH_MODE) ? + true : false; + cctx->modes.db_push = push_enable; + + cctx->modes.dbr_pacing = + resp.flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED ? + true : false; + cctx->modes.dbr_pacing_ext = + resp.flags_ext2 & + HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED ? + true : false; + cctx->modes.dbr_drop_recov = + (resp.flags_ext2 & + HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED) ? + true : false; + cctx->modes.dbr_pacing_v0 = + (resp.flags_ext2 & + HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED) ? + true : false; + dev_dbg(rdev_to_dev(rdev), + "%s: cctx->modes.dbr_pacing = %d cctx->modes.dbr_pacing_ext = %d, dbr_drop_recov %d\n", + __func__, cctx->modes.dbr_pacing, cctx->modes.dbr_pacing_ext, cctx->modes.dbr_drop_recov); + + return 0; +} + +static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; + struct hwrm_func_dbr_pacing_qcfg_output resp = {0}; + struct hwrm_func_dbr_pacing_qcfg_input req = {0}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_fw_msg fw_msg; + u32 primary_nq_id; + int rc; + + cctx = rdev->chip_ctx; + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_DBR_PACING_QCFG, -1, -1); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_dbg(rdev_to_dev(rdev), + "Failed to query dbr pacing config, rc = %#x", rc); + return rc; + } + + primary_nq_id = le32_to_cpu(resp.primary_nq_id); + if (primary_nq_id == 0xffffffff && + !bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { + dev_err(rdev_to_dev(rdev), "%s:%d Invoke bnxt_qplib_dbr_pacing_set_primary_pf with 1\n", + __func__, __LINE__); + bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); + } + + if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { + struct bnxt_qplib_nq *nq; + + nq = &rdev->nqr.nq[0]; + /* Reset the primary capability */ + if (nq->ring_id != primary_nq_id) + bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); + } + + if ((resp.dbr_stat_db_fifo_reg & + HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) == + HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC) + cctx->dbr_stat_db_fifo = + resp.dbr_stat_db_fifo_reg & + ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; + + if ((resp.dbr_throttling_aeq_arm_reg & + HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_MASK) + == HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_GRC) { + cctx->dbr_aeq_arm_reg = resp.dbr_throttling_aeq_arm_reg & + ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; + cctx->dbr_throttling_reg = cctx->dbr_aeq_arm_reg - 4; + } + pacing_data->fifo_max_depth = le32_to_cpu(resp.dbr_stat_db_max_fifo_depth); + if (!pacing_data->fifo_max_depth) + pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH(cctx); + pacing_data->fifo_room_mask = le32_to_cpu(resp.dbr_stat_db_fifo_reg_fifo_room_mask); + pacing_data->fifo_room_shift = resp.dbr_stat_db_fifo_reg_fifo_room_shift; + dev_dbg(rdev_to_dev(rdev), + "%s: nq:0x%x primary_pf:%d db_fifo:0x%x aeq_arm:0x%x i" + "fifo_max_depth 0x%x , resp.dbr_stat_db_max_fifo_depth 0x%x);\n", + __func__, resp.primary_nq_id, cctx->modes.dbr_primary_pf, + cctx->dbr_stat_db_fifo, cctx->dbr_aeq_arm_reg, + pacing_data->fifo_max_depth, + le32_to_cpu(resp.dbr_stat_db_max_fifo_depth)); + return 0; +} + +static int bnxt_re_hwrm_dbr_pacing_cfg(struct bnxt_re_dev *rdev, bool enable) +{ + struct hwrm_func_dbr_pacing_cfg_output resp = {0}; + struct hwrm_func_dbr_pacing_cfg_input req = {0}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + int rc; + + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return 0; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_DBR_PACING_CFG, -1, -1); + if (enable) { + req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_ENABLE; + req.enables = + cpu_to_le32(HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PRIMARY_NQ_ID_VALID | + HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PACING_THRESHOLD_VALID); + } else { + req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_DISABLE; + } + req.primary_nq_id = cpu_to_le32(rdev->dbq_nq_id); + req.pacing_threshold = cpu_to_le32(rdev->dbq_watermark); + dev_dbg(rdev_to_dev(rdev), "%s: nq_id = 0x%x pacing_threshold = 0x%x", + __func__, req.primary_nq_id, req.pacing_threshold); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_dbg(rdev_to_dev(rdev), + "Failed to set dbr pacing config, rc = %#x", rc); + return rc; + } + return 0; +} + +/* Net -> RoCE driver */ + +/* Device */ +struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev) +{ + struct bnxt_re_dev *rdev; + + rcu_read_lock(); + list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { + if (rdev->netdev == netdev) { + rcu_read_unlock(); + dev_dbg(rdev_to_dev(rdev), + "netdev (%p) found, ref_count = 0x%x", + netdev, atomic_read(&rdev->ref_count)); + return rdev; + } + } + rcu_read_unlock(); + return NULL; +} + +static ssize_t show_rev(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); + + return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); +} + + +static ssize_t show_hca(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); + + return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); +} + +static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); +static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); +static struct device_attribute *bnxt_re_attributes[] = { + &dev_attr_hw_rev, + &dev_attr_hca_type +}; + +int ib_register_device_compat(struct bnxt_re_dev *rdev) +{ + struct ib_device *ibdev = &rdev->ibdev; + char name[IB_DEVICE_NAME_MAX]; + + memset(name, 0, IB_DEVICE_NAME_MAX); + strlcpy(name, "bnxt_re%d", IB_DEVICE_NAME_MAX); + + strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); + + return ib_register_device(ibdev, NULL); +} + +static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) +{ + struct ib_device *ibdev = &rdev->ibdev; + int ret = 0; + + /* ib device init */ + ibdev->owner = THIS_MODULE; + ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; + ibdev->node_type = RDMA_NODE_IB_CA; + strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", + strlen(BNXT_RE_DESC) + 5); + ibdev->phys_port_cnt = 1; + + bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ibdev->node_guid); + + /* Data path irqs is one less than the max msix vectors */ + ibdev->num_comp_vectors = rdev->nqr.num_msix - 1; + bnxt_re_set_dma_device(ibdev, rdev); + ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; + + /* User space */ + ibdev->uverbs_cmd_mask = + (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | + (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | + (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | + (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_REG_MR) | + (1ull << IB_USER_VERBS_CMD_DEREG_MR) | + (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | + (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_QP) | + (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | + (1ull << IB_USER_VERBS_CMD_QUERY_QP) | + (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | + (1ull << IB_USER_VERBS_CMD_REREG_MR) | + (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | + (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | + (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | + (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | + (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) | + (1ull << IB_USER_VERBS_CMD_CREATE_AH) | + (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | + (1ull << IB_USER_VERBS_CMD_QUERY_AH) | + (1ull << IB_USER_VERBS_CMD_DESTROY_AH); + + ibdev->uverbs_ex_cmd_mask = (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP); + ibdev->uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_POLL_CQ); + +#define bnxt_re_ib_ah bnxt_re_ah +#define bnxt_re_ib_cq bnxt_re_cq +#define bnxt_re_ib_pd bnxt_re_pd +#define bnxt_re_ib_srq bnxt_re_srq +#define bnxt_re_ib_ucontext bnxt_re_ucontext + INIT_IB_DEVICE_OPS(&ibdev->ops, bnxt_re, BNXT_RE); + + ibdev->query_device = bnxt_re_query_device; + ibdev->modify_device = bnxt_re_modify_device; + ibdev->query_port = bnxt_re_query_port; + ibdev->modify_port = bnxt_re_modify_port; + ibdev->get_port_immutable = bnxt_re_get_port_immutable; + ibdev->query_pkey = bnxt_re_query_pkey; + ibdev->query_gid = bnxt_re_query_gid; + ibdev->get_netdev = bnxt_re_get_netdev; + ibdev->add_gid = bnxt_re_add_gid; + ibdev->del_gid = bnxt_re_del_gid; + ibdev->get_link_layer = bnxt_re_get_link_layer; + ibdev->alloc_pd = bnxt_re_alloc_pd; + ibdev->dealloc_pd = bnxt_re_dealloc_pd; + ibdev->create_ah = bnxt_re_create_ah; + ibdev->modify_ah = bnxt_re_modify_ah; + ibdev->query_ah = bnxt_re_query_ah; + ibdev->destroy_ah = bnxt_re_destroy_ah; + ibdev->create_srq = bnxt_re_create_srq; + ibdev->modify_srq = bnxt_re_modify_srq; + ibdev->query_srq = bnxt_re_query_srq; + ibdev->destroy_srq = bnxt_re_destroy_srq; + ibdev->post_srq_recv = bnxt_re_post_srq_recv; + ibdev->create_qp = bnxt_re_create_qp; + ibdev->modify_qp = bnxt_re_modify_qp; + ibdev->query_qp = bnxt_re_query_qp; + ibdev->destroy_qp = bnxt_re_destroy_qp; + ibdev->post_send = bnxt_re_post_send; + ibdev->post_recv = bnxt_re_post_recv; + ibdev->create_cq = bnxt_re_create_cq; + ibdev->modify_cq = bnxt_re_modify_cq; + ibdev->destroy_cq = bnxt_re_destroy_cq; + ibdev->resize_cq = bnxt_re_resize_cq; + ibdev->poll_cq = bnxt_re_poll_cq; + ibdev->req_notify_cq = bnxt_re_req_notify_cq; + ibdev->get_dma_mr = bnxt_re_get_dma_mr; + ibdev->get_hw_stats = bnxt_re_get_hw_stats; + ibdev->alloc_hw_stats = bnxt_re_alloc_hw_port_stats; + ibdev->dereg_mr = bnxt_re_dereg_mr; + ibdev->alloc_mr = bnxt_re_alloc_mr; + ibdev->map_mr_sg = bnxt_re_map_mr_sg; + ibdev->alloc_mw = bnxt_re_alloc_mw; + ibdev->dealloc_mw = bnxt_re_dealloc_mw; + ibdev->reg_user_mr = bnxt_re_reg_user_mr; + ibdev->rereg_user_mr = bnxt_re_rereg_user_mr; + ibdev->disassociate_ucontext = bnxt_re_disassociate_ucntx; + ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; + ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; + ibdev->mmap = bnxt_re_mmap; + ibdev->process_mad = bnxt_re_process_mad; + + ret = ib_register_device_compat(rdev); + return ret; +} + +static void bnxt_re_dev_dealloc(struct bnxt_re_dev *rdev) +{ + int i = BNXT_RE_REF_WAIT_COUNT; + + dev_dbg(rdev_to_dev(rdev), "%s:Remove the device %p\n", __func__, rdev); + /* Wait for rdev refcount to come down */ + while ((atomic_read(&rdev->ref_count) > 1) && i--) + msleep(100); + + if (atomic_read(&rdev->ref_count) > 1) + dev_err(rdev_to_dev(rdev), + "Failed waiting for ref count to deplete %d", + atomic_read(&rdev->ref_count)); + + atomic_set(&rdev->ref_count, 0); + if_rele(rdev->netdev); + rdev->netdev = NULL; + synchronize_rcu(); + + kfree(rdev->gid_map); + kfree(rdev->dbg_stats); + ib_dealloc_device(&rdev->ibdev); +} + +static struct bnxt_re_dev *bnxt_re_dev_alloc(struct ifnet *netdev, + struct bnxt_en_dev *en_dev) +{ + struct bnxt_re_dev *rdev; + u32 count; + + /* Allocate bnxt_re_dev instance here */ + rdev = (struct bnxt_re_dev *)compat_ib_alloc_device(sizeof(*rdev)); + if (!rdev) { + pr_err("%s: bnxt_re_dev allocation failure!", + ROCE_DRV_MODULE_NAME); + return NULL; + } + /* Default values */ + atomic_set(&rdev->ref_count, 0); + rdev->netdev = netdev; + dev_hold(rdev->netdev); + rdev->en_dev = en_dev; + rdev->id = rdev->en_dev->pdev->devfn; + INIT_LIST_HEAD(&rdev->qp_list); + mutex_init(&rdev->qp_lock); + mutex_init(&rdev->cc_lock); + mutex_init(&rdev->dbq_lock); + bnxt_re_clear_rsors_stat(&rdev->stats.rsors); + rdev->cosq[0] = rdev->cosq[1] = 0xFFFF; + rdev->min_tx_depth = 1; + rdev->stats.stats_query_sec = 1; + /* Disable priority vlan as the default mode is DSCP based PFC */ + rdev->cc_param.disable_prio_vlan_tx = 1; + + /* Initialize worker for DBR Pacing */ + INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check); + INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp); + rdev->gid_map = kzalloc(sizeof(*(rdev->gid_map)) * + BNXT_RE_MAX_SGID_ENTRIES, + GFP_KERNEL); + if (!rdev->gid_map) { + ib_dealloc_device(&rdev->ibdev); + return NULL; + } + for(count = 0; count < BNXT_RE_MAX_SGID_ENTRIES; count++) + rdev->gid_map[count] = -1; + + rdev->dbg_stats = kzalloc(sizeof(*rdev->dbg_stats), GFP_KERNEL); + if (!rdev->dbg_stats) { + ib_dealloc_device(&rdev->ibdev); + return NULL; + } + + return rdev; +} + +static int bnxt_re_handle_unaffi_async_event( + struct creq_func_event *unaffi_async) +{ + switch (unaffi_async->event) { + case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: + case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: + case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: + case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: + case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: + case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: + case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: + case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: + case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: + case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: + case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: + break; + default: + return -EINVAL; + } + return 0; +} + +static int bnxt_re_handle_qp_async_event(void *qp_event, struct bnxt_re_qp *qp) +{ + struct creq_qp_error_notification *err_event; + struct ib_event event; + unsigned int flags; + + if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR && + !qp->qplib_qp.is_user) { + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_add_flush_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + } + memset(&event, 0, sizeof(event)); + event.device = &qp->rdev->ibdev; + event.element.qp = &qp->ib_qp; + event.event = IB_EVENT_QP_FATAL; + + err_event = qp_event; + switch(err_event->res_err_state_reason) { + case CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX: + case CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH: + case CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT: + case CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY: + case CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION: + case CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY: + case CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION: + case CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY: + case CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC: + event.event = IB_EVENT_QP_ACCESS_ERR; + break; + case CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE: + case CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE: + case CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE: + event.event = IB_EVENT_QP_REQ_ERR; + break; + case CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW: + case CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR: + event.event = IB_EVENT_QP_FATAL; + break; + default: + if (qp->qplib_qp.srq) + event.event = IB_EVENT_QP_LAST_WQE_REACHED; + break; + } + + if (err_event->res_err_state_reason) + dev_err(rdev_to_dev(qp->rdev), + "%s %s qp_id: %d cons (%d %d) req (%d %d) res (%d %d)\n", + __func__, qp->qplib_qp.is_user ? "user" : "kernel", + qp->qplib_qp.id, + err_event->sq_cons_idx, + err_event->rq_cons_idx, + err_event->req_slow_path_state, + err_event->req_err_state_reason, + err_event->res_slow_path_state, + err_event->res_err_state_reason); + + if (event.device && qp->ib_qp.event_handler) + qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); + + return 0; +} + +static int bnxt_re_handle_cq_async_error(void *event, struct bnxt_re_cq *cq) +{ + struct creq_cq_error_notification *cqerr; + bool send = false; + + cqerr = event; + switch (cqerr->cq_err_reason) { + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR: + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR: + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR: + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR: + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR: + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR: + send = true; + default: + break; + } + + if (send && cq->ibcq.event_handler) { + struct ib_event ibevent = {}; + + ibevent.event = IB_EVENT_CQ_ERR; + ibevent.element.cq = &cq->ibcq; + ibevent.device = &cq->rdev->ibdev; + + dev_err(rdev_to_dev(cq->rdev), + "%s err reason %d\n", __func__, cqerr->cq_err_reason); + cq->ibcq.event_handler(&ibevent, cq->ibcq.cq_context); + } + + cq->qplib_cq.is_cq_err_event = true; + + return 0; +} + +static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async, + void *obj) +{ + struct bnxt_qplib_qp *qplqp; + struct bnxt_qplib_cq *qplcq; + struct bnxt_re_qp *qp; + struct bnxt_re_cq *cq; + int rc = 0; + u8 event; + + if (!obj) + return rc; /* QP was already dead, still return success */ + + event = affi_async->event; + switch (event) { + case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: + qplqp = obj; + qp = container_of(qplqp, struct bnxt_re_qp, qplib_qp); + rc = bnxt_re_handle_qp_async_event(affi_async, qp); + break; + case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION: + qplcq = obj; + cq = container_of(qplcq, struct bnxt_re_cq, qplib_cq); + rc = bnxt_re_handle_cq_async_error(affi_async, cq); + break; + default: + rc = -EINVAL; + } + + return rc; +} + +static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, + void *aeqe, void *obj) +{ + struct creq_func_event *unaffi_async; + struct creq_qp_event *affi_async; + u8 type; + int rc; + + type = ((struct creq_base *)aeqe)->type; + if (type == CREQ_BASE_TYPE_FUNC_EVENT) { + unaffi_async = aeqe; + rc = bnxt_re_handle_unaffi_async_event(unaffi_async); + } else { + affi_async = aeqe; + rc = bnxt_re_handle_affi_async_event(affi_async, obj); + } + + return rc; +} + +static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_srq *handle, u8 event) +{ + struct bnxt_re_srq *srq = to_bnxt_re(handle, struct bnxt_re_srq, + qplib_srq); + struct ib_event ib_event; + + if (srq == NULL) { + pr_err("%s: SRQ is NULL, SRQN not handled", + ROCE_DRV_MODULE_NAME); + return -EINVAL; + } + ib_event.device = &srq->rdev->ibdev; + ib_event.element.srq = &srq->ibsrq; + if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT) + ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED; + else + ib_event.event = IB_EVENT_SRQ_ERR; + + if (srq->ibsrq.event_handler) { + /* Lock event_handler? */ + (*srq->ibsrq.event_handler)(&ib_event, + srq->ibsrq.srq_context); + } + return 0; +} + +static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_cq *handle) +{ + struct bnxt_re_cq *cq = to_bnxt_re(handle, struct bnxt_re_cq, + qplib_cq); + u32 *cq_ptr; + + if (cq == NULL) { + pr_err("%s: CQ is NULL, CQN not handled", + ROCE_DRV_MODULE_NAME); + return -EINVAL; + } + /* CQ already in destroy path. Do not handle any more events */ + if (handle->destroyed || !atomic_read(&cq->ibcq.usecnt)) { + if (!handle->destroyed) + dev_dbg(NULL, "%s: CQ being destroyed, CQN not handled", + ROCE_DRV_MODULE_NAME); + return 0; + } + + if (cq->ibcq.comp_handler) { + if (cq->uctx_cq_page) { + cq_ptr = (u32 *)cq->uctx_cq_page; + *cq_ptr = cq->qplib_cq.toggle; + } + /* Lock comp_handler? */ + (*cq->ibcq.comp_handler)(&cq->ibcq, cq->ibcq.cq_context); + } + + return 0; +} + +struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev) +{ + int min, indx; + + mutex_lock(&rdev->nqr.load_lock); + for (indx = 0, min = 0; indx < (rdev->nqr.num_msix - 1); indx++) { + if (rdev->nqr.nq[min].load > rdev->nqr.nq[indx].load) + min = indx; + } + rdev->nqr.nq[min].load++; + mutex_unlock(&rdev->nqr.load_lock); + + return &rdev->nqr.nq[min]; +} + +void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq) +{ + mutex_lock(&rdev->nqr.load_lock); + nq->load--; + mutex_unlock(&rdev->nqr.load_lock); +} + +static bool bnxt_re_check_min_attr(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_dev_attr *attr; + bool rc = true; + + attr = rdev->dev_attr; + + if (!attr->max_cq || !attr->max_qp || + !attr->max_sgid || !attr->max_mr) { + dev_err(rdev_to_dev(rdev),"Insufficient RoCE resources"); + dev_dbg(rdev_to_dev(rdev), + "max_cq = %d, max_qp = %d, max_dpi = %d, max_sgid = %d, max_mr = %d", + attr->max_cq, attr->max_qp, attr->max_dpi, + attr->max_sgid, attr->max_mr); + rc = false; + } + return rc; +} + +static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, + u8 port_num, enum ib_event_type event) +{ + struct ib_event ib_event; + + ib_event.device = ibdev; + if (qp) { + ib_event.element.qp = qp; + ib_event.event = event; + if (qp->event_handler) + qp->event_handler(&ib_event, qp->qp_context); + } else { + ib_event.element.port_num = port_num; + ib_event.event = event; + ib_dispatch_event(&ib_event); + } + + dev_dbg(rdev_to_dev(to_bnxt_re_dev(ibdev, ibdev)), + "ibdev %p Event 0x%x port_num 0x%x", ibdev, event, port_num); +} + +static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp) +{ + if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) + return (qp->ib_qp.qp_type == IB_QPT_GSI) || + (qp == rdev->gsi_ctx.gsi_sqp); + else + return (qp->ib_qp.qp_type == IB_QPT_GSI); +} + +static void bnxt_re_stop_all_nonqp1_nonshadow_qps(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_qp *qpl_qp; + bool dev_detached = false; + struct ib_qp_attr qp_attr; + int num_qps_stopped = 0; + int mask = IB_QP_STATE; + struct bnxt_re_qp *qp; + unsigned long flags; + + if (!rdev) + return; + +restart: + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + dev_detached = true; + + qp_attr.qp_state = IB_QPS_ERR; + mutex_lock(&rdev->qp_lock); + list_for_each_entry(qp, &rdev->qp_list, list) { + qpl_qp = &qp->qplib_qp; + if (dev_detached || !bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { + if (qpl_qp->state != + CMDQ_MODIFY_QP_NEW_STATE_RESET && + qpl_qp->state != + CMDQ_MODIFY_QP_NEW_STATE_ERR) { + if (dev_detached) { + /* + * Cant actually send the command down, + * marking the state for bookkeeping + */ + qpl_qp->state = + CMDQ_MODIFY_QP_NEW_STATE_ERR; + qpl_qp->cur_qp_state = qpl_qp->state; + if (!qpl_qp->is_user) { + /* Add to flush list */ + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_add_flush_qp(qpl_qp); + bnxt_re_unlock_cqs(qp, flags); + } + } else { + num_qps_stopped++; + bnxt_re_modify_qp(&qp->ib_qp, + &qp_attr, mask, + NULL); + } + + bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, + 1, IB_EVENT_QP_FATAL); + /* + * 1. Release qp_lock after a budget to unblock other verb + * requests (like qp_destroy) from stack. + * 2. Traverse through the qp_list freshly as addition / deletion + * might have happened since qp_lock is getting released here. + */ + if (num_qps_stopped % BNXT_RE_STOP_QPS_BUDGET == 0) { + mutex_unlock(&rdev->qp_lock); + goto restart; + } + } + } + } + + mutex_unlock(&rdev->qp_lock); +} + +static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; + struct bnxt_qplib_gid gid; + u16 gid_idx, index; + int rc = 0; + + if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) + return 0; + + if (sgid_tbl == NULL) { + dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated"); + return -EINVAL; + } + + for (index = 0; index < sgid_tbl->active; index++) { + gid_idx = sgid_tbl->hw_id[index]; + + if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero))) + continue; + /* Need to modify the VLAN enable setting of non VLAN GID only + * as setting is done for VLAN GID while adding GID + * + * If disable_prio_vlan_tx is enable, then we'll need to remove the + * vlan entry from the sgid_tbl. + */ + if (sgid_tbl->vlan[index] == true) + continue; + + memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid)); + + rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx, + rdev->dev_addr); + } + + return rc; +} + +static void bnxt_re_clear_cc(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; + + if (_is_chip_p7(rdev->chip_ctx)) { + cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; + } else { + cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); + + if (!is_qport_service_type_supported(rdev)) + cc_param->mask |= + (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); + } + + cc_param->cur_mask = cc_param->mask; + + if (bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param)) + dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); +} + +static int bnxt_re_setup_cc(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; + int rc; + + if (_is_chip_p7(rdev->chip_ctx)) { + cc_param->enable = 0x0; + cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; + } else { + cc_param->enable = 0x1; + cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); + + if (!is_qport_service_type_supported(rdev)) + cc_param->mask |= + (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); + } + + cc_param->cur_mask = cc_param->mask; + + rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); + return rc; + } + /* Reset the programming mask */ + cc_param->mask = 0; + if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { + cc_param->qp1_tos_dscp = cc_param->tos_dscp; + rc = bnxt_re_update_qp1_tos_dscp(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1:%d", + __func__, rc); + goto clear; + } + } + return 0; + +clear: + bnxt_re_clear_cc(rdev); + return rc; +} + +int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev, + struct bnxt_re_dscp2pri *d2p, u16 *count, + u16 target_id) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_queue_dscp2pri_qcfg_input req; + struct hwrm_queue_dscp2pri_qcfg_output resp; + struct bnxt_re_dscp2pri *dscp2pri; + struct bnxt_fw_msg fw_msg; + u16 in_count = *count; + dma_addr_t dma_handle; + int rc = 0, i; + u16 data_len; + u8 *kmem; + + data_len = *count * sizeof(*dscp2pri); + memset(&fw_msg, 0, sizeof(fw_msg)); + memset(&req, 0, sizeof(req)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_QUEUE_DSCP2PRI_QCFG, -1, target_id); + req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; + + kmem = dma_zalloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, + GFP_KERNEL); + if (!kmem) { + dev_err(rdev_to_dev(rdev), + "dma_zalloc_coherent failure, length = %u\n", + (unsigned)data_len); + return -ENOMEM; + } + req.dest_data_addr = cpu_to_le64(dma_handle); + req.dest_data_buffer_size = cpu_to_le16(data_len); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) + goto out; + + /* Upload the DSCP-MASK-PRI tuple(s) */ + dscp2pri = (struct bnxt_re_dscp2pri *)kmem; + for (i = 0; i < le16_to_cpu(resp.entry_cnt) && i < in_count; i++) { + d2p[i].dscp = dscp2pri->dscp; + d2p[i].mask = dscp2pri->mask; + d2p[i].pri = dscp2pri->pri; + dscp2pri++; + } + *count = le16_to_cpu(resp.entry_cnt); +out: + dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); + return rc; +} + +int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev) +{ + /* Remove the VLAN from the GID entry */ + if (rdev->cc_param.disable_prio_vlan_tx) + rdev->qplib_res.prio = false; + else + rdev->qplib_res.prio = true; + + return bnxt_re_update_gid(rdev); +} + +int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev, + struct bnxt_re_dscp2pri *d2p, u16 count, + u16 target_id) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_queue_dscp2pri_cfg_input req; + struct hwrm_queue_dscp2pri_cfg_output resp; + struct bnxt_fw_msg fw_msg; + struct bnxt_re_dscp2pri *dscp2pri; + int i, rc, data_len = 3 * 256; + dma_addr_t dma_handle; + u8 *kmem; + + memset(&req, 0, sizeof(req)); + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_QUEUE_DSCP2PRI_CFG, -1, target_id); + req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; + + kmem = dma_alloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, + GFP_KERNEL); + if (!kmem) { + dev_err(rdev_to_dev(rdev), + "dma_alloc_coherent failure, length = %u\n", + (unsigned)data_len); + return -ENOMEM; + } + req.src_data_addr = cpu_to_le64(dma_handle); + + /* Download the DSCP-MASK-PRI tuple(s) */ + dscp2pri = (struct bnxt_re_dscp2pri *)kmem; + for (i = 0; i < count; i++) { + dscp2pri->dscp = d2p[i].dscp; + dscp2pri->mask = d2p[i].mask; + dscp2pri->pri = d2p[i].pri; + dscp2pri++; + } + + req.entry_cnt = cpu_to_le16(count); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); + return rc; +} + +int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev, + struct bnxt_re_tc_rec *tc_rec, u16 tid) +{ + u8 max_tc, tc, *qptr, *type_ptr0, *type_ptr1; + struct hwrm_queue_qportcfg_output resp = {0}; + struct hwrm_queue_qportcfg_input req = {0}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + bool def_init = false; + u8 *tmp_type; + u8 cos_id; + int rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_QPORTCFG, + -1, tid); + req.port_id = (tid == 0xFFFF) ? en_dev->pf_port_id : 1; + if (BNXT_EN_ASYM_Q(en_dev)) + req.flags = htole32(HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX); + + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) + return rc; + + if (!resp.max_configurable_queues) + return -EINVAL; + + max_tc = resp.max_configurable_queues; + tc_rec->max_tc = max_tc; + + if (resp.queue_cfg_info & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_USE_PROFILE_TYPE) + tc_rec->serv_type_enabled = true; + + qptr = &resp.queue_id0; + type_ptr0 = &resp.queue_id0_service_profile_type; + type_ptr1 = &resp.queue_id1_service_profile_type; + for (tc = 0; tc < max_tc; tc++) { + tmp_type = tc ? type_ptr1 + (tc - 1) : type_ptr0; + + cos_id = *qptr++; + /* RoCE CoS queue is the first cos queue. + * For MP12 and MP17 order is 405 and 141015. + */ + if (is_bnxt_roce_queue(rdev, *qptr, *tmp_type)) { + tc_rec->cos_id_roce = cos_id; + tc_rec->tc_roce = tc; + } else if (is_bnxt_cnp_queue(rdev, *qptr, *tmp_type)) { + tc_rec->cos_id_cnp = cos_id; + tc_rec->tc_cnp = tc; + } else if (!def_init) { + def_init = true; + tc_rec->tc_def = tc; + tc_rec->cos_id_def = cos_id; + } + qptr++; + } + + return rc; +} + +int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id, + struct bnxt_re_cos2bw_cfg *cfg) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_queue_cos2bw_qcfg_output resp; + struct hwrm_queue_cos2bw_qcfg_input req = {0}; + struct bnxt_fw_msg fw_msg; + int rc, indx; + void *data; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_QUEUE_COS2BW_QCFG, -1, target_id); + req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; + + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) + return rc; + data = &resp.queue_id0 + offsetof(struct bnxt_re_cos2bw_cfg, + queue_id); + for (indx = 0; indx < 8; indx++, data += (sizeof(cfg->cfg))) { + memcpy(&cfg->cfg, data, sizeof(cfg->cfg)); + if (indx == 0) + cfg->queue_id = resp.queue_id0; + cfg++; + } + + return rc; +} + +int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id, + struct bnxt_re_cos2bw_cfg *cfg) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_queue_cos2bw_cfg_input req = {0}; + struct hwrm_queue_cos2bw_cfg_output resp = {0}; + struct bnxt_fw_msg fw_msg; + void *data; + int indx; + int rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_QUEUE_COS2BW_CFG, -1, target_id); + req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; + + /* Chimp wants enable bit to retain previous + * config done by L2 driver + */ + for (indx = 0; indx < 8; indx++) { + if (cfg[indx].queue_id < 40) { + req.enables |= cpu_to_le32( + HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID0_VALID << + indx); + } + + data = (char *)&req.unused_0 + indx * (sizeof(*cfg) - 4); + memcpy(data, &cfg[indx].queue_id, sizeof(*cfg) - 4); + if (indx == 0) { + req.queue_id0 = cfg[0].queue_id; + req.unused_0 = 0; + } + } + + memset(&resp, 0, sizeof(resp)); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + return rc; +} + +int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev, + struct bnxt_qplib_query_fn_info *fn_info, + u32 *pf_mask, u32 *first_pf) +{ + struct hwrm_func_host_pf_ids_query_output resp = {0}; + struct hwrm_func_host_pf_ids_query_input req; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + int rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + memset(&req, 0, sizeof(req)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_HOST_PF_IDS_QUERY, -1, -1); + /* To query the info from the host EPs */ + switch (fn_info->host) { + case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_SOC: + case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0: + case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_1: + case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_2: + case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_3: + req.host = fn_info->host; + break; + default: + req.host = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0; + break; + } + + req.filter = fn_info->filter; + if (req.filter > HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ROCE) + req.filter = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ALL; + + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + + + *first_pf = le16_to_cpu(resp.first_pf_id); + *pf_mask = le16_to_cpu(resp.pf_ordinal_mask); + + return rc; +} + +static void bnxt_re_put_stats_ctx(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_ctx *hctx; + struct bnxt_qplib_res *res; + u16 tid = 0xffff; + + res = &rdev->qplib_res; + hctx = res->hctx; + + if (test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags)) { + bnxt_re_net_stats_ctx_free(rdev, hctx->stats.fw_id, tid); + bnxt_qplib_free_stat_mem(res, &hctx->stats); + } +} + +static void bnxt_re_put_stats2_ctx(struct bnxt_re_dev *rdev) +{ + test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX2_ALLOC, &rdev->flags); +} + +static int bnxt_re_get_stats_ctx(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_ctx *hctx; + struct bnxt_qplib_res *res; + u16 tid = 0xffff; + int rc; + + res = &rdev->qplib_res; + hctx = res->hctx; + + rc = bnxt_qplib_alloc_stat_mem(res->pdev, rdev->chip_ctx, &hctx->stats); + if (rc) + return -ENOMEM; + rc = bnxt_re_net_stats_ctx_alloc(rdev, tid); + if (rc) + goto free_stat_mem; + set_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags); + + return 0; + +free_stat_mem: + bnxt_qplib_free_stat_mem(res, &hctx->stats); + + return rc; +} + +static int bnxt_re_update_dev_attr(struct bnxt_re_dev *rdev) +{ + int rc; + + rc = bnxt_qplib_get_dev_attr(&rdev->rcfw); + if (rc) + return rc; + if (!bnxt_re_check_min_attr(rdev)) + return -EINVAL; + return 0; +} + +static void bnxt_re_free_tbls(struct bnxt_re_dev *rdev) +{ + bnxt_qplib_clear_tbls(&rdev->qplib_res); + bnxt_qplib_free_tbls(&rdev->qplib_res); +} + +static int bnxt_re_alloc_init_tbls(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx; + u8 pppp_factor = 0; + int rc; + + /* + * TODO: Need a better mechanism for spreading of the + * 512 extended PPP pages. For now, spreading it + * based on port_count + */ + if (_is_chip_p7(chip_ctx) && chip_ctx->modes.db_push) + pppp_factor = rdev->en_dev->port_count; + rc = bnxt_qplib_alloc_tbls(&rdev->qplib_res, pppp_factor); + if (rc) + return rc; + bnxt_qplib_init_tbls(&rdev->qplib_res); + set_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags); + + return 0; +} + +static void bnxt_re_clean_nqs(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_nq *nq; + int i; + + if (!rdev->nqr.max_init) + return; + + for (i = (rdev->nqr.max_init - 1) ; i >= 0; i--) { + nq = &rdev->nqr.nq[i]; + bnxt_qplib_disable_nq(nq); + bnxt_re_net_ring_free(rdev, nq->ring_id); + bnxt_qplib_free_nq_mem(nq); + } + rdev->nqr.max_init = 0; +} + +static int bnxt_re_setup_nqs(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_ring_attr rattr = {}; + struct bnxt_qplib_nq *nq; + int rc, i; + int depth; + u32 offt; + u16 vec; + + mutex_init(&rdev->nqr.load_lock); + /* + * TODO: Optimize the depth based on the + * number of NQs. + */ + depth = BNXT_QPLIB_NQE_MAX_CNT; + for (i = 0; i < rdev->nqr.num_msix - 1; i++) { + nq = &rdev->nqr.nq[i]; + vec = rdev->nqr.msix_entries[i + 1].vector; + offt = rdev->nqr.msix_entries[i + 1].db_offset; + nq->hwq.max_elements = depth; + rc = bnxt_qplib_alloc_nq_mem(&rdev->qplib_res, nq); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to get mem for NQ %d, rc = 0x%x", + i, rc); + goto fail_mem; + } + + rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr; + rattr.pages = nq->hwq.pbl[rdev->nqr.nq[i].hwq.level].pg_count; + rattr.type = bnxt_re_get_rtype(rdev); + rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; + rattr.depth = nq->hwq.max_elements - 1; + rattr.lrid = rdev->nqr.msix_entries[i + 1].ring_idx; + + /* Set DBR pacing capability on the first NQ ring only */ + if (!i && bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) + rattr.flags = HWRM_RING_ALLOC_INPUT_FLAGS_NQ_DBR_PACING; + else + rattr.flags = 0; + + rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id); + if (rc) { + nq->ring_id = 0xffff; /* Invalid ring-id */ + dev_err(rdev_to_dev(rdev), + "Failed to get fw id for NQ %d, rc = 0x%x", + i, rc); + goto fail_ring; + } + + rc = bnxt_qplib_enable_nq(nq, i, vec, offt, + &bnxt_re_cqn_handler, + &bnxt_re_srqn_handler); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to enable NQ %d, rc = 0x%x", i, rc); + goto fail_en; + } + } + + rdev->nqr.max_init = i; + return 0; +fail_en: + /* *nq was i'th nq */ + bnxt_re_net_ring_free(rdev, nq->ring_id); +fail_ring: + bnxt_qplib_free_nq_mem(nq); +fail_mem: + rdev->nqr.max_init = i; + return rc; +} + +static void bnxt_re_sysfs_destroy_file(struct bnxt_re_dev *rdev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) + device_remove_file(&rdev->ibdev.dev, bnxt_re_attributes[i]); +} + +static int bnxt_re_sysfs_create_file(struct bnxt_re_dev *rdev) +{ + int i, j, rc = 0; + + for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { + rc = device_create_file(&rdev->ibdev.dev, + bnxt_re_attributes[i]); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to create IB sysfs with rc = 0x%x", rc); + /* Must clean up all created device files */ + for (j = 0; j < i; j++) + device_remove_file(&rdev->ibdev.dev, + bnxt_re_attributes[j]); + clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); + ib_unregister_device(&rdev->ibdev); + return 1; + } + } + return 0; +} + +/* worker thread for polling periodic events. Now used for QoS programming*/ +static void bnxt_re_worker(struct work_struct *work) +{ + struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, + worker.work); + int rc; + + /* QoS is in 30s cadence for PFs*/ + if (!rdev->is_virtfn && !rdev->worker_30s--) + rdev->worker_30s = 30; + /* Use trylock for bnxt_re_dev_lock as this can be + * held for long time by debugfs show path while issuing + * HWRMS. If the debugfs name update is not done in this + * iteration, the driver will check for the same in the + * next schedule of the worker i.e after 1 sec. + */ + if (mutex_trylock(&bnxt_re_dev_lock)) + mutex_unlock(&bnxt_re_dev_lock); + + if (!rdev->stats.stats_query_sec) + goto resched; + + if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags) && + (rdev->is_virtfn || + !_is_ext_stats_supported(rdev->dev_attr->dev_cap_flags))) { + if (!(rdev->stats.stats_query_counter++ % + rdev->stats.stats_query_sec)) { + rc = bnxt_re_get_qos_stats(rdev); + if (rc && rc != -ENOMEM) + clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, + &rdev->flags); + } + } + +resched: + schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); +} + +static int bnxt_re_alloc_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) +{ + if (!(rdev->dbr_drop_recov || rdev->dbr_pacing)) + return 0; + + rdev->dbr_sw_stats = kzalloc(sizeof(*rdev->dbr_sw_stats), GFP_KERNEL); + if (!rdev->dbr_sw_stats) + return -ENOMEM; + + return 0; +} + +static void bnxt_re_free_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) +{ + kfree(rdev->dbr_sw_stats); + rdev->dbr_sw_stats = NULL; +} + +static int bnxt_re_initialize_dbr_drop_recov(struct bnxt_re_dev *rdev) +{ + rdev->dbr_drop_recov_wq = + create_singlethread_workqueue("bnxt_re_dbr_drop_recov"); + if (!rdev->dbr_drop_recov_wq) { + dev_err(rdev_to_dev(rdev), "DBR Drop Revov wq alloc failed!"); + return -EINVAL; + } + rdev->dbr_drop_recov = true; + + /* Enable configfs setting dbr_drop_recov by default*/ + rdev->user_dbr_drop_recov = true; + + rdev->user_dbr_drop_recov_timeout = BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT; + return 0; +} + +static void bnxt_re_deinitialize_dbr_drop_recov(struct bnxt_re_dev *rdev) +{ + if (rdev->dbr_drop_recov_wq) { + flush_workqueue(rdev->dbr_drop_recov_wq); + destroy_workqueue(rdev->dbr_drop_recov_wq); + rdev->dbr_drop_recov_wq = NULL; + } + rdev->dbr_drop_recov = false; +} + +static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev) +{ + int rc; + + /* Allocate a page for app use */ + rdev->dbr_page = (void *)__get_free_page(GFP_KERNEL); + if (!rdev->dbr_page) { + dev_err(rdev_to_dev(rdev), "DBR page allocation failed!"); + return -ENOMEM; + } + memset((u8 *)rdev->dbr_page, 0, PAGE_SIZE); + rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->dbr_page; + rc = bnxt_re_hwrm_dbr_pacing_qcfg(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to query dbr pacing config %d\n", rc); + goto fail; + } + /* Create a work queue for scheduling dbq event */ + rdev->dbq_wq = create_singlethread_workqueue("bnxt_re_dbq"); + if (!rdev->dbq_wq) { + dev_err(rdev_to_dev(rdev), "DBQ wq alloc failed!"); + rc = -ENOMEM; + goto fail; + } + /* MAP grc window 2 for reading db fifo depth */ + writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4, 0, + rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK); + rdev->dbr_db_fifo_reg_off = + (rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) + + 0x2000; + rdev->qplib_res.pacing_data->grc_reg_offset = rdev->dbr_db_fifo_reg_off; + + rdev->dbr_bar_addr = + pci_resource_start(rdev->qplib_res.pdev, 0) + + rdev->dbr_db_fifo_reg_off; + + /* Percentage of DB FIFO */ + rdev->dbq_watermark = BNXT_RE_PACING_DBQ_THRESHOLD; + rdev->pacing_en_int_th = BNXT_RE_PACING_EN_INT_THRESHOLD; + rdev->pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD; + rdev->dbq_pacing_time = BNXT_RE_DBR_INT_TIME; + rdev->dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION; + rdev->do_pacing_save = rdev->dbr_def_do_pacing; + bnxt_re_set_default_pacing_data(rdev); + dev_dbg(rdev_to_dev(rdev), "Initialized db pacing\n"); + + return 0; +fail: + free_page((u64)rdev->dbr_page); + rdev->dbr_page = NULL; + return rc; +} + +static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev) +{ + if (rdev->dbq_wq) + flush_workqueue(rdev->dbq_wq); + + cancel_work_sync(&rdev->dbq_fifo_check_work); + cancel_delayed_work_sync(&rdev->dbq_pacing_work); + + if (rdev->dbq_wq) { + destroy_workqueue(rdev->dbq_wq); + rdev->dbq_wq = NULL; + } + + if (rdev->dbr_page) + free_page((u64)rdev->dbr_page); + rdev->dbr_page = NULL; + rdev->dbr_pacing = false; +} + +/* enable_dbr_pacing needs to be done only for older FWs + * where host selects primary function. ie. pacing_ext + * flags is not set. + */ +int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_nq *nq; + + nq = &rdev->nqr.nq[0]; + rdev->dbq_nq_id = nq->ring_id; + + if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && + bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { + if (bnxt_re_hwrm_dbr_pacing_cfg(rdev, true)) { + dev_err(rdev_to_dev(rdev), + "Failed to set dbr pacing config\n"); + return -EIO; + } + /* MAP grc window 8 for ARMing the NQ DBQ */ + writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28 , 0, + rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); + rdev->dbr_aeq_arm_reg_off = + (rdev->chip_ctx->dbr_aeq_arm_reg & + BNXT_GRC_OFFSET_MASK) + 0x8000; + writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off , 0, 1); + } + + return 0; +} + +/* disable_dbr_pacing needs to be done only for older FWs + * where host selects primary function. ie. pacing_ext + * flags is not set. + */ + +int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev) +{ + int rc = 0; + + if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && + bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) + rc = bnxt_re_hwrm_dbr_pacing_cfg(rdev, false); + + return rc; +} + +static void bnxt_re_ib_uninit(struct bnxt_re_dev *rdev) +{ + if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { + bnxt_re_sysfs_destroy_file(rdev); + /* Cleanup ib dev */ + ib_unregister_device(&rdev->ibdev); + clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); + return; + } +} + +static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type) +{ + struct bnxt_qplib_dpi *kdpi; + int rc, wait_count = BNXT_RE_RES_FREE_WAIT_COUNT; + + bnxt_re_net_unregister_async_event(rdev); + + bnxt_re_put_stats2_ctx(rdev); + if (test_and_clear_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, + &rdev->flags)) { + /* did the caller hold the lock? */ + mutex_lock(&bnxt_re_dev_lock); + list_del_rcu(&rdev->list); + mutex_unlock(&bnxt_re_dev_lock); + } + + bnxt_re_uninit_resolve_wq(rdev); + bnxt_re_uninit_dcb_wq(rdev); + bnxt_re_uninit_aer_wq(rdev); + + bnxt_re_deinitialize_dbr_drop_recov(rdev); + + if (bnxt_qplib_dbr_pacing_en(rdev->chip_ctx)) + (void)bnxt_re_disable_dbr_pacing(rdev); + + if (test_and_clear_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags)) { + cancel_delayed_work_sync(&rdev->worker); + } + + /* Wait for ULPs to release references */ + while (atomic_read(&rdev->stats.rsors.cq_count) && --wait_count) + usleep_range(500, 1000); + if (!wait_count) + dev_err(rdev_to_dev(rdev), + "CQ resources not freed by stack, count = 0x%x", + atomic_read(&rdev->stats.rsors.cq_count)); + + kdpi = &rdev->dpi_privileged; + if (kdpi->umdbr) { /* kernel DPI was allocated with success */ + (void)bnxt_qplib_dealloc_dpi(&rdev->qplib_res, kdpi); + /* + * Driver just need to know no command had failed + * during driver load sequence and below command is + * required indeed. Piggybacking dpi allocation status. + */ + } + + /* Protect the device uninitialization and start_irq/stop_irq L2 + * callbacks with rtnl lock to avoid race condition between these calls + */ + rtnl_lock(); + if (test_and_clear_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags)) + bnxt_re_clean_nqs(rdev); + rtnl_unlock(); + + if (test_and_clear_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags)) + bnxt_re_free_tbls(rdev); + if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags)) { + rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); + if (rc) + dev_warn(rdev_to_dev(rdev), + "Failed to deinitialize fw, rc = 0x%x", rc); + } + + bnxt_re_put_stats_ctx(rdev); + + if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags)) + bnxt_qplib_free_hwctx(&rdev->qplib_res); + + rtnl_lock(); + if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) + bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); + + if (rdev->dbr_pacing) + bnxt_re_deinitialize_dbr_pacing(rdev); + + bnxt_re_free_dbr_sw_stats_mem(rdev); + + if (test_and_clear_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags)) + bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id); + + if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) + bnxt_qplib_free_rcfw_channel(&rdev->qplib_res); + + if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) + bnxt_re_free_msix(rdev); + rtnl_unlock(); + + bnxt_re_destroy_chip_ctx(rdev); + + if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { + if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, + &rdev->flags)) + bnxt_re_unregister_netdev(rdev); + } +} + +static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type, u8 wqe_mode) +{ + struct bnxt_re_ring_attr rattr = {}; + struct bnxt_qplib_creq_ctx *creq; + int vec, offset; + int rc = 0; + + if (op_type != BNXT_RE_POST_RECOVERY_INIT) { + /* Registered a new RoCE device instance to netdev */ + rc = bnxt_re_register_netdev(rdev); + if (rc) + return -EINVAL; + } + set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); + + rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to get chip context rc 0x%x", rc); + bnxt_re_unregister_netdev(rdev); + clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); + rc = -EINVAL; + return rc; + } + + /* Protect the device initialization and start_irq/stop_irq L2 callbacks + * with rtnl lock to avoid race condition between these calls + */ + rtnl_lock(); + rc = bnxt_re_request_msix(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Requesting MSI-X vectors failed with rc = 0x%x", rc); + rc = -EINVAL; + goto release_rtnl; + } + set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); + + /* Establish RCFW Communication Channel to initialize the context + memory for the function and all child VFs */ + rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to alloc mem for rcfw, rc = %#x\n", rc); + goto release_rtnl; + } + set_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags); + + creq = &rdev->rcfw.creq; + rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr; + rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count; + rattr.type = bnxt_re_get_rtype(rdev); + rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; + rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1; + rattr.lrid = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].ring_idx; + rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id); + if (rc) { + creq->ring_id = 0xffff; + dev_err(rdev_to_dev(rdev), + "Failed to allocate CREQ fw id with rc = 0x%x", rc); + goto release_rtnl; + } + + if (!rdev->chip_ctx) + goto release_rtnl; + /* Program the NQ ID for DBQ notification */ + if (rdev->chip_ctx->modes.dbr_pacing_v0 || + bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) || + bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { + rc = bnxt_re_initialize_dbr_pacing(rdev); + if (!rc) + rdev->dbr_pacing = true; + else + rdev->dbr_pacing = false; + dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n", + __func__, rc); + } + + vec = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].vector; + offset = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].db_offset; + rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw, vec, offset, + &bnxt_re_aeq_handler); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to enable RCFW channel with rc = 0x%x", rc); + goto release_rtnl; + } + set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); + + rc = bnxt_re_update_dev_attr(rdev); + if (rc) + goto release_rtnl; + bnxt_re_set_resource_limits(rdev); + if (!rdev->is_virtfn && !_is_chip_gen_p5_p7(rdev->chip_ctx)) { + rc = bnxt_qplib_alloc_hwctx(&rdev->qplib_res); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to alloc hw contexts, rc = 0x%x", rc); + goto release_rtnl; + } + set_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags); + } + + rc = bnxt_re_get_stats_ctx(rdev); + if (rc) + goto release_rtnl; + + rc = bnxt_qplib_init_rcfw(&rdev->rcfw, rdev->is_virtfn); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to initialize fw with rc = 0x%x", rc); + goto release_rtnl; + } + set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags); + + /* Based resource count on the 'new' device caps */ + rc = bnxt_re_update_dev_attr(rdev); + if (rc) + goto release_rtnl; + rc = bnxt_re_alloc_init_tbls(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), "tbls alloc-init failed rc = %#x", + rc); + goto release_rtnl; + } + rc = bnxt_re_setup_nqs(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), "NQs alloc-init failed rc = %#x\n", + rc); + if (rdev->nqr.max_init == 0) + goto release_rtnl; + + dev_warn(rdev_to_dev(rdev), + "expected nqs %d available nqs %d\n", + rdev->nqr.num_msix, rdev->nqr.max_init); + } + set_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags); + rtnl_unlock(); + + rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &rdev->dpi_privileged, + rdev, BNXT_QPLIB_DPI_TYPE_KERNEL); + if (rc) + goto fail; + + if (rdev->dbr_pacing) + bnxt_re_enable_dbr_pacing(rdev); + + if (rdev->chip_ctx->modes.dbr_drop_recov) + bnxt_re_initialize_dbr_drop_recov(rdev); + + rc = bnxt_re_alloc_dbr_sw_stats_mem(rdev); + if (rc) + goto fail; + + /* This block of code is needed for error recovery support */ + if (!rdev->is_virtfn) { + struct bnxt_re_tc_rec *tc_rec; + + tc_rec = &rdev->tc_rec[0]; + rc = bnxt_re_query_hwrm_qportcfg(rdev, tc_rec, 0xFFFF); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to query port config rc:%d", rc); + return rc; + } + + /* Query f/w defaults of CC params */ + rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, &rdev->cc_param); + if (rc) + dev_warn(rdev_to_dev(rdev), + "Failed to query CC defaults\n"); + if (1) { + rdev->num_vfs = pci_num_vf(rdev->en_dev->pdev); + if (rdev->num_vfs) { + bnxt_re_set_resource_limits(rdev); + bnxt_qplib_set_func_resources(&rdev->qplib_res); + } + } + } + INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); + set_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags); + schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); + + bnxt_re_init_dcb_wq(rdev); + bnxt_re_init_aer_wq(rdev); + bnxt_re_init_resolve_wq(rdev); + mutex_lock(&bnxt_re_dev_lock); + list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); + /* Added to the list, not in progress anymore */ + gadd_dev_inprogress--; + set_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, &rdev->flags); + mutex_unlock(&bnxt_re_dev_lock); + + + return rc; +release_rtnl: + rtnl_unlock(); +fail: + bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); + + return rc; +} + +static int bnxt_re_ib_init(struct bnxt_re_dev *rdev) +{ + int rc = 0; + + rc = bnxt_re_register_ib(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Register IB failed with rc = 0x%x", rc); + goto fail; + } + if (bnxt_re_sysfs_create_file(rdev)) { + bnxt_re_stopqps_and_ib_uninit(rdev); + goto fail; + } + + set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); + set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags); + set_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); + + return rc; +fail: + bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); + return rc; +} + +/* wrapper for ib_init funcs */ +int _bnxt_re_ib_init(struct bnxt_re_dev *rdev) +{ + return bnxt_re_ib_init(rdev); +} + +/* wrapper for aux init funcs */ +int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev) +{ + bnxt_re_ib_init_2(rdev); + return 0; /* add return for future proof */ +} + +static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) +{ + bnxt_re_dev_dealloc(rdev); +} + + +static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct ifnet *netdev, + struct bnxt_en_dev *en_dev) +{ + struct ifnet *realdev = NULL; + + realdev = netdev; + if (realdev) + dev_dbg(NULL, "%s: realdev = %p netdev = %p\n", __func__, + realdev, netdev); + /* + * Note: + * The first argument to bnxt_re_dev_alloc() is 'netdev' and + * not 'realdev', since in the case of bonding we want to + * register the bonded virtual netdev (master) to the ib stack. + * And 'en_dev' (for L2/PCI communication) is the first slave + * device (PF0 on the card). + * In the case of a regular netdev, both netdev and the en_dev + * correspond to the same device. + */ + *rdev = bnxt_re_dev_alloc(netdev, en_dev); + if (!*rdev) { + pr_err("%s: netdev %p not handled", + ROCE_DRV_MODULE_NAME, netdev); + return -ENOMEM; + } + bnxt_re_hold(*rdev); + + return 0; +} + +void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev) +{ + rdev->espeed = rdev->en_dev->espeed; + return; +} + +void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev) +{ + dev_dbg(rdev_to_dev(rdev), "%s: Stopping QPs, IB uninit on rdev: %p\n", + __func__, rdev); + bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); + bnxt_re_ib_uninit(rdev); +} + +void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type, + struct auxiliary_device *aux_dev) +{ + struct bnxt_re_en_dev_info *en_info; + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_rcfw *rcfw; + + rcfw = &rdev->rcfw; + cmdq = &rcfw->cmdq; + if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) + set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); + + dev_dbg(rdev_to_dev(rdev), "%s: Removing rdev: %p\n", __func__, rdev); + bnxt_re_dev_uninit(rdev, op_type); + en_info = auxiliary_get_drvdata(aux_dev); + if (en_info) { + rtnl_lock(); + en_info->rdev = NULL; + rtnl_unlock(); + if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { + clear_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags); + clear_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags); + clear_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); + } + } + bnxt_re_dev_unreg(rdev); +} + +int bnxt_re_add_device(struct bnxt_re_dev **rdev, + struct ifnet *netdev, + u8 qp_mode, u8 op_type, u8 wqe_mode, + u32 num_msix_requested, + struct auxiliary_device *aux_dev) +{ + struct bnxt_re_en_dev_info *en_info; + struct bnxt_en_dev *en_dev; + int rc = 0; + + en_info = auxiliary_get_drvdata(aux_dev); + en_dev = en_info->en_dev; + + mutex_lock(&bnxt_re_dev_lock); + /* Check if driver already in mod exit and aux_dev is valid */ + if (gmod_exit || !aux_dev) { + mutex_unlock(&bnxt_re_dev_lock); + return -ENODEV; + } + /* Add device in progress */ + gadd_dev_inprogress++; + mutex_unlock(&bnxt_re_dev_lock); + + rc = bnxt_re_dev_reg(rdev, netdev, en_dev); + if (rc) { + dev_dbg(NULL, "Failed to create add device for netdev %p\n", + netdev); + /* + * For BNXT_RE_POST_RECOVERY_INIT special case + * called from bnxt_re_start, the work is + * complete only after, bnxt_re_start completes + * bnxt_unregister_device in case of failure. + * So bnxt_re_start will decrement gadd_dev_inprogress + * in case of failure. + */ + if (op_type != BNXT_RE_POST_RECOVERY_INIT) { + mutex_lock(&bnxt_re_dev_lock); + gadd_dev_inprogress--; + mutex_unlock(&bnxt_re_dev_lock); + } + return rc; + } + + if (rc != 0) + goto ref_error; + + /* + * num_msix_requested = BNXT_RE_MSIX_FROM_MOD_PARAM indicates fresh driver load. + * Otherwaise, this invocation can be the result of lag create / destroy, + * err revovery, hot fw upgrade, etc.. + */ + if (num_msix_requested == BNXT_RE_MSIX_FROM_MOD_PARAM) { + if (bnxt_re_probe_count < BNXT_RE_MAX_DEVICES) + num_msix_requested = max_msix_vec[bnxt_re_probe_count++]; + else + /* Consider as default when probe_count exceeds its limit */ + num_msix_requested = 0; + + /* if user specifies only one value, use the same for all PFs */ + if (max_msix_vec_argc == 1) + num_msix_requested = max_msix_vec[0]; + } + + (*rdev)->num_msix_requested = num_msix_requested; + (*rdev)->gsi_ctx.gsi_qp_mode = qp_mode; + (*rdev)->adev = aux_dev; + (*rdev)->dev_addr = en_dev->softc->func.mac_addr; + /* Before updating the rdev pointer in bnxt_re_en_dev_info structure, + * take the rtnl lock to avoid accessing invalid rdev pointer from + * L2 ULP callbacks. This is applicable in all the places where rdev + * pointer is updated in bnxt_re_en_dev_info. + */ + rtnl_lock(); + en_info->rdev = *rdev; + rtnl_unlock(); + rc = bnxt_re_dev_init(*rdev, op_type, wqe_mode); + if (rc) { +ref_error: + bnxt_re_dev_unreg(*rdev); + *rdev = NULL; + /* + * For BNXT_RE_POST_RECOVERY_INIT special case + * called from bnxt_re_start, the work is + * complete only after, bnxt_re_start completes + * bnxt_unregister_device in case of failure. + * So bnxt_re_start will decrement gadd_dev_inprogress + * in case of failure. + */ + if (op_type != BNXT_RE_POST_RECOVERY_INIT) { + mutex_lock(&bnxt_re_dev_lock); + gadd_dev_inprogress--; + mutex_unlock(&bnxt_re_dev_lock); + } + } + dev_dbg(rdev_to_dev(*rdev), "%s: Adding rdev: %p\n", __func__, *rdev); + if (!rc) { + set_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); + } + return rc; +} + +struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev) +{ + struct pci_dev *pdev_in = rdev->en_dev->pdev; + int tmp_bus_num, bus_num = pdev_in->bus->number; + int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); + int tmp_func_num, func_num = PCI_FUNC(pdev_in->devfn); + struct bnxt_re_dev *tmp_rdev; + + rcu_read_lock(); + list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { + tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; + tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); + tmp_func_num = PCI_FUNC(tmp_rdev->en_dev->pdev->devfn); + + if (bus_num == tmp_bus_num && dev_num == tmp_dev_num && + func_num != tmp_func_num) { + rcu_read_unlock(); + return tmp_rdev; + } + } + rcu_read_unlock(); + return NULL; +} + + +int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event, + struct ifnet *vlan_dev, + struct ifnet *netdev, + struct auxiliary_device *adev) +{ + struct bnxt_re_work *re_work; + + /* Allocate for the deferred task */ + re_work = kzalloc(sizeof(*re_work), GFP_KERNEL); + if (!re_work) + return -ENOMEM; + + re_work->rdev = rdev; + re_work->event = event; + re_work->vlan_dev = vlan_dev; + re_work->adev = adev; + INIT_WORK(&re_work->work, bnxt_re_task); + if (rdev) + atomic_inc(&rdev->sched_count); + re_work->netdev = netdev; + queue_work(bnxt_re_wq, &re_work->work); + + return 0; +} + + +int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev) +{ + struct pci_dev *pdev_in = rdev->en_dev->pdev; + int tmp_bus_num, bus_num = pdev_in->bus->number; + int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); + struct bnxt_re_dev *tmp_rdev; + int pf_cnt = 0; + + rcu_read_lock(); + list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { + tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; + tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); + + if (bus_num == tmp_bus_num && dev_num == tmp_dev_num) + pf_cnt++; + } + rcu_read_unlock(); + return pf_cnt; +} + +/* Handle all deferred netevents tasks */ +static void bnxt_re_task(struct work_struct *work) +{ + struct bnxt_re_en_dev_info *en_info; + struct auxiliary_device *aux_dev; + struct bnxt_re_work *re_work; + struct bnxt_re_dev *rdev; + + re_work = container_of(work, struct bnxt_re_work, work); + + mutex_lock(&bnxt_re_mutex); + rdev = re_work->rdev; + + /* + * If the previous rdev is deleted due to bond creation + * do not handle the event + */ + if (!bnxt_re_is_rdev_valid(rdev)) + goto exit; + + /* Ignore the event, if the device is not registred with IB stack. This + * is to avoid handling any event while the device is added/removed. + */ + if (rdev && !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { + dev_dbg(rdev_to_dev(rdev), "%s: Ignoring netdev event 0x%lx", + __func__, re_work->event); + goto done; + } + + /* Extra check to silence coverity. We shouldn't handle any event + * when rdev is NULL. + */ + if (!rdev) + goto exit; + + dev_dbg(rdev_to_dev(rdev), "Scheduled work for event 0x%lx", + re_work->event); + + switch (re_work->event) { + case NETDEV_UP: + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, + IB_EVENT_PORT_ACTIVE); + bnxt_re_net_register_async_event(rdev); + break; + + case NETDEV_DOWN: + bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); + bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, + IB_EVENT_PORT_ERR); + break; + + case NETDEV_CHANGE: + if (bnxt_re_get_link_state(rdev) == IB_PORT_DOWN) { + bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, + IB_EVENT_PORT_ERR); + break; + } else if (bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE) { + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, + IB_EVENT_PORT_ACTIVE); + } + + /* temporarily disable the check for SR2 */ + if (!bnxt_qplib_query_cc_param(&rdev->qplib_res, + &rdev->cc_param) && + !_is_chip_p7(rdev->chip_ctx)) { + /* + * Disable CC for 10G speed + * for non p5 devices + */ + if (rdev->sl_espeed == SPEED_10000 && + !_is_chip_gen_p5_p7(rdev->chip_ctx)) { + if (rdev->cc_param.enable) + bnxt_re_clear_cc(rdev); + } else { + if (!rdev->cc_param.enable && + rdev->cc_param.admin_enable) + bnxt_re_setup_cc(rdev); + } + } + break; + + case NETDEV_UNREGISTER: + bnxt_re_stopqps_and_ib_uninit(rdev); + aux_dev = rdev->adev; + if (re_work->adev) + goto done; + + bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, aux_dev); + + break; + + default: + break; + } +done: + if (rdev) { + /* memory barrier to guarantee task completion + * before decrementing sched count + */ + mmiowb(); + atomic_dec(&rdev->sched_count); + } +exit: + if (re_work->adev && re_work->event == NETDEV_UNREGISTER) { + en_info = auxiliary_get_drvdata(re_work->adev); + en_info->ib_uninit_done = true; + wake_up(&en_info->waitq); + } + kfree(re_work); + mutex_unlock(&bnxt_re_mutex); +} + +/* + "Notifier chain callback can be invoked for the same chain from + different CPUs at the same time". + + For cases when the netdev is already present, our call to the + register_netdevice_notifier() will actually get the rtnl_lock() + before sending NETDEV_REGISTER and (if up) NETDEV_UP + events. + + But for cases when the netdev is not already present, the notifier + chain is subjected to be invoked from different CPUs simultaneously. + + This is protected by the netdev_mutex. +*/ +static int bnxt_re_netdev_event(struct notifier_block *notifier, + unsigned long event, void *ptr) +{ + struct ifnet *real_dev, *netdev; + struct bnxt_re_dev *rdev = NULL; + + netdev = netdev_notifier_info_to_ifp(ptr); + real_dev = rdma_vlan_dev_real_dev(netdev); + if (!real_dev) + real_dev = netdev; + /* In case of bonding,this will be bond's rdev */ + rdev = bnxt_re_from_netdev(real_dev); + + if (!rdev) + goto exit; + + dev_info(rdev_to_dev(rdev), "%s: Event = %s (0x%lx), rdev %s (real_dev %s)\n", + __func__, bnxt_re_netevent(event), event, + rdev ? rdev->netdev ? rdev->netdev->if_dname : "->netdev = NULL" : "= NULL", + (real_dev == netdev) ? "= netdev" : real_dev->if_dname); + + if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) + goto exit; + + bnxt_re_hold(rdev); + + if (real_dev != netdev) { + switch (event) { + case NETDEV_UP: + bnxt_re_schedule_work(rdev, event, netdev, + NULL, NULL); + break; + case NETDEV_DOWN: + break; + default: + break; + } + goto done; + } + + switch (event) { + case NETDEV_CHANGEADDR: + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + bnxt_re_update_shadow_ah(rdev); + bnxt_qplib_get_guid(rdev->dev_addr, + (u8 *)&rdev->ibdev.node_guid); + break; + + case NETDEV_CHANGE: + bnxt_re_get_link_speed(rdev); + bnxt_re_schedule_work(rdev, event, NULL, NULL, NULL); + break; + case NETDEV_UNREGISTER: + /* netdev notifier will call NETDEV_UNREGISTER again later since + * we are still holding the reference to the netdev + */ + + /* + * Workaround to avoid ib_unregister hang. Check for module + * reference and dont free up the device if the reference + * is non zero. Checking only for PF functions. + */ + + if (rdev) { + dev_info(rdev_to_dev(rdev), + "bnxt_re:Unreg recvd when module refcnt > 0"); + dev_info(rdev_to_dev(rdev), + "bnxt_re:Close all apps using bnxt_re devs"); + dev_info(rdev_to_dev(rdev), + "bnxt_re:Remove the configfs entry created for the device"); + dev_info(rdev_to_dev(rdev), + "bnxt_re:Refer documentation for details"); + goto done; + } + + if (atomic_read(&rdev->sched_count) > 0) + goto done; + if (!rdev->unreg_sched) { + bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, + NULL, NULL, NULL); + rdev->unreg_sched = true; + goto done; + } + + break; + default: + break; + } +done: + if (rdev) + bnxt_re_put(rdev); +exit: + return NOTIFY_DONE; +} + +static struct notifier_block bnxt_re_netdev_notifier = { + .notifier_call = bnxt_re_netdev_event +}; + +static void bnxt_re_remove_base_interface(struct bnxt_re_dev *rdev, + struct auxiliary_device *adev) +{ + bnxt_re_stopqps_and_ib_uninit(rdev); + bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, adev); + auxiliary_set_drvdata(adev, NULL); +} + +/* + * bnxt_re_remove - Removes the roce aux device + * @adev - aux device pointer + * + * This function removes the roce device. This gets + * called in the mod exit path and pci unbind path. + * If the rdev is bond interace, destroys the lag + * in module exit path, and in pci unbind case + * destroys the lag and recreates other base interface. + * If the device is already removed in error recovery + * path, it just unregister with the L2. + */ +static void bnxt_re_remove(struct auxiliary_device *adev) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); + struct bnxt_en_dev *en_dev; + struct bnxt_re_dev *rdev; + bool primary_dev = false; + bool secondary_dev = false; + + if (!en_info) + return; + + mutex_lock(&bnxt_re_mutex); + en_dev = en_info->en_dev; + + rdev = en_info->rdev; + + if (rdev && bnxt_re_is_rdev_valid(rdev)) { + if (pci_channel_offline(rdev->rcfw.pdev)) + set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); + + if (test_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags)) + primary_dev = true; + if (test_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags)) + secondary_dev = true; + + /* + * en_dev_info of primary device and secondary device have the + * same rdev pointer when LAG is configured. This rdev pointer + * is rdev of bond interface. + */ + if (!primary_dev && !secondary_dev) { + /* removal of non bond interface */ + bnxt_re_remove_base_interface(rdev, adev); + } else { + /* + * removal of bond primary/secondary interface. In this + * case bond device is already removed, so rdev->binfo + * is NULL. + */ + auxiliary_set_drvdata(adev, NULL); + } + } else { + /* device is removed from ulp stop, unregister the net dev */ + if (test_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags)) { + rtnl_lock(); + en_dev->en_ops->bnxt_unregister_device(en_dev, + BNXT_ROCE_ULP); + rtnl_unlock(); + } + } + mutex_unlock(&bnxt_re_mutex); + return; +} + +/* wrapper for all external user context callers */ +void _bnxt_re_remove(struct auxiliary_device *adev) +{ + bnxt_re_remove(adev); +} + +static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev) +{ + int rc; + + rc = bnxt_re_get_device_stats(rdev); + if (rc) + dev_err(rdev_to_dev(rdev), + "Failed initial device stat query"); + + bnxt_re_net_register_async_event(rdev); +} + +static int bnxt_re_probe(struct auxiliary_device *adev, + const struct auxiliary_device_id *id) +{ + struct bnxt_aux_dev *aux_dev = + container_of(adev, struct bnxt_aux_dev, aux_dev); + struct bnxt_re_en_dev_info *en_info; + struct bnxt_en_dev *en_dev = NULL; + struct bnxt_re_dev *rdev; + int rc = -ENODEV; + + if (aux_dev) + en_dev = aux_dev->edev; + + if (!en_dev) + return rc; + + if (en_dev->ulp_version != BNXT_ULP_VERSION) { + pr_err("%s: probe error: bnxt_en ulp version magic %x is not compatible!\n", + ROCE_DRV_MODULE_NAME, en_dev->ulp_version); + return -EINVAL; + } + + en_info = kzalloc(sizeof(*en_info), GFP_KERNEL); + if (!en_info) + return -ENOMEM; + memset(en_info, 0, sizeof(struct bnxt_re_en_dev_info)); + en_info->en_dev = en_dev; + auxiliary_set_drvdata(adev, en_info); + + mutex_lock(&bnxt_re_mutex); + rc = bnxt_re_add_device(&rdev, en_dev->net, + BNXT_RE_GSI_MODE_ALL, + BNXT_RE_COMPLETE_INIT, + BNXT_QPLIB_WQE_MODE_STATIC, + BNXT_RE_MSIX_FROM_MOD_PARAM, adev); + if (rc) { + mutex_unlock(&bnxt_re_mutex); + return rc; + } + + rc = bnxt_re_ib_init(rdev); + if (rc) + goto err; + + bnxt_re_ib_init_2(rdev); + + dev_dbg(rdev_to_dev(rdev), "%s: adev: %p\n", __func__, adev); + rdev->adev = adev; + + mutex_unlock(&bnxt_re_mutex); + + return 0; + +err: + mutex_unlock(&bnxt_re_mutex); + bnxt_re_remove(adev); + + return rc; +} + +static const struct auxiliary_device_id bnxt_re_id_table[] = { + { .name = BNXT_ADEV_NAME ".rdma", }, + {}, +}; + +MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table); + +static struct auxiliary_driver bnxt_re_driver = { + .name = "rdma", + .probe = bnxt_re_probe, + .remove = bnxt_re_remove, + .id_table = bnxt_re_id_table, +}; + +static int __init bnxt_re_mod_init(void) +{ + int rc = 0; + + pr_info("%s: %s", ROCE_DRV_MODULE_NAME, drv_version); + + bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); + if (!bnxt_re_wq) + return -ENOMEM; + + rc = bnxt_re_register_netdevice_notifier(&bnxt_re_netdev_notifier); + if (rc) { + pr_err("%s: Cannot register to netdevice_notifier", + ROCE_DRV_MODULE_NAME); + goto err_netdev; + } + + INIT_LIST_HEAD(&bnxt_re_dev_list); + + rc = auxiliary_driver_register(&bnxt_re_driver); + if (rc) { + pr_err("%s: Failed to register auxiliary driver\n", + ROCE_DRV_MODULE_NAME); + goto err_auxdrv; + } + + return 0; + +err_auxdrv: + bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); + +err_netdev: + destroy_workqueue(bnxt_re_wq); + + return rc; +} + +static void __exit bnxt_re_mod_exit(void) +{ + gmod_exit = 1; + auxiliary_driver_unregister(&bnxt_re_driver); + + bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); + + if (bnxt_re_wq) + destroy_workqueue(bnxt_re_wq); +} + +module_init(bnxt_re_mod_init); +module_exit(bnxt_re_mod_exit); diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.c b/sys/dev/bnxt/bnxt_re/qplib_fp.c new file mode 100644 index 00000000000..438f7ddd468 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_fp.c @@ -0,0 +1,3544 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Fast Path Operators + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hsi_struct_def.h" +#include "qplib_tlv.h" +#include "qplib_res.h" +#include "qplib_rcfw.h" +#include "qplib_sp.h" +#include "qplib_fp.h" +#include "ib_verbs.h" + +static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp); + +static void bnxt_re_legacy_cancel_phantom_processing(struct bnxt_qplib_qp *qp) +{ + qp->sq.condition = false; + qp->sq.legacy_send_phantom = false; + qp->sq.single = false; +} + +static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_cq *scq, *rcq; + + scq = qp->scq; + rcq = qp->rcq; + + if (!qp->sq.flushed) { + dev_dbg(&scq->hwq.pdev->dev, + "QPLIB: FP: Adding to SQ Flush list = %p\n", + qp); + bnxt_re_legacy_cancel_phantom_processing(qp); + list_add_tail(&qp->sq_flush, &scq->sqf_head); + qp->sq.flushed = true; + } + if (!qp->srq) { + if (!qp->rq.flushed) { + dev_dbg(&rcq->hwq.pdev->dev, + "QPLIB: FP: Adding to RQ Flush list = %p\n", + qp); + list_add_tail(&qp->rq_flush, &rcq->rqf_head); + qp->rq.flushed = true; + } + } +} + +static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp) + __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock) +{ + /* Interrupts are already disabled in calling functions */ + spin_lock(&qp->scq->flush_lock); + if (qp->scq == qp->rcq) + __acquire(&qp->rcq->flush_lock); + else + spin_lock(&qp->rcq->flush_lock); +} + +static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp) + __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock) +{ + if (qp->scq == qp->rcq) + __release(&qp->rcq->flush_lock); + else + spin_unlock(&qp->rcq->flush_lock); + spin_unlock(&qp->scq->flush_lock); +} + +void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) +{ + + bnxt_qplib_acquire_cq_flush_locks(qp); + __bnxt_qplib_add_flush_qp(qp); + bnxt_qplib_release_cq_flush_locks(qp); +} + +static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp) +{ + if (qp->sq.flushed) { + qp->sq.flushed = false; + list_del(&qp->sq_flush); + } + if (!qp->srq) { + if (qp->rq.flushed) { + qp->rq.flushed = false; + list_del(&qp->rq_flush); + } + } +} + +void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp) +{ + + bnxt_qplib_acquire_cq_flush_locks(qp); + __clean_cq(qp->scq, (u64)(unsigned long)qp); + qp->sq.hwq.prod = 0; + qp->sq.hwq.cons = 0; + qp->sq.swq_start = 0; + qp->sq.swq_last = 0; + __clean_cq(qp->rcq, (u64)(unsigned long)qp); + qp->rq.hwq.prod = 0; + qp->rq.hwq.cons = 0; + qp->rq.swq_start = 0; + qp->rq.swq_last = 0; + + __bnxt_qplib_del_flush_qp(qp); + bnxt_qplib_release_cq_flush_locks(qp); +} + +static void bnxt_qpn_cqn_sched_task(struct work_struct *work) +{ + struct bnxt_qplib_nq_work *nq_work = + container_of(work, struct bnxt_qplib_nq_work, work); + + struct bnxt_qplib_cq *cq = nq_work->cq; + struct bnxt_qplib_nq *nq = nq_work->nq; + + if (cq && nq) { + spin_lock_bh(&cq->compl_lock); + if (nq->cqn_handler) { + dev_dbg(&nq->res->pdev->dev, + "%s:Trigger cq = %p event nq = %p\n", + __func__, cq, nq); + nq->cqn_handler(nq, cq); + } + spin_unlock_bh(&cq->compl_lock); + } + kfree(nq_work); +} + +static void bnxt_qplib_put_hdr_buf(struct pci_dev *pdev, + struct bnxt_qplib_hdrbuf *buf) +{ + dma_free_coherent(&pdev->dev, buf->len, buf->va, buf->dma_map); + kfree(buf); +} + +static void *bnxt_qplib_get_hdr_buf(struct pci_dev *pdev, u32 step, u32 cnt) +{ + struct bnxt_qplib_hdrbuf *hdrbuf; + u32 len; + + hdrbuf = kmalloc(sizeof(*hdrbuf), GFP_KERNEL); + if (!hdrbuf) + return NULL; + + len = ALIGN((step * cnt), PAGE_SIZE); + hdrbuf->va = dma_alloc_coherent(&pdev->dev, len, + &hdrbuf->dma_map, GFP_KERNEL); + if (!hdrbuf->va) + goto out; + + hdrbuf->len = len; + hdrbuf->step = step; + return hdrbuf; +out: + kfree(hdrbuf); + return NULL; +} + +void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp) +{ + if (qp->rq_hdr_buf) { + bnxt_qplib_put_hdr_buf(res->pdev, qp->rq_hdr_buf); + qp->rq_hdr_buf = NULL; + } + + if (qp->sq_hdr_buf) { + bnxt_qplib_put_hdr_buf(res->pdev, qp->sq_hdr_buf); + qp->sq_hdr_buf = NULL; + } +} + +int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp, u32 sstep, u32 rstep) +{ + struct pci_dev *pdev; + int rc = 0; + + pdev = res->pdev; + if (sstep) { + qp->sq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, sstep, + qp->sq.max_wqe); + if (!qp->sq_hdr_buf) { + dev_err(&pdev->dev, "QPLIB: Failed to get sq_hdr_buf\n"); + return -ENOMEM; + } + } + + if (rstep) { + qp->rq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, rstep, + qp->rq.max_wqe); + if (!qp->rq_hdr_buf) { + rc = -ENOMEM; + dev_err(&pdev->dev, "QPLIB: Failed to get rq_hdr_buf\n"); + goto fail; + } + } + + return 0; +fail: + bnxt_qplib_free_hdr_buf(res, qp); + return rc; +} + +/* + * clean_nq - Invalidate cqe from given nq. + * @cq - Completion queue + * + * Traverse whole notification queue and invalidate any completion + * associated cq handler provided by caller. + * Note - This function traverse the hardware queue but do not update + * consumer index. Invalidated cqe(marked from this function) will be + * ignored from actual completion of notification queue. + */ +static void clean_nq(struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_hwq *nq_hwq = NULL; + struct bnxt_qplib_nq *nq = NULL; + struct nq_base *hw_nqe = NULL; + struct nq_cn *nqcne = NULL; + u32 peek_flags, peek_cons; + u64 q_handle; + u32 type; + int i; + + nq = cq->nq; + nq_hwq = &nq->hwq; + + spin_lock_bh(&nq_hwq->lock); + peek_flags = nq->nq_db.dbinfo.flags; + peek_cons = nq_hwq->cons; + for (i = 0; i < nq_hwq->max_elements; i++) { + hw_nqe = bnxt_qplib_get_qe(nq_hwq, peek_cons, NULL); + if (!NQE_CMP_VALID(hw_nqe, peek_flags)) + break; + + /* The valid test of the entry must be done first + * before reading any further. + */ + dma_rmb(); + type = le16_to_cpu(hw_nqe->info10_type) & + NQ_BASE_TYPE_MASK; + + /* Processing only NQ_BASE_TYPE_CQ_NOTIFICATION */ + if (type == NQ_BASE_TYPE_CQ_NOTIFICATION) { + nqcne = (struct nq_cn *)hw_nqe; + + q_handle = le32_to_cpu(nqcne->cq_handle_low); + q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32; + if (q_handle == (u64)cq) { + nqcne->cq_handle_low = 0; + nqcne->cq_handle_high = 0; + cq->cnq_events++; + } + } + bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &peek_cons, + 1, &peek_flags); + } + spin_unlock_bh(&nq_hwq->lock); +} + +/* + * Wait for receiving all NQEs for this CQ. + * clean_nq is tried 100 times, each time clean_cq + * loops upto budget times. budget is based on the + * number of CQs shared by that NQ. So any NQE from + * CQ would be already in the NQ. + */ +static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events) +{ + u32 retry_cnt = 100; + u16 total_events; + + if (!cnq_events) { + clean_nq(cq); + return; + } + while (retry_cnt--) { + total_events = cq->cnq_events; + + /* Increment total_events by 1 if any CREQ event received with CQ notification */ + if (cq->is_cq_err_event) + total_events++; + + if (cnq_events == total_events) { + dev_dbg(&cq->nq->res->pdev->dev, + "QPLIB: NQ cleanup - Received all NQ events\n"); + return; + } + msleep(1); + clean_nq(cq); + } +} + +static void bnxt_qplib_service_nq(unsigned long data) +{ + struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; + struct bnxt_qplib_hwq *nq_hwq = &nq->hwq; + int budget = nq->budget; + struct bnxt_qplib_res *res; + struct bnxt_qplib_cq *cq; + struct pci_dev *pdev; + struct nq_base *nqe; + u32 hw_polled = 0; + u64 q_handle; + u32 type; + + res = nq->res; + pdev = res->pdev; + + spin_lock_bh(&nq_hwq->lock); + /* Service the NQ until empty or budget expired */ + while (budget--) { + nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL); + if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) + break; + /* The valid test of the entry must be done first before + * reading any further. + */ + dma_rmb(); + type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; + switch (type) { + case NQ_BASE_TYPE_CQ_NOTIFICATION: + { + struct nq_cn *nqcne = (struct nq_cn *)nqe; + + q_handle = le32_to_cpu(nqcne->cq_handle_low); + q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32; + cq = (struct bnxt_qplib_cq *)q_handle; + if (!cq) + break; + cq->toggle = (le16_to_cpu(nqe->info10_type) & NQ_CN_TOGGLE_MASK) >> NQ_CN_TOGGLE_SFT; + cq->dbinfo.toggle = cq->toggle; + bnxt_qplib_armen_db(&cq->dbinfo, + DBC_DBC_TYPE_CQ_ARMENA); + spin_lock_bh(&cq->compl_lock); + atomic_set(&cq->arm_state, 0) ; + if (!nq->cqn_handler(nq, (cq))) + nq->stats.num_cqne_processed++; + else + dev_warn(&pdev->dev, + "QPLIB: cqn - type 0x%x not handled\n", + type); + cq->cnq_events++; + spin_unlock_bh(&cq->compl_lock); + break; + } + case NQ_BASE_TYPE_SRQ_EVENT: + { + struct bnxt_qplib_srq *srq; + struct nq_srq_event *nqsrqe = + (struct nq_srq_event *)nqe; + + q_handle = le32_to_cpu(nqsrqe->srq_handle_low); + q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) << 32; + srq = (struct bnxt_qplib_srq *)q_handle; + bnxt_qplib_armen_db(&srq->dbinfo, + DBC_DBC_TYPE_SRQ_ARMENA); + if (!nq->srqn_handler(nq, + (struct bnxt_qplib_srq *)q_handle, + nqsrqe->event)) + nq->stats.num_srqne_processed++; + else + dev_warn(&pdev->dev, + "QPLIB: SRQ event 0x%x not handled\n", + nqsrqe->event); + break; + } + default: + dev_warn(&pdev->dev, + "QPLIB: nqe with opcode = 0x%x not handled\n", + type); + break; + } + hw_polled++; + bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &nq_hwq->cons, + 1, &nq->nq_db.dbinfo.flags); + } + nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL); + if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) { + nq->stats.num_nq_rearm++; + bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); + } else if (nq->requested) { + bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); + nq->stats.num_tasklet_resched++; + } + dev_dbg(&pdev->dev, "QPLIB: cqn/srqn/dbqn \n"); + if (hw_polled >= 0) + dev_dbg(&pdev->dev, + "QPLIB: serviced %llu/%llu/%llu budget 0x%x reaped 0x%x\n", + nq->stats.num_cqne_processed, nq->stats.num_srqne_processed, + nq->stats.num_dbqne_processed, budget, hw_polled); + dev_dbg(&pdev->dev, + "QPLIB: resched_cnt = %llu arm_count = %llu\n", + nq->stats.num_tasklet_resched, nq->stats.num_nq_rearm); + spin_unlock_bh(&nq_hwq->lock); +} + +static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) +{ + struct bnxt_qplib_nq *nq = dev_instance; + struct bnxt_qplib_hwq *nq_hwq = &nq->hwq; + u32 sw_cons; + + /* Prefetch the NQ element */ + sw_cons = HWQ_CMP(nq_hwq->cons, nq_hwq); + if (sw_cons >= 0) + prefetch(bnxt_qplib_get_qe(nq_hwq, sw_cons, NULL)); + + bnxt_qplib_service_nq((unsigned long)nq); + + return IRQ_HANDLED; +} + +void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill) +{ + struct bnxt_qplib_res *res; + + if (!nq->requested) + return; + + nq->requested = false; + res = nq->res; + /* Mask h/w interrupt */ + bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, false); + /* Sync with last running IRQ handler */ + synchronize_irq(nq->msix_vec); + free_irq(nq->msix_vec, nq); + kfree(nq->name); + nq->name = NULL; +} + +void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) +{ + if (nq->cqn_wq) { + destroy_workqueue(nq->cqn_wq); + nq->cqn_wq = NULL; + } + /* Make sure the HW is stopped! */ + bnxt_qplib_nq_stop_irq(nq, true); + + nq->nq_db.reg.bar_reg = NULL; + nq->nq_db.db = NULL; + + nq->cqn_handler = NULL; + nq->srqn_handler = NULL; + nq->msix_vec = 0; +} + +int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, + int msix_vector, bool need_init) +{ + struct bnxt_qplib_res *res; + int rc; + + res = nq->res; + if (nq->requested) + return -EFAULT; + + nq->msix_vec = msix_vector; + nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s\n", + nq_indx, pci_name(res->pdev)); + if (!nq->name) + return -ENOMEM; + rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq); + if (rc) { + kfree(nq->name); + nq->name = NULL; + return rc; + } + nq->requested = true; + bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); + + return rc; +} + +static void bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq, u32 reg_offt) +{ + struct bnxt_qplib_reg_desc *dbreg; + struct bnxt_qplib_nq_db *nq_db; + struct bnxt_qplib_res *res; + + nq_db = &nq->nq_db; + res = nq->res; + dbreg = &res->dpi_tbl.ucreg; + + nq_db->reg.bar_id = dbreg->bar_id; + nq_db->reg.bar_base = dbreg->bar_base; + nq_db->reg.bar_reg = dbreg->bar_reg + reg_offt; + nq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) : + sizeof(u32); + + nq_db->dbinfo.db = nq_db->reg.bar_reg; + nq_db->dbinfo.hwq = &nq->hwq; + nq_db->dbinfo.xid = nq->ring_id; + nq_db->dbinfo.seed = nq->ring_id; + nq_db->dbinfo.flags = 0; + spin_lock_init(&nq_db->dbinfo.lock); + nq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + nq_db->dbinfo.res = nq->res; + + return; +} + +int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx, + int msix_vector, int bar_reg_offset, + cqn_handler_t cqn_handler, + srqn_handler_t srqn_handler) +{ + struct pci_dev *pdev; + int rc; + + pdev = nq->res->pdev; + nq->cqn_handler = cqn_handler; + nq->srqn_handler = srqn_handler; + nq->load = 0; + mutex_init(&nq->lock); + + /* Have a task to schedule CQ notifiers in post send case */ + nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq\n"); + if (!nq->cqn_wq) + return -ENOMEM; + + bnxt_qplib_map_nq_db(nq, bar_reg_offset); + rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true); + if (rc) { + dev_err(&pdev->dev, + "QPLIB: Failed to request irq for nq-idx %d\n", nq_idx); + goto fail; + } + dev_dbg(&pdev->dev, "QPLIB: NQ max = 0x%x\n", nq->hwq.max_elements); + + return 0; +fail: + bnxt_qplib_disable_nq(nq); + return rc; +} + +void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq) +{ + if (nq->hwq.max_elements) { + bnxt_qplib_free_hwq(nq->res, &nq->hwq); + nq->hwq.max_elements = 0; + } +} + +int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res, + struct bnxt_qplib_nq *nq) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_sg_info sginfo = {}; + + nq->res = res; + if (!nq->hwq.max_elements || + nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) + nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; + + sginfo.pgsize = PAGE_SIZE; + sginfo.pgshft = PAGE_SHIFT; + hwq_attr.res = res; + hwq_attr.sginfo = &sginfo; + hwq_attr.depth = nq->hwq.max_elements; + hwq_attr.stride = sizeof(struct nq_base); + hwq_attr.type = _get_hwq_type(res); + if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) { + dev_err(&res->pdev->dev, "QPLIB: FP NQ allocation failed\n"); + return -ENOMEM; + } + nq->budget = 8; + return 0; +} + +/* SRQ */ +static int __qplib_destroy_srq(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_srq *srq) +{ + struct creq_destroy_srq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_destroy_srq req = {}; + /* Configure the request */ + req.srq_cid = cpu_to_le32(srq->id); + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_SRQ, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + return bnxt_qplib_rcfw_send_message(rcfw, &msg); +} + +int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + int rc; + + rc = __qplib_destroy_srq(rcfw, srq); + if (rc) + return rc; + bnxt_qplib_free_hwq(res, &srq->hwq); + kfree(srq->swq); + return 0; +} + +int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_create_srq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_create_srq req = {}; + u16 pg_sz_lvl = 0; + u16 srq_size; + int rc, idx; + + hwq_attr.res = res; + hwq_attr.sginfo = &srq->sginfo; + hwq_attr.depth = srq->max_wqe; + hwq_attr.stride = srq->wqe_size; + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr); + if (rc) + goto exit; + /* Configure the request */ + req.dpi = cpu_to_le32(srq->dpi->dpi); + req.srq_handle = cpu_to_le64(srq); + srq_size = min_t(u32, srq->hwq.depth, U16_MAX); + req.srq_size = cpu_to_le16(srq_size); + pg_sz_lvl |= (_get_base_pg_size(&srq->hwq) << + CMDQ_CREATE_SRQ_PG_SIZE_SFT); + pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK); + req.pg_size_lvl = cpu_to_le16(pg_sz_lvl); + req.pbl = cpu_to_le64(_get_base_addr(&srq->hwq)); + req.pd_id = cpu_to_le32(srq->pd->id); + req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id); + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_SRQ, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail; + if (!srq->is_user) { + srq->swq = kcalloc(srq->hwq.depth, sizeof(*srq->swq), + GFP_KERNEL); + if (!srq->swq) + goto srq_fail; + srq->start_idx = 0; + srq->last_idx = srq->hwq.depth - 1; + for (idx = 0; idx < srq->hwq.depth; idx++) + srq->swq[idx].next_idx = idx + 1; + srq->swq[srq->last_idx].next_idx = -1; + } + + spin_lock_init(&srq->lock); + srq->id = le32_to_cpu(resp.xid); + srq->cctx = res->cctx; + srq->dbinfo.hwq = &srq->hwq; + srq->dbinfo.xid = srq->id; + srq->dbinfo.db = srq->dpi->dbr; + srq->dbinfo.max_slot = 1; + srq->dbinfo.priv_db = res->dpi_tbl.priv_db; + srq->dbinfo.flags = 0; + spin_lock_init(&srq->dbinfo.lock); + srq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + srq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID; + srq->dbinfo.res = res; + srq->dbinfo.seed = srq->id; + if (srq->threshold) + bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); + srq->arm_req = false; + return 0; +srq_fail: + __qplib_destroy_srq(rcfw, srq); +fail: + bnxt_qplib_free_hwq(res, &srq->hwq); +exit: + return rc; +} + +int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; + u32 avail = 0; + + avail = __bnxt_qplib_get_avail(srq_hwq); + if (avail <= srq->threshold) { + srq->arm_req = false; + bnxt_qplib_srq_arm_db(&srq->dbinfo); + } else { + /* Deferred arming */ + srq->arm_req = true; + } + return 0; +} + +int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_query_srq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct creq_query_srq_resp_sb *sb; + struct bnxt_qplib_rcfw_sbuf sbuf; + struct cmdq_query_srq req = {}; + int rc = 0; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_SRQ, + sizeof(req)); + sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) + return -ENOMEM; + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + req.srq_cid = cpu_to_le32(srq->id); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + /* TODO: What to do with the query? */ + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + + return rc; +} + +int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; + struct sq_sge *hw_sge; + struct rq_wqe *srqe; + int i, rc = 0, next; + u32 avail; + + spin_lock(&srq_hwq->lock); + if (srq->start_idx == srq->last_idx) { + dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!\n", + srq->id); + rc = -EINVAL; + spin_unlock(&srq_hwq->lock); + goto done; + } + next = srq->start_idx; + srq->start_idx = srq->swq[next].next_idx; + spin_unlock(&srq_hwq->lock); + + srqe = bnxt_qplib_get_qe(srq_hwq, srq_hwq->prod, NULL); + memset(srqe, 0, srq->wqe_size); + /* Calculate wqe_size and data_len */ + for (i = 0, hw_sge = (struct sq_sge *)srqe->data; + i < wqe->num_sge; i++, hw_sge++) { + hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); + hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); + hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); + } + srqe->wqe_type = wqe->type; + srqe->flags = wqe->flags; + srqe->wqe_size = wqe->num_sge + + ((offsetof(typeof(*srqe), data) + 15) >> 4); + if (!wqe->num_sge) + srqe->wqe_size++; + srqe->wr_id |= cpu_to_le32((u32)next); + srq->swq[next].wr_id = wqe->wr_id; + bnxt_qplib_hwq_incr_prod(&srq->dbinfo, srq_hwq, srq->dbinfo.max_slot); + /* retaining srq_hwq->cons for this logic actually the lock is only + * required to read srq_hwq->cons. + */ + spin_lock(&srq_hwq->lock); + avail = __bnxt_qplib_get_avail(srq_hwq); + spin_unlock(&srq_hwq->lock); + /* Ring DB */ + bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ); + if (srq->arm_req && avail <= srq->threshold) { + srq->arm_req = false; + bnxt_qplib_srq_arm_db(&srq->dbinfo); + } +done: + return rc; +} + +/* QP */ +static int __qplib_destroy_qp(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_qp *qp) +{ + struct creq_destroy_qp_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_destroy_qp req = {}; + + req.qp_cid = cpu_to_le32(qp->id); + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_QP, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + return bnxt_qplib_rcfw_send_message(rcfw, &msg); +} + +static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que) +{ + int rc = 0; + int indx; + + que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL); + if (!que->swq) { + rc = -ENOMEM; + goto out; + } + + que->swq_start = 0; + que->swq_last = que->max_wqe - 1; + for (indx = 0; indx < que->max_wqe; indx++) + que->swq[indx].next_idx = indx + 1; + que->swq[que->swq_last].next_idx = 0; /* Make it circular */ + que->swq_last = 0; +out: + return rc; +} + +static struct bnxt_qplib_swq *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que, + u32 *swq_idx) +{ + u32 idx; + + idx = que->swq_start; + if (swq_idx) + *swq_idx = idx; + return &que->swq[idx]; +} + +static void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx) +{ + que->swq_start = que->swq[idx].next_idx; +} + +static u32 bnxt_qplib_get_stride(void) +{ + return sizeof(struct sq_sge); +} + +static u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que) +{ + u8 stride; + + stride = bnxt_qplib_get_stride(); + return (que->wqe_size * que->max_wqe) / stride; +} + +static u32 _set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode) +{ + /* For Variable mode supply number of 16B slots */ + return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + que->max_wqe : bnxt_qplib_get_depth(que); +} + +static u32 _set_sq_max_slot(u8 wqe_mode) +{ + /* for static mode index divisor is 8 */ + return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + sizeof(struct sq_send) / sizeof(struct sq_sge) : 1; +} + +static u32 _set_rq_max_slot(struct bnxt_qplib_q *que) +{ + return (que->wqe_size / sizeof(struct sq_sge)); +} + +int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_create_qp1_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_q *rq = &qp->rq; + struct cmdq_create_qp1 req = {}; + struct bnxt_qplib_reftbl *tbl; + unsigned long flag; + u8 pg_sz_lvl = 0; + u32 qp_flags = 0; + int rc; + + /* General */ + req.type = qp->type; + req.dpi = cpu_to_le32(qp->dpi->dpi); + req.qp_handle = cpu_to_le64(qp->qp_handle); + /* SQ */ + hwq_attr.res = res; + hwq_attr.sginfo = &sq->sginfo; + hwq_attr.stride = bnxt_qplib_get_stride(); + hwq_attr.depth = bnxt_qplib_get_depth(sq); + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); + if (rc) + goto exit; + + req.sq_size = cpu_to_le32(_set_sq_size(sq, qp->wqe_mode)); + req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq)); + pg_sz_lvl = _get_base_pg_size(&sq->hwq) << + CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT; + pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) << + CMDQ_CREATE_QP1_SQ_LVL_SFT); + req.sq_pg_size_sq_lvl = pg_sz_lvl; + req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP1_SQ_FWO_SFT) & + CMDQ_CREATE_QP1_SQ_FWO_MASK) | + (sq->max_sge & + CMDQ_CREATE_QP1_SQ_SGE_MASK)); + req.scq_cid = cpu_to_le32(qp->scq->id); + + /* RQ */ + if (!qp->srq) { + hwq_attr.res = res; + hwq_attr.sginfo = &rq->sginfo; + hwq_attr.stride = bnxt_qplib_get_stride(); + hwq_attr.depth = bnxt_qplib_get_depth(rq); + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); + if (rc) + goto fail_sq; + req.rq_size = cpu_to_le32(rq->max_wqe); + req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq)); + pg_sz_lvl = _get_base_pg_size(&rq->hwq) << + CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT; + pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) << + CMDQ_CREATE_QP1_RQ_LVL_SFT); + req.rq_pg_size_rq_lvl = pg_sz_lvl; + req.rq_fwo_rq_sge = + cpu_to_le16(((0 << CMDQ_CREATE_QP1_RQ_FWO_SFT) & + CMDQ_CREATE_QP1_RQ_FWO_MASK) | + (rq->max_sge & + CMDQ_CREATE_QP1_RQ_SGE_MASK)); + } else { + /* SRQ */ + qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED; + req.srq_cid = cpu_to_le32(qp->srq->id); + } + req.rcq_cid = cpu_to_le32(qp->rcq->id); + + qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; + req.qp_flags = cpu_to_le32(qp_flags); + req.pd_id = cpu_to_le32(qp->pd->id); + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP1, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail_rq; + + rc = bnxt_qplib_alloc_init_swq(sq); + if (rc) + goto sq_swq; + + if (!qp->srq) { + rc = bnxt_qplib_alloc_init_swq(rq); + if (rc) + goto rq_swq; + } + + qp->id = le32_to_cpu(resp.xid); + qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; + qp->cctx = res->cctx; + sq->dbinfo.hwq = &sq->hwq; + sq->dbinfo.xid = qp->id; + sq->dbinfo.db = qp->dpi->dbr; + sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode); + sq->dbinfo.flags = 0; + spin_lock_init(&sq->dbinfo.lock); + sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + sq->dbinfo.res = res; + if (rq->max_wqe) { + rq->dbinfo.hwq = &rq->hwq; + rq->dbinfo.xid = qp->id; + rq->dbinfo.db = qp->dpi->dbr; + rq->dbinfo.max_slot = _set_rq_max_slot(rq); + rq->dbinfo.flags = 0; + spin_lock_init(&rq->dbinfo.lock); + rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + rq->dbinfo.res = res; + } + + tbl = &res->reftbl.qpref; + spin_lock_irqsave(&tbl->lock, flag); + tbl->rec[tbl->max].xid = qp->id; + tbl->rec[tbl->max].handle = qp; + spin_unlock_irqrestore(&tbl->lock, flag); + + return 0; +rq_swq: + kfree(sq->swq); +sq_swq: + __qplib_destroy_qp(rcfw, qp); +fail_rq: + bnxt_qplib_free_hwq(res, &rq->hwq); +fail_sq: + bnxt_qplib_free_hwq(res, &sq->hwq); +exit: + return rc; +} + +static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size) +{ + struct bnxt_qplib_hwq *sq_hwq; + struct bnxt_qplib_q *sq; + u64 fpsne, psn_pg; + u16 indx_pad = 0; + + sq = &qp->sq; + sq_hwq = &sq->hwq; + /* First psn entry */ + fpsne = (u64)bnxt_qplib_get_qe(sq_hwq, sq_hwq->depth, &psn_pg); + if (!IS_ALIGNED(fpsne, PAGE_SIZE)) + indx_pad = (fpsne & ~PAGE_MASK) / size; + sq_hwq->pad_pgofft = indx_pad; + sq_hwq->pad_pg = (u64 *)psn_pg; + sq_hwq->pad_stride = size; +} + +int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct bnxt_qplib_sg_info sginfo = {}; + struct creq_create_qp_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_q *rq = &qp->rq; + struct cmdq_create_qp req = {}; + struct bnxt_qplib_reftbl *tbl; + struct bnxt_qplib_hwq *xrrq; + int rc, req_size, psn_sz; + unsigned long flag; + u8 pg_sz_lvl = 0; + u32 qp_flags = 0; + u32 qp_idx; + u16 nsge; + u32 sqsz; + + qp->cctx = res->cctx; + if (res->dattr) + qp->dev_cap_flags = res->dattr->dev_cap_flags; + /* General */ + req.type = qp->type; + req.dpi = cpu_to_le32(qp->dpi->dpi); + req.qp_handle = cpu_to_le64(qp->qp_handle); + + /* SQ */ + if (qp->type == CMDQ_CREATE_QP_TYPE_RC) { + psn_sz = _is_chip_gen_p5_p7(qp->cctx) ? + sizeof(struct sq_psn_search_ext) : + sizeof(struct sq_psn_search); + if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + psn_sz = sizeof(struct sq_msn_search); + qp->msn = 0; + } + } else { + psn_sz = 0; + } + + hwq_attr.res = res; + hwq_attr.sginfo = &sq->sginfo; + hwq_attr.stride = bnxt_qplib_get_stride(); + hwq_attr.depth = bnxt_qplib_get_depth(sq); + hwq_attr.aux_stride = psn_sz; + hwq_attr.aux_depth = (psn_sz) ? + _set_sq_size(sq, qp->wqe_mode) : 0; + /* Update msn tbl size */ + if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { + if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)); + else + hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)) / 2; + qp->msn_tbl_sz = hwq_attr.aux_depth; + qp->msn = 0; + } + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); + if (rc) + goto exit; + + sqsz = _set_sq_size(sq, qp->wqe_mode); + /* 0xffff is the max sq size hw limits to */ + if (sqsz > BNXT_QPLIB_MAX_SQSZ) { + pr_err("QPLIB: FP: QP (0x%x) exceeds sq size %d\n", qp->id, sqsz); + goto fail_sq; + } + req.sq_size = cpu_to_le32(sqsz); + req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq)); + pg_sz_lvl = _get_base_pg_size(&sq->hwq) << + CMDQ_CREATE_QP_SQ_PG_SIZE_SFT; + pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) << + CMDQ_CREATE_QP_SQ_LVL_SFT); + req.sq_pg_size_sq_lvl = pg_sz_lvl; + req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP_SQ_FWO_SFT) & + CMDQ_CREATE_QP_SQ_FWO_MASK) | + ((BNXT_RE_HW_RETX(qp->dev_cap_flags)) ? + BNXT_MSN_TBLE_SGE : sq->max_sge & + CMDQ_CREATE_QP_SQ_SGE_MASK)); + req.scq_cid = cpu_to_le32(qp->scq->id); + + /* RQ/SRQ */ + if (!qp->srq) { + hwq_attr.res = res; + hwq_attr.sginfo = &rq->sginfo; + hwq_attr.stride = bnxt_qplib_get_stride(); + hwq_attr.depth = bnxt_qplib_get_depth(rq); + hwq_attr.aux_stride = 0; + hwq_attr.aux_depth = 0; + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); + if (rc) + goto fail_sq; + req.rq_size = cpu_to_le32(rq->max_wqe); + req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq)); + pg_sz_lvl = _get_base_pg_size(&rq->hwq) << + CMDQ_CREATE_QP_RQ_PG_SIZE_SFT; + pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) << + CMDQ_CREATE_QP_RQ_LVL_SFT); + req.rq_pg_size_rq_lvl = pg_sz_lvl; + nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + res->dattr->max_qp_sges : rq->max_sge; + req.rq_fwo_rq_sge = + cpu_to_le16(((0 << CMDQ_CREATE_QP_RQ_FWO_SFT) & + CMDQ_CREATE_QP_RQ_FWO_MASK) | + (nsge & CMDQ_CREATE_QP_RQ_SGE_MASK)); + } else { + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED; + req.srq_cid = cpu_to_le32(qp->srq->id); + } + req.rcq_cid = cpu_to_le32(qp->rcq->id); + + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; + if (qp->sig_type) + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; + if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED; + if (res->cctx->modes.te_bypass) + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_OPTIMIZED_TRANSMIT_ENABLED; + if (res->dattr && + bnxt_ext_stats_supported(qp->cctx, res->dattr->dev_cap_flags, res->is_vf)) + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED; + req.qp_flags = cpu_to_le32(qp_flags); + + /* ORRQ and IRRQ */ + if (psn_sz) { + xrrq = &qp->orrq; + xrrq->max_elements = + ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); + req_size = xrrq->max_elements * + BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; + req_size &= ~(PAGE_SIZE - 1); + sginfo.pgsize = req_size; + sginfo.pgshft = PAGE_SHIFT; + + hwq_attr.res = res; + hwq_attr.sginfo = &sginfo; + hwq_attr.depth = xrrq->max_elements; + hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE; + hwq_attr.aux_stride = 0; + hwq_attr.aux_depth = 0; + hwq_attr.type = HWQ_TYPE_CTX; + rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); + if (rc) + goto fail_rq; + req.orrq_addr = cpu_to_le64(_get_base_addr(xrrq)); + + xrrq = &qp->irrq; + xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( + qp->max_dest_rd_atomic); + req_size = xrrq->max_elements * + BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; + req_size &= ~(PAGE_SIZE - 1); + sginfo.pgsize = req_size; + hwq_attr.depth = xrrq->max_elements; + hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE; + rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); + if (rc) + goto fail_orrq; + req.irrq_addr = cpu_to_le64(_get_base_addr(xrrq)); + } + req.pd_id = cpu_to_le32(qp->pd->id); + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail; + + if (!qp->is_user) { + rc = bnxt_qplib_alloc_init_swq(sq); + if (rc) + goto swq_sq; + if (!qp->srq) { + rc = bnxt_qplib_alloc_init_swq(rq); + if (rc) + goto swq_rq; + } + if (psn_sz) + bnxt_qplib_init_psn_ptr(qp, psn_sz); + } + qp->id = le32_to_cpu(resp.xid); + qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; + INIT_LIST_HEAD(&qp->sq_flush); + INIT_LIST_HEAD(&qp->rq_flush); + + sq->dbinfo.hwq = &sq->hwq; + sq->dbinfo.xid = qp->id; + sq->dbinfo.db = qp->dpi->dbr; + sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode); + sq->dbinfo.flags = 0; + spin_lock_init(&sq->dbinfo.lock); + sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + sq->dbinfo.res = res; + sq->dbinfo.seed = qp->id; + if (rq->max_wqe) { + rq->dbinfo.hwq = &rq->hwq; + rq->dbinfo.xid = qp->id; + rq->dbinfo.db = qp->dpi->dbr; + rq->dbinfo.max_slot = _set_rq_max_slot(rq); + rq->dbinfo.flags = 0; + spin_lock_init(&rq->dbinfo.lock); + rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + rq->dbinfo.res = res; + rq->dbinfo.seed = qp->id; + } + + tbl = &res->reftbl.qpref; + qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl); + spin_lock_irqsave(&tbl->lock, flag); + tbl->rec[qp_idx].xid = qp->id; + tbl->rec[qp_idx].handle = qp; + spin_unlock_irqrestore(&tbl->lock, flag); + + return 0; +swq_rq: + kfree(sq->swq); +swq_sq: + __qplib_destroy_qp(rcfw, qp); +fail: + bnxt_qplib_free_hwq(res, &qp->irrq); +fail_orrq: + bnxt_qplib_free_hwq(res, &qp->orrq); +fail_rq: + bnxt_qplib_free_hwq(res, &rq->hwq); +fail_sq: + bnxt_qplib_free_hwq(res, &sq->hwq); +exit: + return rc; +} + +static void __filter_modify_flags(struct bnxt_qplib_qp *qp) +{ + switch (qp->cur_qp_state) { + case CMDQ_MODIFY_QP_NEW_STATE_RESET: + switch (qp->state) { + case CMDQ_MODIFY_QP_NEW_STATE_INIT: + break; + default: + break; + } + break; + case CMDQ_MODIFY_QP_NEW_STATE_INIT: + switch (qp->state) { + case CMDQ_MODIFY_QP_NEW_STATE_RTR: + if (!(qp->modify_flags & + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) { + qp->modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; + qp->path_mtu = CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; + } + qp->modify_flags &= + ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; + /* Bono FW requires the max_dest_rd_atomic to be >= 1 */ + if (qp->max_dest_rd_atomic < 1) + qp->max_dest_rd_atomic = 1; + qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; + /* Bono FW 20.6.5 requires SGID_INDEX to be configured */ + if (!(qp->modify_flags & + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) { + qp->modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX; + qp->ah.sgid_index = 0; + } + break; + default: + break; + } + break; + case CMDQ_MODIFY_QP_NEW_STATE_RTR: + switch (qp->state) { + case CMDQ_MODIFY_QP_NEW_STATE_RTS: + /* Bono FW requires the max_rd_atomic to be >= 1 */ + if (qp->max_rd_atomic < 1) + qp->max_rd_atomic = 1; + qp->modify_flags &= + ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY | + CMDQ_MODIFY_QP_MODIFY_MASK_DGID | + CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | + CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | + CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN | + CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER | + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID); + break; + default: + break; + } + break; + case CMDQ_MODIFY_QP_NEW_STATE_RTS: + break; + case CMDQ_MODIFY_QP_NEW_STATE_SQD: + break; + case CMDQ_MODIFY_QP_NEW_STATE_SQE: + break; + case CMDQ_MODIFY_QP_NEW_STATE_ERR: + break; + default: + break; + } +} + +int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_modify_qp_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_modify_qp req = {}; + bool ppp_requested = false; + u32 temp32[4]; + u32 bmask; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_QP, + sizeof(req)); + + /* Filter out the qp_attr_mask based on the state->new transition */ + __filter_modify_flags(qp); + bmask = qp->modify_flags; + req.modify_mask = cpu_to_le32(qp->modify_flags); + req.qp_cid = cpu_to_le32(qp->id); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) { + req.network_type_en_sqd_async_notify_new_state = + (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | + (qp->en_sqd_async_notify == true ? + CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0); + if (__can_request_ppp(qp)) { + req.path_mtu_pingpong_push_enable = + CMDQ_MODIFY_QP_PINGPONG_PUSH_ENABLE; + req.pingpong_push_dpi = qp->ppp.dpi; + ppp_requested = true; + } + } + req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) { + req.access = qp->access; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) + req.pkey = IB_DEFAULT_PKEY_FULL; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) { + req.qkey = cpu_to_le32(qp->qkey); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) { + memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); + req.dgid[0] = cpu_to_le32(temp32[0]); + req.dgid[1] = cpu_to_le32(temp32[1]); + req.dgid[2] = cpu_to_le32(temp32[2]); + req.dgid[3] = cpu_to_le32(temp32[3]); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) { + req.flow_label = cpu_to_le32(qp->ah.flow_label); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) { + req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[qp->ah.sgid_index]); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) { + req.hop_limit = qp->ah.hop_limit; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) { + req.traffic_class = qp->ah.traffic_class; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) { + memcpy(req.dest_mac, qp->ah.dmac, 6); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) { + req.path_mtu_pingpong_push_enable = qp->path_mtu; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) { + req.timeout = qp->timeout; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) { + req.retry_cnt = qp->retry_cnt; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) { + req.rnr_retry = qp->rnr_retry; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) { + req.min_rnr_timer = qp->min_rnr_timer; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) { + req.rq_psn = cpu_to_le32(qp->rq.psn); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) { + req.sq_psn = cpu_to_le32(qp->sq.psn); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) { + req.max_rd_atomic = + ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) { + req.max_dest_rd_atomic = + IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); + } + req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); + req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); + req.sq_sge = cpu_to_le16(qp->sq.max_sge); + req.rq_sge = cpu_to_le16(qp->rq.max_sge); + req.max_inline_data = cpu_to_le32(qp->max_inline_data); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID) + req.dest_qp_id = cpu_to_le32(qp->dest_qpn); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC) + req.enable_cc = cpu_to_le16(CMDQ_MODIFY_QP_ENABLE_CC); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN) + req.tos_dscp_tos_ecn = + ((qp->tos_ecn << CMDQ_MODIFY_QP_TOS_ECN_SFT) & + CMDQ_MODIFY_QP_TOS_ECN_MASK); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP) + req.tos_dscp_tos_ecn |= + ((qp->tos_dscp << CMDQ_MODIFY_QP_TOS_DSCP_SFT) & + CMDQ_MODIFY_QP_TOS_DSCP_MASK); + req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + msg.qp_state = qp->state; + + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc == -ETIMEDOUT && (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR)) { + qp->cur_qp_state = qp->state; + return 0; + } else if (rc) { + return rc; + } + if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR) + qp->lag_src_mac = be32_to_cpu(resp.lag_src_mac); + + if (ppp_requested) + qp->ppp.st_idx_en = resp.pingpong_push_state_index_enabled; + + qp->cur_qp_state = qp->state; + return 0; +} + +int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_query_qp_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_rcfw_sbuf sbuf; + struct creq_query_qp_resp_sb *sb; + struct cmdq_query_qp req = {}; + u32 temp32[4]; + int i, rc; + + sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) + return -ENOMEM; + sb = sbuf.sb; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_QP, + sizeof(req)); + req.qp_cid = cpu_to_le32(qp->id); + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto bail; + + /* Extract the context from the side buffer */ + qp->state = sb->en_sqd_async_notify_state & + CREQ_QUERY_QP_RESP_SB_STATE_MASK; + qp->cur_qp_state = qp->state; + qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & + CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? + true : false; + qp->access = sb->access; + qp->pkey_index = le16_to_cpu(sb->pkey); + qp->qkey = le32_to_cpu(sb->qkey); + + temp32[0] = le32_to_cpu(sb->dgid[0]); + temp32[1] = le32_to_cpu(sb->dgid[1]); + temp32[2] = le32_to_cpu(sb->dgid[2]); + temp32[3] = le32_to_cpu(sb->dgid[3]); + memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); + + qp->ah.flow_label = le32_to_cpu(sb->flow_label); + + qp->ah.sgid_index = 0; + for (i = 0; i < res->sgid_tbl.max; i++) { + if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { + qp->ah.sgid_index = i; + break; + } + } + if (i == res->sgid_tbl.max) + dev_dbg(&res->pdev->dev, + "QPLIB: SGID not found qp->id = 0x%x sgid_index = 0x%x\n", + qp->id, le16_to_cpu(sb->sgid_index)); + + qp->ah.hop_limit = sb->hop_limit; + qp->ah.traffic_class = sb->traffic_class; + memcpy(qp->ah.dmac, sb->dest_mac, ETH_ALEN); + qp->ah.vlan_id = le16_to_cpu(sb->path_mtu_dest_vlan_id) & + CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK >> + CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT; + qp->path_mtu = le16_to_cpu(sb->path_mtu_dest_vlan_id) & + CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK; + qp->timeout = sb->timeout; + qp->retry_cnt = sb->retry_cnt; + qp->rnr_retry = sb->rnr_retry; + qp->min_rnr_timer = sb->min_rnr_timer; + qp->rq.psn = le32_to_cpu(sb->rq_psn); + qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); + qp->sq.psn = le32_to_cpu(sb->sq_psn); + qp->max_dest_rd_atomic = + IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); + qp->sq.max_wqe = qp->sq.hwq.max_elements; + qp->rq.max_wqe = qp->rq.hwq.max_elements; + qp->sq.max_sge = le16_to_cpu(sb->sq_sge); + qp->rq.max_sge = le16_to_cpu(sb->rq_sge); + qp->max_inline_data = le32_to_cpu(sb->max_inline_data); + qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); + memcpy(qp->smac, sb->src_mac, ETH_ALEN); + qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); + qp->port_id = le16_to_cpu(sb->port_id); +bail: + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + return rc; +} + + +static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) +{ + struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; + u32 peek_flags, peek_cons; + struct cq_base *hw_cqe; + int i; + + peek_flags = cq->dbinfo.flags; + peek_cons = cq_hwq->cons; + for (i = 0; i < cq_hwq->depth; i++) { + hw_cqe = bnxt_qplib_get_qe(cq_hwq, peek_cons, NULL); + if (CQE_CMP_VALID(hw_cqe, peek_flags)) { + dma_rmb(); + switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { + case CQ_BASE_CQE_TYPE_REQ: + case CQ_BASE_CQE_TYPE_TERMINAL: + { + struct cq_req *cqe = (struct cq_req *)hw_cqe; + + if (qp == le64_to_cpu(cqe->qp_handle)) + cqe->qp_handle = 0; + break; + } + case CQ_BASE_CQE_TYPE_RES_RC: + case CQ_BASE_CQE_TYPE_RES_UD: + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: + { + struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe; + + if (qp == le64_to_cpu(cqe->qp_handle)) + cqe->qp_handle = 0; + break; + } + default: + break; + } + } + bnxt_qplib_hwq_incr_cons(cq_hwq->depth, &peek_cons, + 1, &peek_flags); + } +} + +int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct bnxt_qplib_reftbl *tbl; + unsigned long flags; + u32 qp_idx; + int rc; + + tbl = &res->reftbl.qpref; + qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl); + spin_lock_irqsave(&tbl->lock, flags); + tbl->rec[qp_idx].xid = BNXT_QPLIB_QP_ID_INVALID; + tbl->rec[qp_idx].handle = NULL; + spin_unlock_irqrestore(&tbl->lock, flags); + + rc = __qplib_destroy_qp(rcfw, qp); + if (rc) { + spin_lock_irqsave(&tbl->lock, flags); + tbl->rec[qp_idx].xid = qp->id; + tbl->rec[qp_idx].handle = qp; + spin_unlock_irqrestore(&tbl->lock, flags); + return rc; + } + + return 0; +} + +void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp) +{ + if (qp->irrq.max_elements) + bnxt_qplib_free_hwq(res, &qp->irrq); + if (qp->orrq.max_elements) + bnxt_qplib_free_hwq(res, &qp->orrq); + + if (!qp->is_user) + kfree(qp->rq.swq); + bnxt_qplib_free_hwq(res, &qp->rq.hwq); + + if (!qp->is_user) + kfree(qp->sq.swq); + bnxt_qplib_free_hwq(res, &qp->sq.hwq); +} + +void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge) +{ + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_hdrbuf *buf; + u32 sw_prod; + + memset(sge, 0, sizeof(*sge)); + + buf = qp->sq_hdr_buf; + if (buf) { + sw_prod = sq->swq_start; + sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step); + sge->lkey = 0xFFFFFFFF; + sge->size = buf->step; + return buf->va + sw_prod * sge->size; + } + return NULL; +} + +u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_q *rq = &qp->rq; + + return rq->swq_start; +} + +void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge) +{ + struct bnxt_qplib_q *rq = &qp->rq; + struct bnxt_qplib_hdrbuf *buf; + u32 sw_prod; + + memset(sge, 0, sizeof(*sge)); + + buf = qp->rq_hdr_buf; + if (buf) { + sw_prod = rq->swq_start; + sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step); + sge->lkey = 0xFFFFFFFF; + sge->size = buf->step; + return buf->va + sw_prod * sge->size; + } + return NULL; +} + +/* Fil the MSN table into the next psn row */ +static void bnxt_qplib_fill_msn_search(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe, + struct bnxt_qplib_swq *swq) +{ + struct sq_msn_search *msns; + u32 start_psn, next_psn; + u16 start_idx; + + msns = (struct sq_msn_search *)swq->psn_search; + msns->start_idx_next_psn_start_psn = 0; + + start_psn = swq->start_psn; + next_psn = swq->next_psn; + start_idx = swq->slot_idx; + msns->start_idx_next_psn_start_psn |= + bnxt_re_update_msn_tbl(start_idx, next_psn, start_psn); + pr_debug("QP_LIB MSN %d START_IDX %u NEXT_PSN %u START_PSN %u\n", + qp->msn, + (u16) + cpu_to_le16(BNXT_RE_MSN_IDX(msns->start_idx_next_psn_start_psn)), + (u32) + cpu_to_le32(BNXT_RE_MSN_NPSN(msns->start_idx_next_psn_start_psn)), + (u32) + cpu_to_le32(BNXT_RE_MSN_SPSN(msns->start_idx_next_psn_start_psn))); + qp->msn++; + qp->msn %= qp->msn_tbl_sz; +} + +static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe, + struct bnxt_qplib_swq *swq) +{ + struct sq_psn_search_ext *psns_ext; + struct sq_psn_search *psns; + u32 flg_npsn; + u32 op_spsn; + + if (!swq->psn_search) + return; + + /* Handle MSN differently on cap flags */ + if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + bnxt_qplib_fill_msn_search(qp, wqe, swq); + return; + } + psns = (struct sq_psn_search *)swq->psn_search; + psns_ext = (struct sq_psn_search_ext *)swq->psn_search; + + op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & + SQ_PSN_SEARCH_START_PSN_MASK); + op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & + SQ_PSN_SEARCH_OPCODE_MASK); + flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & + SQ_PSN_SEARCH_NEXT_PSN_MASK); + + if (_is_chip_gen_p5_p7(qp->cctx)) { + psns_ext->opcode_start_psn = cpu_to_le32(op_spsn); + psns_ext->flags_next_psn = cpu_to_le32(flg_npsn); + psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx); + } else { + psns->opcode_start_psn = cpu_to_le32(op_spsn); + psns->flags_next_psn = cpu_to_le32(flg_npsn); + } +} + +static u16 _calc_ilsize(struct bnxt_qplib_swqe *wqe) +{ + u16 size = 0; + int indx; + + for (indx = 0; indx < wqe->num_sge; indx++) + size += wqe->sg_list[indx].size; + return size; +} + +static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe, + u32 *sw_prod) +{ + struct bnxt_qplib_hwq *sq_hwq; + int len, t_len, offt = 0; + int t_cplen = 0, cplen; + bool pull_dst = true; + void *il_dst = NULL; + void *il_src = NULL; + int indx; + + sq_hwq = &qp->sq.hwq; + t_len = 0; + for (indx = 0; indx < wqe->num_sge; indx++) { + len = wqe->sg_list[indx].size; + il_src = (void *)wqe->sg_list[indx].addr; + t_len += len; + if (t_len > qp->max_inline_data) + goto bad; + while (len) { + if (pull_dst) { + pull_dst = false; + il_dst = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % + sq_hwq->depth), NULL); + (*sw_prod)++; + t_cplen = 0; + offt = 0; + } + cplen = min_t(int, len, sizeof(struct sq_sge)); + cplen = min_t(int, cplen, + (sizeof(struct sq_sge) - offt)); + memcpy(il_dst, il_src, cplen); + t_cplen += cplen; + il_src += cplen; + il_dst += cplen; + offt += cplen; + len -= cplen; + if (t_cplen == sizeof(struct sq_sge)) + pull_dst = true; + } + } + + return t_len; +bad: + return -ENOMEM; +} + +static int bnxt_qplib_put_sges(struct bnxt_qplib_hwq *sq_hwq, + struct bnxt_qplib_sge *ssge, + u32 nsge, u32 *sw_prod) +{ + struct sq_sge *dsge; + int indx, len = 0; + + for (indx = 0; indx < nsge; indx++, (*sw_prod)++) { + dsge = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % sq_hwq->depth), NULL); + dsge->va_or_pa = cpu_to_le64(ssge[indx].addr); + dsge->l_key = cpu_to_le32(ssge[indx].lkey); + dsge->size = cpu_to_le32(ssge[indx].size); + len += ssge[indx].size; + } + return len; +} + +static u16 _calculate_wqe_byte(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe, u16 *wqe_byte) +{ + u16 wqe_size; + u32 ilsize; + u16 nsge; + + nsge = wqe->num_sge; + if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { + ilsize = _calc_ilsize(wqe); + wqe_size = (ilsize > qp->max_inline_data) ? + qp->max_inline_data : ilsize; + wqe_size = ALIGN(wqe_size, sizeof(struct sq_sge)); + } else { + wqe_size = nsge * sizeof(struct sq_sge); + } + /* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */ + wqe_size += sizeof(struct sq_send_hdr); + if (wqe_byte) + *wqe_byte = wqe_size; + return wqe_size / sizeof(struct sq_sge); +} + +static u16 _translate_q_full_delta(struct bnxt_qplib_q *que, u16 wqe_bytes) +{ + /* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128 + * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128. + * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512. + * when 8916 is disabled. + */ + return (que->q_full_delta * wqe_bytes) / que->hwq.element_size; +} + +static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib_q *sq, + struct bnxt_qplib_swq *swq, bool hw_retx) +{ + struct bnxt_qplib_hwq *sq_hwq; + u32 pg_num, pg_indx; + void *buff; + u32 tail; + + sq_hwq = &sq->hwq; + if (!sq_hwq->pad_pg) + return; + + tail = swq->slot_idx / sq->dbinfo.max_slot; + if (hw_retx) + tail %= qp->msn_tbl_sz; + pg_num = (tail + sq_hwq->pad_pgofft) / (PAGE_SIZE / sq_hwq->pad_stride); + pg_indx = (tail + sq_hwq->pad_pgofft) % (PAGE_SIZE / sq_hwq->pad_stride); + buff = (void *)(sq_hwq->pad_pg[pg_num] + pg_indx * sq_hwq->pad_stride); + /* the start ptr for buff is same ie after the SQ */ + swq->psn_search = buff; +} + +void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_q *sq = &qp->sq; + + bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ); +} + +int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_qplib_nq_work *nq_work = NULL; + int i, rc = 0, data_len = 0, pkt_num = 0; + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_hwq *sq_hwq; + struct bnxt_qplib_swq *swq; + bool sch_handler = false; + u16 slots_needed; + void *base_hdr; + void *ext_hdr; + __le32 temp32; + u16 qfd_slots; + u8 wqe_slots; + u16 wqe_size; + u32 sw_prod; + u32 wqe_idx; + + sq_hwq = &sq->hwq; + if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS && + qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) { + dev_err(&sq_hwq->pdev->dev, + "QPLIB: FP: QP (0x%x) is in the 0x%x state\n", + qp->id, qp->state); + rc = -EINVAL; + goto done; + } + + wqe_slots = _calculate_wqe_byte(qp, wqe, &wqe_size); + slots_needed = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + sq->dbinfo.max_slot : wqe_slots; + qfd_slots = _translate_q_full_delta(sq, wqe_size); + if (bnxt_qplib_queue_full(sq_hwq, (slots_needed + qfd_slots))) { + dev_err(&sq_hwq->pdev->dev, + "QPLIB: FP: QP (0x%x) SQ is full!\n", qp->id); + dev_err(&sq_hwq->pdev->dev, + "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x slots = %#x\n", + HWQ_CMP(sq_hwq->prod, sq_hwq), + HWQ_CMP(sq_hwq->cons, sq_hwq), + sq_hwq->max_elements, qfd_slots, slots_needed); + dev_err(&sq_hwq->pdev->dev, + "QPLIB: phantom_wqe_cnt: %d phantom_cqe_cnt: %d\n", + sq->phantom_wqe_cnt, sq->phantom_cqe_cnt); + rc = -ENOMEM; + goto done; + } + + sw_prod = sq_hwq->prod; + swq = bnxt_qplib_get_swqe(sq, &wqe_idx); + swq->slot_idx = sw_prod; + bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags)); + + swq->wr_id = wqe->wr_id; + swq->type = wqe->type; + swq->flags = wqe->flags; + swq->slots = slots_needed; + swq->start_psn = sq->psn & BTH_PSN_MASK; + if (qp->sig_type || wqe->flags & BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP) + swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; + + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: QP(0x%x) post SQ wr_id[%d] = 0x%llx\n", + qp->id, wqe_idx, swq->wr_id); + if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { + sch_handler = true; + dev_dbg(&sq_hwq->pdev->dev, + "%s Error QP. Scheduling for poll_cq\n", __func__); + goto queue_err; + } + + base_hdr = bnxt_qplib_get_qe(sq_hwq, sw_prod, NULL); + sw_prod++; + ext_hdr = bnxt_qplib_get_qe(sq_hwq, (sw_prod % sq_hwq->depth), NULL); + sw_prod++; + memset(base_hdr, 0, sizeof(struct sq_sge)); + memset(ext_hdr, 0, sizeof(struct sq_sge)); + + if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) + data_len = bnxt_qplib_put_inline(qp, wqe, &sw_prod); + else + data_len = bnxt_qplib_put_sges(sq_hwq, wqe->sg_list, + wqe->num_sge, &sw_prod); + if (data_len < 0) + goto queue_err; + /* Specifics */ + switch (wqe->type) { + case BNXT_QPLIB_SWQE_TYPE_SEND: + if (qp->type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE || + qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { + /* Assemble info for Raw Ethertype QPs */ + struct sq_send_raweth_qp1_hdr *sqe = base_hdr; + struct sq_raw_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->wqe_size = wqe_slots; + sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); + sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); + sqe->length = cpu_to_le32(data_len); + ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & + SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << + SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); + + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: RAW/QP1 Send WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\twqe_size = 0x%x\n" + "\tlflags = 0x%x\n" + "\tcfa_action = 0x%x\n" + "\tlength = 0x%x\n" + "\tcfa_meta = 0x%x\n", + sqe->wqe_type, sqe->flags, sqe->wqe_size, + sqe->lflags, sqe->cfa_action, + sqe->length, ext_sqe->cfa_meta); + break; + } + fallthrough; + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: + fallthrough; + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: + { + struct sq_send_hdr *sqe = base_hdr; + struct sq_ud_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->wqe_size = wqe_slots; + sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key); + if (qp->type == CMDQ_CREATE_QP_TYPE_UD || + qp->type == CMDQ_CREATE_QP_TYPE_GSI) { + sqe->q_key = cpu_to_le32(wqe->send.q_key); + sqe->length = cpu_to_le32(data_len); + ext_sqe->dst_qp = cpu_to_le32( + wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); + ext_sqe->avid = cpu_to_le32(wqe->send.avid & + SQ_SEND_AVID_MASK); + sq->psn = (sq->psn + 1) & BTH_PSN_MASK; + } else { + sqe->length = cpu_to_le32(data_len); + if (qp->mtu) + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; + if (!pkt_num) + pkt_num = 1; + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; + } + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: Send WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\twqe_size = 0x%x\n" + "\tinv_key/immdata = 0x%x\n" + "\tq_key = 0x%x\n" + "\tdst_qp = 0x%x\n" + "\tlength = 0x%x\n" + "\tavid = 0x%x\n", + sqe->wqe_type, sqe->flags, sqe->wqe_size, + sqe->inv_key_or_imm_data, sqe->q_key, ext_sqe->dst_qp, + sqe->length, ext_sqe->avid); + break; + } + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: + /* fall-thru */ + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: + /* fall-thru */ + case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: + { + struct sq_rdma_hdr *sqe = base_hdr; + struct sq_rdma_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->wqe_size = wqe_slots; + sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); + sqe->length = cpu_to_le32((u32)data_len); + ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); + ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); + if (qp->mtu) + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; + if (!pkt_num) + pkt_num = 1; + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; + + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: RDMA WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\twqe_size = 0x%x\n" + "\timmdata = 0x%x\n" + "\tlength = 0x%x\n" + "\tremote_va = 0x%llx\n" + "\tremote_key = 0x%x\n", + sqe->wqe_type, sqe->flags, sqe->wqe_size, + sqe->imm_data, sqe->length, ext_sqe->remote_va, + ext_sqe->remote_key); + break; + } + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: + /* fall-thru */ + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: + { + struct sq_atomic_hdr *sqe = base_hdr; + struct sq_atomic_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); + sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); + ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); + ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); + if (qp->mtu) + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; + if (!pkt_num) + pkt_num = 1; + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; + break; + } + case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: + { + struct sq_localinvalidate_hdr *sqe = base_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); + + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: LOCAL INV WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\tinv_l_key = 0x%x\n", + sqe->wqe_type, sqe->flags, sqe->inv_l_key); + break; + } + case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: + { + struct sq_fr_pmr_hdr *sqe = base_hdr; + struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->access_cntl = wqe->frmr.access_cntl | + SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; + sqe->zero_based_page_size_log = + (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << + SQ_FR_PMR_PAGE_SIZE_LOG_SFT | + (wqe->frmr.zero_based == true ? SQ_FR_PMR_ZERO_BASED : 0); + sqe->l_key = cpu_to_le32(wqe->frmr.l_key); + /* TODO: OFED only provides length of MR up to 32-bits for FRMR */ + temp32 = cpu_to_le32(wqe->frmr.length); + memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); + sqe->numlevels_pbl_page_size_log = + ((wqe->frmr.pbl_pg_sz_log << + SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) & + SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) | + ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & + SQ_FR_PMR_NUMLEVELS_MASK); + if (!wqe->frmr.levels && !wqe->frmr.pbl_ptr) { + ext_sqe->pblptr = cpu_to_le64(wqe->frmr.page_list[0]); + } else { + for (i = 0; i < wqe->frmr.page_list_len; i++) + wqe->frmr.pbl_ptr[i] = cpu_to_le64( + wqe->frmr.page_list[i] | + PTU_PTE_VALID); + ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); + } + ext_sqe->va = cpu_to_le64(wqe->frmr.va); + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: FRMR WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\taccess_cntl = 0x%x\n" + "\tzero_based_page_size_log = 0x%x\n" + "\tl_key = 0x%x\n" + "\tlength = 0x%x\n" + "\tnumlevels_pbl_page_size_log = 0x%x\n" + "\tpblptr = 0x%llx\n" + "\tva = 0x%llx\n", + sqe->wqe_type, sqe->flags, sqe->access_cntl, + sqe->zero_based_page_size_log, sqe->l_key, + *(u32 *)sqe->length, sqe->numlevels_pbl_page_size_log, + ext_sqe->pblptr, ext_sqe->va); + break; + } + case BNXT_QPLIB_SWQE_TYPE_BIND_MW: + { + struct sq_bind_hdr *sqe = base_hdr; + struct sq_bind_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->access_cntl = wqe->bind.access_cntl; + sqe->mw_type_zero_based = wqe->bind.mw_type | + (wqe->bind.zero_based == true ? SQ_BIND_ZERO_BASED : 0); + sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); + sqe->l_key = cpu_to_le32(wqe->bind.r_key); + ext_sqe->va = cpu_to_le64(wqe->bind.va); + ext_sqe->length_lo = cpu_to_le32(wqe->bind.length); + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: BIND WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\taccess_cntl = 0x%x\n" + "\tmw_type_zero_based = 0x%x\n" + "\tparent_l_key = 0x%x\n" + "\tl_key = 0x%x\n" + "\tva = 0x%llx\n" + "\tlength = 0x%x\n", + sqe->wqe_type, sqe->flags, sqe->access_cntl, + sqe->mw_type_zero_based, sqe->parent_l_key, + sqe->l_key, sqe->va, ext_sqe->length_lo); + break; + } + default: + /* Bad wqe, return error */ + rc = -EINVAL; + goto done; + } + swq->next_psn = sq->psn & BTH_PSN_MASK; + bnxt_qplib_fill_psn_search(qp, wqe, swq); + +queue_err: + bnxt_qplib_swq_mod_start(sq, wqe_idx); + bnxt_qplib_hwq_incr_prod(&sq->dbinfo, sq_hwq, swq->slots); + qp->wqe_cnt++; +done: + if (sch_handler) { + nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); + if (nq_work) { + nq_work->cq = qp->scq; + nq_work->nq = qp->scq->nq; + INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); + queue_work(qp->scq->nq->cqn_wq, &nq_work->work); + } else { + dev_err(&sq->hwq.pdev->dev, + "QPLIB: FP: Failed to allocate SQ nq_work!\n"); + rc = -ENOMEM; + } + } + return rc; +} + +void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_q *rq = &qp->rq; + + bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ); +} + +void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_nq *nq; + + if (!(cq && cq->nq)) + return; + + nq = cq->nq; + spin_lock_bh(&cq->compl_lock); + if (nq->cqn_handler) { + dev_dbg(&nq->res->pdev->dev, + "%s:Trigger cq = %p event nq = %p\n", + __func__, cq, nq); + nq->cqn_handler(nq, cq); + } + spin_unlock_bh(&cq->compl_lock); +} + +int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_qplib_nq_work *nq_work = NULL; + struct bnxt_qplib_q *rq = &qp->rq; + struct bnxt_qplib_hwq *rq_hwq; + struct bnxt_qplib_swq *swq; + bool sch_handler = false; + struct rq_wqe_hdr *base_hdr; + struct rq_ext_hdr *ext_hdr; + struct sq_sge *dsge; + u8 wqe_slots; + u32 wqe_idx; + u32 sw_prod; + int rc = 0; + + rq_hwq = &rq->hwq; + if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { + dev_err(&rq_hwq->pdev->dev, + "QPLIB: FP: QP (0x%x) is in the 0x%x state\n", + qp->id, qp->state); + rc = -EINVAL; + goto done; + } + + wqe_slots = _calculate_wqe_byte(qp, wqe, NULL); + if (bnxt_qplib_queue_full(rq_hwq, rq->dbinfo.max_slot)) { + dev_err(&rq_hwq->pdev->dev, + "QPLIB: FP: QP (0x%x) RQ is full!\n", qp->id); + rc = -EINVAL; + goto done; + } + + swq = bnxt_qplib_get_swqe(rq, &wqe_idx); + swq->wr_id = wqe->wr_id; + swq->slots = rq->dbinfo.max_slot; + dev_dbg(&rq_hwq->pdev->dev, + "QPLIB: FP: post RQ wr_id[%d] = 0x%llx\n", + wqe_idx, swq->wr_id); + if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { + sch_handler = true; + dev_dbg(&rq_hwq->pdev->dev, "%s Error QP. Sched a flushed cmpl\n", + __func__); + goto queue_err; + } + + sw_prod = rq_hwq->prod; + base_hdr = bnxt_qplib_get_qe(rq_hwq, sw_prod, NULL); + sw_prod++; + ext_hdr = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL); + sw_prod++; + memset(base_hdr, 0, sizeof(struct sq_sge)); + memset(ext_hdr, 0, sizeof(struct sq_sge)); + + if (!wqe->num_sge) { + dsge = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL); + dsge->size = 0; + wqe_slots++; + } else { + bnxt_qplib_put_sges(rq_hwq, wqe->sg_list, wqe->num_sge, &sw_prod); + } + base_hdr->wqe_type = wqe->type; + base_hdr->flags = wqe->flags; + base_hdr->wqe_size = wqe_slots; + base_hdr->wr_id |= cpu_to_le32(wqe_idx); +queue_err: + bnxt_qplib_swq_mod_start(rq, wqe_idx); + bnxt_qplib_hwq_incr_prod(&rq->dbinfo, &rq->hwq, swq->slots); +done: + if (sch_handler) { + nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); + if (nq_work) { + nq_work->cq = qp->rcq; + nq_work->nq = qp->rcq->nq; + INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); + queue_work(qp->rcq->nq->cqn_wq, &nq_work->work); + } else { + dev_err(&rq->hwq.pdev->dev, + "QPLIB: FP: Failed to allocate RQ nq_work!\n"); + rc = -ENOMEM; + } + } + return rc; +} + +/* CQ */ +int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_create_cq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_create_cq req = {}; + struct bnxt_qplib_reftbl *tbl; + unsigned long flag; + u32 pg_sz_lvl = 0; + int rc; + + hwq_attr.res = res; + hwq_attr.depth = cq->max_wqe; + hwq_attr.stride = sizeof(struct cq_base); + hwq_attr.type = HWQ_TYPE_QUEUE; + hwq_attr.sginfo = &cq->sginfo; + rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr); + if (rc) + goto exit; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_CQ, + sizeof(req)); + + if (!cq->dpi) { + dev_err(&rcfw->pdev->dev, + "QPLIB: FP: CREATE_CQ failed due to NULL DPI\n"); + return -EINVAL; + } + req.dpi = cpu_to_le32(cq->dpi->dpi); + req.cq_handle = cpu_to_le64(cq->cq_handle); + + req.cq_size = cpu_to_le32(cq->max_wqe); + req.pbl = cpu_to_le64(_get_base_addr(&cq->hwq)); + pg_sz_lvl = _get_base_pg_size(&cq->hwq) << CMDQ_CREATE_CQ_PG_SIZE_SFT; + pg_sz_lvl |= ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) << + CMDQ_CREATE_CQ_LVL_SFT); + req.pg_size_lvl = cpu_to_le32(pg_sz_lvl); + + req.cq_fco_cnq_id = cpu_to_le32( + (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << + CMDQ_CREATE_CQ_CNQ_ID_SFT); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail; + cq->id = le32_to_cpu(resp.xid); + cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; + init_waitqueue_head(&cq->waitq); + INIT_LIST_HEAD(&cq->sqf_head); + INIT_LIST_HEAD(&cq->rqf_head); + spin_lock_init(&cq->flush_lock); + spin_lock_init(&cq->compl_lock); + + /* init dbinfo */ + cq->cctx = res->cctx; + cq->dbinfo.hwq = &cq->hwq; + cq->dbinfo.xid = cq->id; + cq->dbinfo.db = cq->dpi->dbr; + cq->dbinfo.priv_db = res->dpi_tbl.priv_db; + cq->dbinfo.flags = 0; + cq->dbinfo.toggle = 0; + cq->dbinfo.res = res; + cq->dbinfo.seed = cq->id; + spin_lock_init(&cq->dbinfo.lock); + cq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + cq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID; + + tbl = &res->reftbl.cqref; + spin_lock_irqsave(&tbl->lock, flag); + tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = cq->id; + tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = cq; + spin_unlock_irqrestore(&tbl->lock, flag); + + bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA); + return 0; + +fail: + bnxt_qplib_free_hwq(res, &cq->hwq); +exit: + return rc; +} + +int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) +{ + /* TODO: Modify CQ threshold are passed to the HW via DBR */ + return 0; +} + +void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res, + struct bnxt_qplib_cq *cq) +{ + bnxt_qplib_free_hwq(res, &cq->hwq); + memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq)); + /* Reset only the cons bit in the flags */ + cq->dbinfo.flags &= ~(1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT); + + /* Tell HW to switch over to the new CQ */ + if (!cq->resize_hwq.is_user) + bnxt_qplib_cq_coffack_db(&cq->dbinfo); +} + +int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq, + int new_cqes) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_resize_cq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_resize_cq req = {}; + u32 pgsz = 0, lvl = 0, nsz = 0; + struct bnxt_qplib_pbl *pbl; + u16 count = -1; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_RESIZE_CQ, + sizeof(req)); + + hwq_attr.sginfo = &cq->sginfo; + hwq_attr.res = res; + hwq_attr.depth = new_cqes; + hwq_attr.stride = sizeof(struct cq_base); + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&cq->resize_hwq, &hwq_attr); + if (rc) + return rc; + + dev_dbg(&rcfw->pdev->dev, "QPLIB: FP: %s: pbl_lvl: %d\n", __func__, + cq->resize_hwq.level); + req.cq_cid = cpu_to_le32(cq->id); + pbl = &cq->resize_hwq.pbl[PBL_LVL_0]; + pgsz = ((pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_4K : + pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8K : + pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_64K : + pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_2M : + pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8M : + pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_RESIZE_CQ_PG_SIZE_PG_1G : + CMDQ_RESIZE_CQ_PG_SIZE_PG_4K) & CMDQ_RESIZE_CQ_PG_SIZE_MASK); + lvl = (cq->resize_hwq.level << CMDQ_RESIZE_CQ_LVL_SFT) & + CMDQ_RESIZE_CQ_LVL_MASK; + nsz = (new_cqes << CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT) & + CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK; + req.new_cq_size_pg_size_lvl = cpu_to_le32(nsz|pgsz|lvl); + req.new_pbl = cpu_to_le64(pbl->pg_map_arr[0]); + + if (!cq->resize_hwq.is_user) + set_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail; + + if (!cq->resize_hwq.is_user) { +wait: + /* Wait here for the HW to switch the CQ over */ + if (wait_event_interruptible_timeout(cq->waitq, + !test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags), + msecs_to_jiffies(CQ_RESIZE_WAIT_TIME_MS)) == + -ERESTARTSYS && count--) + goto wait; + + if (test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags)) { + dev_err(&rcfw->pdev->dev, + "QPLIB: FP: RESIZE_CQ timed out\n"); + rc = -ETIMEDOUT; + goto fail; + } + + bnxt_qplib_resize_cq_complete(res, cq); + } + + return 0; +fail: + if (!cq->resize_hwq.is_user) { + bnxt_qplib_free_hwq(res, &cq->resize_hwq); + clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); + } + return rc; +} + +void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) +{ + bnxt_qplib_free_hwq(res, &cq->hwq); +} + +static void bnxt_qplib_sync_cq(struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_nq *nq = cq->nq; + /* Flush any pending work and synchronize irq */ + flush_workqueue(cq->nq->cqn_wq); + mutex_lock(&nq->lock); + if (nq->requested) + synchronize_irq(nq->msix_vec); + mutex_unlock(&nq->lock); +} + +int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_destroy_cq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_destroy_cq req = {}; + struct bnxt_qplib_reftbl *tbl; + u16 total_cnq_events; + unsigned long flag; + int rc; + + tbl = &res->reftbl.cqref; + spin_lock_irqsave(&tbl->lock, flag); + tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = NULL; + tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = 0; + spin_unlock_irqrestore(&tbl->lock, flag); + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_CQ, + sizeof(req)); + + req.cq_cid = cpu_to_le32(cq->id); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + + total_cnq_events = le16_to_cpu(resp.total_cnq_events); + if (total_cnq_events >= 0) + dev_dbg(&rcfw->pdev->dev, + "%s: cq_id = 0x%x cq = 0x%p resp.total_cnq_events = 0x%x\n", + __func__, cq->id, cq, total_cnq_events); + __wait_for_all_nqes(cq, total_cnq_events); + bnxt_qplib_sync_cq(cq); + bnxt_qplib_free_hwq(res, &cq->hwq); + return 0; +} + +static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, + struct bnxt_qplib_cqe **pcqe, int *budget) +{ + struct bnxt_qplib_cqe *cqe; + u32 start, last; + int rc = 0; + + /* Now complete all outstanding SQEs with FLUSHED_ERR */ + start = sq->swq_start; + cqe = *pcqe; + while (*budget) { + last = sq->swq_last; + if (start == last) { + break; + } + /* Skip the FENCE WQE completions */ + if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) { + bnxt_re_legacy_cancel_phantom_processing(qp); + goto skip_compl; + } + + memset(cqe, 0, sizeof(*cqe)); + cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; + cqe->qp_handle = (u64)qp; + cqe->wr_id = sq->swq[last].wr_id; + cqe->src_qp = qp->id; + cqe->type = sq->swq[last].type; + dev_dbg(&sq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed terminal Req \n"); + dev_dbg(&sq->hwq.pdev->dev, + "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n", + last, cqe->wr_id, cqe->status); + cqe++; + (*budget)--; +skip_compl: + bnxt_qplib_hwq_incr_cons(sq->hwq.depth, + &sq->hwq.cons, + sq->swq[last].slots, + &sq->dbinfo.flags); + sq->swq_last = sq->swq[last].next_idx; + } + *pcqe = cqe; + if (!*budget && sq->swq_last != start) + /* Out of budget */ + rc = -EAGAIN; + dev_dbg(&sq->hwq.pdev->dev, "QPLIB: FP: Flush SQ rc = 0x%x\n", rc); + + return rc; +} + +static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, + struct bnxt_qplib_cqe **pcqe, int *budget) +{ + struct bnxt_qplib_cqe *cqe; + u32 start, last; + int opcode = 0; + int rc = 0; + + switch (qp->type) { + case CMDQ_CREATE_QP1_TYPE_GSI: + opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1; + break; + case CMDQ_CREATE_QP_TYPE_RC: + opcode = CQ_BASE_CQE_TYPE_RES_RC; + break; + case CMDQ_CREATE_QP_TYPE_UD: + opcode = CQ_BASE_CQE_TYPE_RES_UD; + break; + } + + /* Flush the rest of the RQ */ + start = rq->swq_start; + cqe = *pcqe; + while (*budget) { + last = rq->swq_last; + if (last == start) + break; + memset(cqe, 0, sizeof(*cqe)); + cqe->status = + CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; + cqe->opcode = opcode; + cqe->qp_handle = (u64)qp; + cqe->wr_id = rq->swq[last].wr_id; + dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Res RC \n"); + dev_dbg(&rq->hwq.pdev->dev, + "QPLIB: rq[%d] = 0x%llx with status 0x%x\n", + last, cqe->wr_id, cqe->status); + cqe++; + (*budget)--; + bnxt_qplib_hwq_incr_cons(rq->hwq.depth, + &rq->hwq.cons, + rq->swq[last].slots, + &rq->dbinfo.flags); + rq->swq_last = rq->swq[last].next_idx; + } + *pcqe = cqe; + if (!*budget && rq->swq_last != start) + /* Out of budget */ + rc = -EAGAIN; + + dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: Flush RQ rc = 0x%x\n", rc); + return rc; +} + +void bnxt_qplib_mark_qp_error(void *qp_handle) +{ + struct bnxt_qplib_qp *qp = qp_handle; + + if (!qp) + return; + + /* Must block new posting of SQ and RQ */ + qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_ERR; + qp->state = qp->cur_qp_state; + + /* Add qp to flush list of the CQ */ + if (!qp->is_user) + bnxt_qplib_add_flush_qp(qp); +} + +/* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive) + * CQE is track from sw_cq_cons to max_element but valid only if VALID=1 + */ +static int bnxt_re_legacy_do_wa9060(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_cq *cq, + u32 cq_cons, u32 swq_last, + u32 cqe_sq_cons) +{ + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_swq *swq; + u32 peek_sw_cq_cons, peek_sq_cons_idx, peek_flags; + struct cq_terminal *peek_term_hwcqe; + struct cq_req *peek_req_hwcqe; + struct bnxt_qplib_qp *peek_qp; + struct bnxt_qplib_q *peek_sq; + struct cq_base *peek_hwcqe; + int i, rc = 0; + + /* Check for the psn_search marking before completing */ + swq = &sq->swq[swq_last]; + if (swq->psn_search && + le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { + /* Unmark */ + swq->psn_search->flags_next_psn = cpu_to_le32 + (le32_to_cpu(swq->psn_search->flags_next_psn) + & ~0x80000000); + dev_dbg(&cq->hwq.pdev->dev, + "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n", + cq_cons, qp->id, swq_last, cqe_sq_cons); + sq->condition = true; + sq->legacy_send_phantom = true; + + /* TODO: Only ARM if the previous SQE is ARMALL */ + bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL); + + rc = -EAGAIN; + goto out; + } + if (sq->condition == true) { + /* Peek at the completions */ + peek_flags = cq->dbinfo.flags; + peek_sw_cq_cons = cq_cons; + i = cq->hwq.depth; + while (i--) { + peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq, + peek_sw_cq_cons, NULL); + /* If the next hwcqe is VALID */ + if (CQE_CMP_VALID(peek_hwcqe, peek_flags)) { + /* If the next hwcqe is a REQ */ + dma_rmb(); + switch (peek_hwcqe->cqe_type_toggle & + CQ_BASE_CQE_TYPE_MASK) { + case CQ_BASE_CQE_TYPE_REQ: + peek_req_hwcqe = (struct cq_req *) + peek_hwcqe; + peek_qp = (struct bnxt_qplib_qp *) + le64_to_cpu( + peek_req_hwcqe->qp_handle); + peek_sq = &peek_qp->sq; + peek_sq_cons_idx = + ((le16_to_cpu( + peek_req_hwcqe->sq_cons_idx) + - 1) % sq->max_wqe); + /* If the hwcqe's sq's wr_id matches */ + if (peek_sq == sq && + sq->swq[peek_sq_cons_idx].wr_id == + BNXT_QPLIB_FENCE_WRID) { + /* Unbreak only if the phantom + comes back */ + dev_dbg(&cq->hwq.pdev->dev, + "FP: Process Req qp=0x%x current sq cons sw=0x%x cqe=0x%x\n", + qp->id, swq_last, + cqe_sq_cons); + sq->condition = false; + sq->single = true; + sq->phantom_cqe_cnt++; + dev_dbg(&cq->hwq.pdev->dev, + "qp %#x condition restored at peek cq_cons=%#x sq_cons_idx %#x, phantom_cqe_cnt: %d unmark\n", + peek_qp->id, + peek_sw_cq_cons, + peek_sq_cons_idx, + sq->phantom_cqe_cnt); + rc = 0; + goto out; + } + break; + + case CQ_BASE_CQE_TYPE_TERMINAL: + /* In case the QP has gone into the + error state */ + peek_term_hwcqe = (struct cq_terminal *) + peek_hwcqe; + peek_qp = (struct bnxt_qplib_qp *) + le64_to_cpu( + peek_term_hwcqe->qp_handle); + if (peek_qp == qp) { + sq->condition = false; + rc = 0; + goto out; + } + break; + default: + break; + } + /* Valid but not the phantom, so keep looping */ + } else { + /* Not valid yet, just exit and wait */ + rc = -EINVAL; + goto out; + } + bnxt_qplib_hwq_incr_cons(cq->hwq.depth, + &peek_sw_cq_cons, + 1, &peek_flags); + } + dev_err(&cq->hwq.pdev->dev, + "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n", + cq_cons, qp->id, swq_last, cqe_sq_cons); + rc = -EINVAL; + } +out: + return rc; +} + +static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, + struct cq_req *hwcqe, + struct bnxt_qplib_cqe **pcqe, int *budget, + u32 cq_cons, struct bnxt_qplib_qp **lib_qp) +{ + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *sq; + struct bnxt_qplib_cqe *cqe; + u32 cqe_sq_cons; + struct bnxt_qplib_swq *swq; + int rc = 0; + + qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); + dev_dbg(&cq->hwq.pdev->dev, "FP: Process Req qp=0x%p\n", qp); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: Process Req qp is NULL\n"); + return -EINVAL; + } + sq = &qp->sq; + + cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe; + if (qp->sq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + + /* Require to walk the sq's swq to fabricate CQEs for all previously + * signaled SWQEs due to CQE aggregation from the current sq cons + * to the cqe_sq_cons + */ + cqe = *pcqe; + while (*budget) { + if (sq->swq_last == cqe_sq_cons) + /* Done */ + break; + + swq = &sq->swq[sq->swq_last]; + memset(cqe, 0, sizeof(*cqe)); + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; + cqe->qp_handle = (u64)qp; + cqe->src_qp = qp->id; + cqe->wr_id = swq->wr_id; + + if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID) + goto skip; + + cqe->type = swq->type; + + /* For the last CQE, check for status. For errors, regardless + * of the request being signaled or not, it must complete with + * the hwcqe error status + */ + if (swq->next_idx == cqe_sq_cons && + hwcqe->status != CQ_REQ_STATUS_OK) { + cqe->status = hwcqe->status; + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed Req \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: QP 0x%x wr_id[%d] = 0x%lx vendor type 0x%x with vendor status 0x%x\n", + cqe->src_qp, sq->swq_last, cqe->wr_id, cqe->type, cqe->status); + cqe++; + (*budget)--; + bnxt_qplib_mark_qp_error(qp); + } else { + /* Before we complete, do WA 9060 */ + if (!_is_chip_gen_p5_p7(qp->cctx)) { + if (bnxt_re_legacy_do_wa9060(qp, cq, cq_cons, + sq->swq_last, + cqe_sq_cons)) { + *lib_qp = qp; + goto out; + } + } + if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) { + + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed Req \n"); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: wr_id[%d] = 0x%llx \n", + sq->swq_last, cqe->wr_id); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: with status 0x%x\n", cqe->status); + cqe->status = CQ_REQ_STATUS_OK; + cqe++; + (*budget)--; + } + } +skip: + bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons, + swq->slots, &sq->dbinfo.flags); + sq->swq_last = swq->next_idx; + if (sq->single == true) + break; + } +out: + *pcqe = cqe; + if (sq->swq_last != cqe_sq_cons) { + /* Out of budget */ + rc = -EAGAIN; + goto done; + } + /* Back to normal completion mode only after it has completed all of + the WC for this CQE */ + sq->single = false; +done: + return rc; +} + +static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag) +{ + spin_lock(&srq->hwq.lock); + srq->swq[srq->last_idx].next_idx = (int)tag; + srq->last_idx = (int)tag; + srq->swq[srq->last_idx].next_idx = -1; + bnxt_qplib_hwq_incr_cons(srq->hwq.depth, &srq->hwq.cons, + srq->dbinfo.max_slot, &srq->dbinfo.flags); + spin_unlock(&srq->hwq.lock); +} + +static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, + struct cq_res_rc *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_srq *srq; + struct bnxt_qplib_cqe *cqe; + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *rq; + u32 wr_id_idx; + int rc = 0; + + qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL\n"); + return -EINVAL; + } + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + + cqe = *pcqe; + cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; + cqe->length = le32_to_cpu(hwcqe->length); + cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); + cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); + cqe->flags = le16_to_cpu(hwcqe->flags); + cqe->status = hwcqe->status; + cqe->qp_handle = (u64)(unsigned long)qp; + + wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & + CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK; + if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) { + srq = qp->srq; + if (!srq) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: SRQ used but not defined??\n"); + return -EINVAL; + } + if (wr_id_idx > srq->hwq.depth - 1) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process RC \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", + wr_id_idx, srq->hwq.depth); + return -EINVAL; + } + cqe->wr_id = srq->swq[wr_id_idx].wr_id; + bnxt_qplib_release_srqe(srq, wr_id_idx); + dev_dbg(&srq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed RC SRQ wr_id[%d] = 0x%llx\n", + wr_id_idx, cqe->wr_id); + cqe++; + (*budget)--; + *pcqe = cqe; + } else { + rq = &qp->rq; + if (wr_id_idx > (rq->max_wqe - 1)) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process RC \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n", + wr_id_idx, rq->hwq.depth); + return -EINVAL; + } + if (wr_id_idx != rq->swq_last) + return -EINVAL; + cqe->wr_id = rq->swq[rq->swq_last].wr_id; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed RC RQ wr_id[%d] = 0x%llx\n", + rq->swq_last, cqe->wr_id); + cqe++; + (*budget)--; + bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, + rq->swq[rq->swq_last].slots, + &rq->dbinfo.flags); + rq->swq_last = rq->swq[rq->swq_last].next_idx; + *pcqe = cqe; + + if (hwcqe->status != CQ_RES_RC_STATUS_OK) + bnxt_qplib_mark_qp_error(qp); + } +done: + return rc; +} + +static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, + struct cq_res_ud_v2 *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_srq *srq; + struct bnxt_qplib_cqe *cqe; + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *rq; + u32 wr_id_idx; + int rc = 0; + u16 *smac; + + qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL\n"); + return -EINVAL; + } + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + cqe = *pcqe; + cqe->opcode = hwcqe->cqe_type_toggle & CQ_RES_UD_V2_CQE_TYPE_MASK; + cqe->length = le32_to_cpu((hwcqe->length & CQ_RES_UD_V2_LENGTH_MASK)); + cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata0); + /* V2 format has metadata1 */ + cqe->cfa_meta |= (((le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) & + CQ_RES_UD_V2_CFA_METADATA1_MASK) >> + CQ_RES_UD_V2_CFA_METADATA1_SFT) << + BNXT_QPLIB_META1_SHIFT); + cqe->invrkey = le32_to_cpu(hwcqe->imm_data); + cqe->flags = le16_to_cpu(hwcqe->flags); + cqe->status = hwcqe->status; + cqe->qp_handle = (u64)(unsigned long)qp; + smac = (u16 *)cqe->smac; + smac[2] = ntohs(le16_to_cpu(hwcqe->src_mac[0])); + smac[1] = ntohs(le16_to_cpu(hwcqe->src_mac[1])); + smac[0] = ntohs(le16_to_cpu(hwcqe->src_mac[2])); + wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) + & CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_MASK; + cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | + ((le32_to_cpu( + hwcqe->src_qp_high_srq_or_rq_wr_id) & + CQ_RES_UD_V2_SRC_QP_HIGH_MASK) >> 8); + + if (cqe->flags & CQ_RES_UD_V2_FLAGS_SRQ) { + srq = qp->srq; + if (!srq) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: SRQ used but not defined??\n"); + return -EINVAL; + } + if (wr_id_idx > srq->hwq.depth - 1) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process UD \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", + wr_id_idx, srq->hwq.depth); + return -EINVAL; + } + cqe->wr_id = srq->swq[wr_id_idx].wr_id; + bnxt_qplib_release_srqe(srq, wr_id_idx); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed UD SRQ wr_id[%d] = 0x%llx\n", + wr_id_idx, cqe->wr_id); + cqe++; + (*budget)--; + *pcqe = cqe; + } else { + rq = &qp->rq; + if (wr_id_idx > (rq->max_wqe - 1)) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process UD \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n", + wr_id_idx, rq->hwq.depth); + return -EINVAL; + } + if (rq->swq_last != wr_id_idx) + return -EINVAL; + + cqe->wr_id = rq->swq[rq->swq_last].wr_id; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed UD RQ wr_id[%d] = 0x%llx\n", + rq->swq_last, cqe->wr_id); + cqe++; + (*budget)--; + bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, + rq->swq[rq->swq_last].slots, + &rq->dbinfo.flags); + rq->swq_last = rq->swq[rq->swq_last].next_idx; + *pcqe = cqe; + + if (hwcqe->status != CQ_RES_UD_V2_STATUS_OK) + bnxt_qplib_mark_qp_error(qp); + } +done: + return rc; +} + +bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq) +{ + + struct cq_base *hw_cqe; + unsigned long flags; + bool rc = true; + + spin_lock_irqsave(&cq->hwq.lock, flags); + hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL); + + /* Check for Valid bit. If the CQE is valid, return false */ + rc = !CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags); + spin_unlock_irqrestore(&cq->hwq.lock, flags); + return rc; +} + +static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, + struct cq_res_raweth_qp1 *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *rq; + struct bnxt_qplib_srq *srq; + struct bnxt_qplib_cqe *cqe; + u32 wr_id_idx; + int rc = 0; + + qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: process_cq Raw/QP1 qp is NULL\n"); + return -EINVAL; + } + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + cqe = *pcqe; + cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; + cqe->flags = le16_to_cpu(hwcqe->flags); + cqe->qp_handle = (u64)(unsigned long)qp; + + wr_id_idx = le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) + & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK; + cqe->src_qp = qp->id; + if (qp->id == 1 && !cqe->length) { + /* Add workaround for the length misdetection */ + cqe->length = 296; + } else { + cqe->length = le16_to_cpu(hwcqe->length); + } + cqe->pkey_index = qp->pkey_index; + memcpy(cqe->smac, qp->smac, 6); + + cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); + cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); + cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata); + + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: raweth_qp1_flags = 0x%x raweth_qp1_flags2 = 0x%x\n", + cqe->raweth_qp1_flags, cqe->raweth_qp1_flags2); + + if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) { + srq = qp->srq; + if (!srq) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: SRQ used but not defined??\n"); + return -EINVAL; + } + if (wr_id_idx > srq->hwq.depth - 1) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Raw/QP1 \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", + wr_id_idx, srq->hwq.depth); + return -EINVAL; + } + cqe->wr_id = srq->swq[wr_id_idx].wr_id; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed Raw/QP1 SRQ \n"); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n", + wr_id_idx, cqe->wr_id, hwcqe->status); + cqe++; + (*budget)--; + srq->hwq.cons++; + *pcqe = cqe; + } else { + rq = &qp->rq; + if (wr_id_idx > (rq->max_wqe - 1)) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: ix 0x%x exceeded RQ max 0x%x\n", + wr_id_idx, rq->max_wqe); + return -EINVAL; + } + if (wr_id_idx != rq->swq_last) + return -EINVAL; + cqe->wr_id = rq->swq[rq->swq_last].wr_id; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed Raw/QP1 RQ \n"); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n", + wr_id_idx, cqe->wr_id, hwcqe->status); + cqe++; + (*budget)--; + bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, + rq->swq[wr_id_idx].slots, + &rq->dbinfo.flags); + rq->swq_last = rq->swq[rq->swq_last].next_idx; + *pcqe = cqe; + + if (hwcqe->status != CQ_RES_RC_STATUS_OK) + bnxt_qplib_mark_qp_error(qp); + } +done: + return rc; +} + +static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, + struct cq_terminal *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_q *sq, *rq; + struct bnxt_qplib_cqe *cqe; + struct bnxt_qplib_qp *qp; + u32 swq_last; + u32 cqe_cons; + int rc = 0; + + /* Check the Status */ + if (hwcqe->status != CQ_TERMINAL_STATUS_OK) + dev_warn(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Terminal Error status = 0x%x\n", + hwcqe->status); + + qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); + if (!qp) + return -EINVAL; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process terminal for qp (0x%x)\n", qp->id); + + /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR + * from the current rq->cons to the rq->prod regardless what the + * rq->cons the terminal CQE indicates. + */ + bnxt_qplib_mark_qp_error(qp); + + sq = &qp->sq; + rq = &qp->rq; + + cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); + if (cqe_cons == 0xFFFF) + goto do_rq; + + cqe_cons %= sq->max_wqe; + if (qp->sq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto sq_done; + } + + /* Terminal CQE can also include aggregated successful CQEs prior. + So we must complete all CQEs from the current sq's cons to the + cq_cons with status OK */ + cqe = *pcqe; + while (*budget) { + /*sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);*/ + swq_last = sq->swq_last; + if (swq_last == cqe_cons) + break; + if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { + memset(cqe, 0, sizeof(*cqe)); + cqe->status = CQ_REQ_STATUS_OK; + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; + cqe->qp_handle = (u64)qp; + cqe->src_qp = qp->id; + cqe->wr_id = sq->swq[swq_last].wr_id; + cqe->type = sq->swq[swq_last].type; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed terminal Req \n"); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n", + swq_last, cqe->wr_id, cqe->status); + cqe++; + (*budget)--; + } + bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons, + sq->swq[swq_last].slots, + &sq->dbinfo.flags); + sq->swq_last = sq->swq[swq_last].next_idx; + } + *pcqe = cqe; + if (!*budget && swq_last != cqe_cons) { + /* Out of budget */ + rc = -EAGAIN; + goto sq_done; + } +sq_done: + if (rc) + return rc; +do_rq: + cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); + if (cqe_cons == 0xFFFF) { + goto done; + } else if (cqe_cons > (rq->max_wqe - 1)) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed terminal \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x\n", + cqe_cons, rq->hwq.depth); + goto done; + } + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + rc = 0; + goto rq_done; + } + +rq_done: +done: + return rc; +} + +static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq, + struct cq_cutoff *hwcqe) +{ + /* Check the Status */ + if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Cutoff Error status = 0x%x\n", + hwcqe->status); + return -EINVAL; + } + clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); + wake_up_interruptible(&cq->waitq); + + dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Cutoff\n"); + return 0; +} + +int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, + struct bnxt_qplib_cqe *cqe, + int num_cqes) +{ + struct bnxt_qplib_qp *qp = NULL; + u32 budget = num_cqes; + unsigned long flags; + + spin_lock_irqsave(&cq->flush_lock, flags); + list_for_each_entry(qp, &cq->sqf_head, sq_flush) { + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: Flushing SQ QP= %p\n", + qp); + __flush_sq(&qp->sq, qp, &cqe, &budget); + } + + list_for_each_entry(qp, &cq->rqf_head, rq_flush) { + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: Flushing RQ QP= %p\n", + qp); + __flush_rq(&qp->rq, qp, &cqe, &budget); + } + spin_unlock_irqrestore(&cq->flush_lock, flags); + + return num_cqes - budget; +} + +int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, + int num_cqes, struct bnxt_qplib_qp **lib_qp) +{ + struct cq_base *hw_cqe; + u32 hw_polled = 0; + int budget, rc = 0; + u8 type; + + budget = num_cqes; + + while (budget) { + hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL); + + /* Check for Valid bit */ + if (!CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags)) + break; + + /* The valid test of the entry must be done first before + * reading any further. + */ + dma_rmb(); + /* From the device's respective CQE format to qplib_wc*/ + type = hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; + switch (type) { + case CQ_BASE_CQE_TYPE_REQ: + rc = bnxt_qplib_cq_process_req(cq, + (struct cq_req *)hw_cqe, &cqe, &budget, + cq->hwq.cons, lib_qp); + break; + case CQ_BASE_CQE_TYPE_RES_RC: + rc = bnxt_qplib_cq_process_res_rc(cq, + (struct cq_res_rc *)hw_cqe, &cqe, + &budget); + break; + case CQ_BASE_CQE_TYPE_RES_UD: + rc = bnxt_qplib_cq_process_res_ud(cq, + (struct cq_res_ud_v2 *)hw_cqe, + &cqe, &budget); + break; + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: + rc = bnxt_qplib_cq_process_res_raweth_qp1(cq, + (struct cq_res_raweth_qp1 *) + hw_cqe, &cqe, &budget); + break; + case CQ_BASE_CQE_TYPE_TERMINAL: + rc = bnxt_qplib_cq_process_terminal(cq, + (struct cq_terminal *)hw_cqe, + &cqe, &budget); + break; + case CQ_BASE_CQE_TYPE_CUT_OFF: + bnxt_qplib_cq_process_cutoff(cq, + (struct cq_cutoff *)hw_cqe); + /* Done processing this CQ */ + goto exit; + default: + dev_err(&cq->hwq.pdev->dev, + "QPLIB: process_cq unknown type 0x%x\n", + hw_cqe->cqe_type_toggle & + CQ_BASE_CQE_TYPE_MASK); + rc = -EINVAL; + break; + } + if (rc < 0) { + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: process_cqe rc = 0x%x\n", rc); + if (rc == -EAGAIN) + break; + /* Error while processing the CQE, just skip to the + next one */ + if (type != CQ_BASE_CQE_TYPE_TERMINAL) + dev_err(&cq->hwq.pdev->dev, + "QPLIB: process_cqe error rc = 0x%x\n", + rc); + } + hw_polled++; + bnxt_qplib_hwq_incr_cons(cq->hwq.depth, &cq->hwq.cons, + 1, &cq->dbinfo.flags); + } + if (hw_polled) + bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ); +exit: + return num_cqes - budget; +} + +void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type) +{ + cq->dbinfo.toggle = cq->toggle; + if (arm_type) + bnxt_qplib_ring_db(&cq->dbinfo, arm_type); + /* Using cq->arm_state variable to track whether to issue cq handler */ + atomic_set(&cq->arm_state, 1); +} + +void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp) +{ + flush_workqueue(qp->scq->nq->cqn_wq); + if (qp->scq != qp->rcq) + flush_workqueue(qp->rcq->nq->cqn_wq); +} diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.h b/sys/dev/bnxt/bnxt_re/qplib_fp.h new file mode 100644 index 00000000000..527c377f0aa --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_fp.h @@ -0,0 +1,638 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Fast Path Operators (header) + */ + +#ifndef __BNXT_QPLIB_FP_H__ +#define __BNXT_QPLIB_FP_H__ + +/* Temp header structures for SQ */ +struct sq_ud_ext_hdr { + __le32 dst_qp; + __le32 avid; + __le64 rsvd; +}; + +struct sq_raw_ext_hdr { + __le32 cfa_meta; + __le32 rsvd0; + __le64 rsvd1; +}; + +struct sq_rdma_ext_hdr { + __le64 remote_va; + __le32 remote_key; + __le32 rsvd; +}; + +struct sq_atomic_ext_hdr { + __le64 swap_data; + __le64 cmp_data; +}; + +struct sq_fr_pmr_ext_hdr { + __le64 pblptr; + __le64 va; +}; + +struct sq_bind_ext_hdr { + __le64 va; + __le32 length_lo; + __le32 length_hi; +}; + +struct rq_ext_hdr { + __le64 rsvd1; + __le64 rsvd2; +}; + +#define BNXT_QPLIB_ETHTYPE_ROCEV1 0x8915 + +struct bnxt_qplib_srq { + struct bnxt_qplib_pd *pd; + struct bnxt_qplib_dpi *dpi; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_qplib_cq *cq; + struct bnxt_qplib_swq *swq; + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_db_info dbinfo; + struct bnxt_qplib_sg_info sginfo; + u64 srq_handle; + u32 id; + u16 wqe_size; + u32 max_wqe; + u32 max_sge; + u32 threshold; + bool arm_req; + int start_idx; + int last_idx; + u16 eventq_hw_ring_id; + bool is_user; + spinlock_t lock; +}; + +struct bnxt_qplib_sge { + u64 addr; + u32 size; + u32 lkey; +}; + +/* + * Buffer space for ETH(14), IP or GRH(40), UDP header(8) + * and ib_bth + ib_deth (20). + * Max required is 82 when RoCE V2 is enabled + */ + +/* + * RoCE V1 (38 bytes needed) + * +------------+----------+--------+--------+-------+ + * |Eth-hdr(14B)| GRH (40B)|bth+deth| Mad | iCRC | + * | | supplied | 20B |payload | 4B | + * | | by user |supplied| 256B | | + * | | mad | |by user | | + * | | | | | | + * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 | + * +------------+----------+--------+--------+-------+ + */ + +/* + * RoCE V2-IPv4 (46 Bytes needed) + * +------------+----------+--------+--------+-------+ + * |Eth-hdr(14B)| IP-hdr |UDP-hdr | Mad | iCRC | + * | | supplied | 8B |payload | 4B | + * | | by user |bth+deth| 256B | | + * | | mad lower| 20B |supplied| | + * | | 20B out | (sge 3)|by user | | + * | | of 40B | | | | + * | | grh space| | | | + * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 | + * +------------+----------+--------+--------+-------+ + */ + +/* + * RoCE V2-IPv6 (46 Bytes needed) + * +------------+----------+--------+--------+-------+ + * |Eth-hdr(14B)| IPv6 |UDP-hdr | Mad | iCRC | + * | | supplied | 8B |payload | 4B | + * | | by user |bth+deth| 256B | | + * | | mad lower| 20B |supplied| | + * | | 40 bytes | |by user | | + * | | grh space| | | | + * | | | | | | + * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 | + * +------------+----------+--------+--------+-------+ + */ + +#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE 74 +#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86 +#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE 46 +#define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14 +#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512 +#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20 +#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40 +#define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20 +#define BNXT_QPLIB_MAX_SQSZ 0xFFFF + +struct bnxt_qplib_hdrbuf { + dma_addr_t dma_map; + void *va; + u32 len; + u32 step; +}; + +struct bnxt_qplib_swq { + u64 wr_id; + int next_idx; + u8 type; + u8 flags; + u32 start_psn; + u32 next_psn; + u32 slot_idx; + u8 slots; + /* WIP: make it void * to handle legacy also */ + struct sq_psn_search *psn_search; + void *inline_data; +}; + +struct bnxt_qplib_swqe { + /* General */ +#define BNXT_QPLIB_FENCE_WRID 0x46454E43 /* "FENC" */ +#define BNXT_QPLIB_QP1_DUMMY_WRID 0x44554D59 /* "DUMY" */ + u64 wr_id; + u8 reqs_type; + u8 type; +#define BNXT_QPLIB_SWQE_TYPE_SEND 0 +#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1 +#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2 +#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4 +#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5 +#define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6 +#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8 +#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11 +#define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12 +#define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13 +#define BNXT_QPLIB_SWQE_TYPE_REG_MR 13 +#define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14 +#define BNXT_QPLIB_SWQE_TYPE_RECV 128 +#define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129 + u8 flags; +#define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP (1 << 0) +#define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE (1 << 1) +#define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE (1 << 2) +#define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT (1 << 3) +#define BNXT_QPLIB_SWQE_FLAGS_INLINE (1 << 4) + struct bnxt_qplib_sge *sg_list; + int num_sge; + + union { + /* Send, with imm, inval key */ + struct { + union { + __be32 imm_data; + u32 inv_key; + }; + u32 q_key; + u32 dst_qp; + u16 avid; + } send; + + /* Send Raw Ethernet and QP1 */ + struct { + u16 lflags; + u16 cfa_action; + u32 cfa_meta; + } rawqp1; + + /* RDMA write, with imm, read */ + struct { + union { + __be32 imm_data; + u32 inv_key; + }; + u64 remote_va; + u32 r_key; + } rdma; + + /* Atomic cmp/swap, fetch/add */ + struct { + u64 remote_va; + u32 r_key; + u64 swap_data; + u64 cmp_data; + } atomic; + + /* Local Invalidate */ + struct { + u32 inv_l_key; + } local_inv; + + /* FR-PMR */ + struct { + u8 access_cntl; + u8 pg_sz_log; + bool zero_based; + u32 l_key; + u32 length; + u8 pbl_pg_sz_log; +#define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18 + u8 levels; +#define PAGE_SHIFT_4K 12 + __le64 *pbl_ptr; + dma_addr_t pbl_dma_ptr; + u64 *page_list; + u16 page_list_len; + u64 va; + } frmr; + + /* Bind */ + struct { + u8 access_cntl; +#define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE (1 << 0) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ (1 << 1) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE (1 << 2) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC (1 << 3) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND (1 << 4) + bool zero_based; + u8 mw_type; + u32 parent_l_key; + u32 r_key; + u64 va; + u32 length; + } bind; + }; +}; + +struct bnxt_qplib_q { + struct bnxt_qplib_swq *swq; + struct bnxt_qplib_db_info dbinfo; + struct bnxt_qplib_sg_info sginfo; + struct bnxt_qplib_hwq hwq; + u32 max_wqe; + u16 max_sge; + u16 wqe_size; + u16 q_full_delta; + u32 psn; + bool condition; + bool single; + bool legacy_send_phantom; + u32 phantom_wqe_cnt; + u32 phantom_cqe_cnt; + u32 next_cq_cons; + bool flushed; + u32 swq_start; + u32 swq_last; +}; + +#define BNXT_QPLIB_PPP_REQ 0x1 +#define BNXT_QPLIB_PPP_ST_IDX_SHIFT 0x1 + +struct bnxt_qplib_ppp { + u32 dpi; + u8 req; + u8 st_idx_en; +}; + +struct bnxt_qplib_qp { + struct bnxt_qplib_pd *pd; + struct bnxt_qplib_dpi *dpi; + struct bnxt_qplib_chip_ctx *cctx; + u64 qp_handle; +#define BNXT_QPLIB_QP_ID_INVALID 0xFFFFFFFF + u32 id; + u8 type; + u8 sig_type; + u8 wqe_mode; + u8 state; + u8 cur_qp_state; + u8 is_user; + u64 modify_flags; + u32 max_inline_data; + u32 mtu; + u32 path_mtu; + bool en_sqd_async_notify; + u16 pkey_index; + u32 qkey; + u32 dest_qp_id; + u8 access; + u8 timeout; + u8 retry_cnt; + u8 rnr_retry; + u64 wqe_cnt; + u32 min_rnr_timer; + u32 max_rd_atomic; + u32 max_dest_rd_atomic; + u32 dest_qpn; + u8 smac[6]; + u16 vlan_id; + u8 nw_type; + u16 port_id; + struct bnxt_qplib_ah ah; + struct bnxt_qplib_ppp ppp; + +#define BTH_PSN_MASK ((1 << 24) - 1) + /* SQ */ + struct bnxt_qplib_q sq; + /* RQ */ + struct bnxt_qplib_q rq; + /* SRQ */ + struct bnxt_qplib_srq *srq; + /* CQ */ + struct bnxt_qplib_cq *scq; + struct bnxt_qplib_cq *rcq; + /* IRRQ and ORRQ */ + struct bnxt_qplib_hwq irrq; + struct bnxt_qplib_hwq orrq; + /* Header buffer for QP1 */ + struct bnxt_qplib_hdrbuf *sq_hdr_buf; + struct bnxt_qplib_hdrbuf *rq_hdr_buf; + + /* ToS */ + u8 tos_ecn; + u8 tos_dscp; + /* To track the SQ and RQ flush list */ + struct list_head sq_flush; + struct list_head rq_flush; + /* 4 bytes of QP's scrabled mac received from FW */ + u32 lag_src_mac; + u32 msn; + u32 msn_tbl_sz; + /* get devflags in PI code */ + u16 dev_cap_flags; +}; + + +#define CQE_CMP_VALID(hdr, pass) \ + (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \ + !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK)) + +static inline u32 __bnxt_qplib_get_avail(struct bnxt_qplib_hwq *hwq) +{ + int cons, prod, avail; + + /* False full is possible retrying post-send makes sense */ + cons = hwq->cons; + prod = hwq->prod; + avail = cons - prod; + if (cons <= prod) + avail += hwq->depth; + return avail; +} + +static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_hwq *hwq, u8 slots) +{ + return __bnxt_qplib_get_avail(hwq) <= slots; +} + +struct bnxt_qplib_cqe { + u8 status; + u8 type; + u8 opcode; + u32 length; + /* Lower 16 is cfa_metadata0, Upper 16 is cfa_metadata1 */ + u32 cfa_meta; +#define BNXT_QPLIB_META1_SHIFT 16 +#define BNXT_QPLIB_CQE_CFA_META1_VALID 0x80000UL + u64 wr_id; + union { + __be32 immdata; + u32 invrkey; + }; + u64 qp_handle; + u64 mr_handle; + u16 flags; + u8 smac[6]; + u32 src_qp; + u16 raweth_qp1_flags; + u16 raweth_qp1_errors; + u16 raweth_qp1_cfa_code; + u32 raweth_qp1_flags2; + u32 raweth_qp1_metadata; + u8 raweth_qp1_payload_offset; + u16 pkey_index; +}; + +#define BNXT_QPLIB_QUEUE_START_PERIOD 0x01 +struct bnxt_qplib_cq { + struct bnxt_qplib_dpi *dpi; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_qplib_nq *nq; + struct bnxt_qplib_db_info dbinfo; + struct bnxt_qplib_sg_info sginfo; + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_hwq resize_hwq; + struct list_head sqf_head; + struct list_head rqf_head; + u32 max_wqe; + u32 id; + u16 count; + u16 period; + u32 cnq_hw_ring_id; + u64 cq_handle; + atomic_t arm_state; +#define CQ_RESIZE_WAIT_TIME_MS 500 + unsigned long flags; +#define CQ_FLAGS_RESIZE_IN_PROG 1 + wait_queue_head_t waitq; + spinlock_t flush_lock; /* lock flush queue list */ + spinlock_t compl_lock; /* synch CQ handlers */ + u16 cnq_events; + bool is_cq_err_event; + bool destroyed; + u8 toggle; +}; + +#define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq) +#define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq) +#define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * x + 2) +#define IRRQ_SLOTS_TO_IRD_LIMIT(s) ((s >> 1) - 1) +#define ORD_LIMIT_TO_ORRQ_SLOTS(x) (x + 1) +#define ORRQ_SLOTS_TO_ORD_LIMIT(s) (s - 1) + +#define NQE_CMP_VALID(hdr, pass) \ + (!!(le32_to_cpu((hdr)->info63_v & 0xffffffff) & NQ_BASE_V) == \ + !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK)) + +#define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024) + +/* MSN table print macros for debugging */ +#define BNXT_RE_MSN_IDX(m) (((m) & SQ_MSN_SEARCH_START_IDX_MASK) >> \ + SQ_MSN_SEARCH_START_IDX_SFT) +#define BNXT_RE_MSN_NPSN(m) (((m) & SQ_MSN_SEARCH_NEXT_PSN_MASK) >> \ + SQ_MSN_SEARCH_NEXT_PSN_SFT) +#define BNXT_RE_MSN_SPSN(m) (((m) & SQ_MSN_SEARCH_START_PSN_MASK) >> \ + SQ_MSN_SEARCH_START_PSN_SFT) +#define BNXT_MSN_TBLE_SGE 6 + +struct bnxt_qplib_nq_stats { + u64 num_dbqne_processed; + u64 num_srqne_processed; + u64 num_cqne_processed; + u64 num_tasklet_resched; + u64 num_nq_rearm; +}; + +struct bnxt_qplib_nq_db { + struct bnxt_qplib_reg_desc reg; + void __iomem *db; + struct bnxt_qplib_db_info dbinfo; +}; + +typedef int (*cqn_handler_t)(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_cq *cq); +typedef int (*srqn_handler_t)(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_srq *srq, u8 event); + +struct bnxt_qplib_nq { + struct bnxt_qplib_res *res; + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_nq_db nq_db; + + char *name; + u16 ring_id; + int msix_vec; + bool requested; + int budget; + u32 load; + struct mutex lock; + + cqn_handler_t cqn_handler; + srqn_handler_t srqn_handler; + struct workqueue_struct *cqn_wq; + struct bnxt_qplib_nq_stats stats; +}; + +struct bnxt_qplib_nq_work { + struct work_struct work; + struct bnxt_qplib_nq *nq; + struct bnxt_qplib_cq *cq; +}; + +static inline dma_addr_t +bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) +{ + struct bnxt_qplib_hdrbuf *buf; + + buf = qp->rq_hdr_buf; + return (buf->dma_map + index * buf->step); +} + +void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill); +void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq); +int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, + int msix_vector, bool need_init); +int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx, + int msix_vector, int bar_reg_offset, + cqn_handler_t cqn_handler, + srqn_handler_t srq_handler); +int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, + struct bnxt_qplib_swqe *wqe); +int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp); +void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge); +void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge); +u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp); +void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp); +int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe); +void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp); +int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe); +int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); +int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); +int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq, + int new_cqes); +void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res, + struct bnxt_qplib_cq *cq); +int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); +void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); +int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, + int num, struct bnxt_qplib_qp **qp); +bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq); +void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type); +void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq); +int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res, + struct bnxt_qplib_nq *nq); +void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp); +void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp); +int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, + struct bnxt_qplib_cqe *cqe, + int num_cqes); +void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp); +void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp); +int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp, u32 slen, u32 rlen); + +static inline bool __can_request_ppp(struct bnxt_qplib_qp *qp) +{ + bool can_request = false; + + if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RESET && + qp->state == CMDQ_MODIFY_QP_NEW_STATE_INIT && + qp->ppp.req && + !(qp->ppp.st_idx_en & + CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED)) + can_request = true; + return can_request; +} + +/* MSN table update inlin */ +static inline uint64_t bnxt_re_update_msn_tbl(uint32_t st_idx, uint32_t npsn, uint32_t start_psn) +{ + return cpu_to_le64((((u64)(st_idx) << SQ_MSN_SEARCH_START_IDX_SFT) & + SQ_MSN_SEARCH_START_IDX_MASK) | + (((u64)(npsn) << SQ_MSN_SEARCH_NEXT_PSN_SFT) & + SQ_MSN_SEARCH_NEXT_PSN_MASK) | + (((start_psn) << SQ_MSN_SEARCH_START_PSN_SFT) & + SQ_MSN_SEARCH_START_PSN_MASK)); +} + +void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res); +#endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_rcfw.c b/sys/dev/bnxt/bnxt_re/qplib_rcfw.c new file mode 100644 index 00000000000..7e3453a1e04 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_rcfw.c @@ -0,0 +1,1338 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: RDMA Controller HW interface + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hsi_struct_def.h" +#include "qplib_tlv.h" +#include "qplib_res.h" +#include "qplib_sp.h" +#include "qplib_rcfw.h" +#include "bnxt_re.h" + +static void bnxt_qplib_service_creq(unsigned long data); + +int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw, + u32 *cur_prod, u32 *cur_cons) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + + cmdq = &rcfw->cmdq; + + if (*cur_prod == cmdq->hwq.prod && + *cur_cons == cmdq->hwq.cons) + /* No activity on CMDQ or CREQ. FW down */ + return -ETIMEDOUT; + + *cur_prod = cmdq->hwq.prod; + *cur_cons = cmdq->hwq.cons; + return 0; +} + +static int bnxt_qplib_map_rc(u8 opcode) +{ + switch (opcode) { + case CMDQ_BASE_OPCODE_DESTROY_QP: + case CMDQ_BASE_OPCODE_DESTROY_SRQ: + case CMDQ_BASE_OPCODE_DESTROY_CQ: + case CMDQ_BASE_OPCODE_DEALLOCATE_KEY: + case CMDQ_BASE_OPCODE_DEREGISTER_MR: + case CMDQ_BASE_OPCODE_DELETE_GID: + case CMDQ_BASE_OPCODE_DESTROY_QP1: + case CMDQ_BASE_OPCODE_DESTROY_AH: + case CMDQ_BASE_OPCODE_DEINITIALIZE_FW: + case CMDQ_BASE_OPCODE_MODIFY_ROCE_CC: + case CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE: + return 0; + default: + return -ETIMEDOUT; + } +} + +/** + * bnxt_re_is_fw_stalled - Check firmware health + * @rcfw - rcfw channel instance of rdev + * @cookie - cookie to track the command + * + * If firmware has not responded any rcfw command within + * rcfw->max_timeout, consider firmware as stalled. + * + * Returns: + * 0 if firmware is responding + * -ENODEV if firmware is not responding + */ +static int bnxt_re_is_fw_stalled(struct bnxt_qplib_rcfw *rcfw, u16 cookie) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_crsqe *crsqe; + + crsqe = &rcfw->crsqe_tbl[cookie]; + cmdq = &rcfw->cmdq; + + if (time_after(jiffies, cmdq->last_seen + + (rcfw->max_timeout * HZ))) { + dev_warn_ratelimited(&rcfw->pdev->dev, + "%s: FW STALL Detected. cmdq[%#x]=%#x waited (%ld > %d) msec active %d\n", + __func__, cookie, crsqe->opcode, + (long)jiffies_to_msecs(jiffies - cmdq->last_seen), + rcfw->max_timeout * 1000, + crsqe->is_in_used); + return -ENODEV; + } + + return 0; +} +/** + * __wait_for_resp - Don't hold the cpu context and wait for response + * @rcfw - rcfw channel instance of rdev + * @cookie - cookie to track the command + * + * Wait for command completion in sleepable context. + * + * Returns: + * 0 if command is completed by firmware. + * Non zero error code for rest of the case. + */ +static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_crsqe *crsqe; + unsigned long issue_time; + int ret; + + cmdq = &rcfw->cmdq; + issue_time = jiffies; + crsqe = &rcfw->crsqe_tbl[cookie]; + + do { + if (RCFW_NO_FW_ACCESS(rcfw)) + return bnxt_qplib_map_rc(crsqe->opcode); + if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) + return -ETIMEDOUT; + + /* Non zero means command completed */ + ret = wait_event_timeout(cmdq->waitq, + !crsqe->is_in_used || + RCFW_NO_FW_ACCESS(rcfw), + msecs_to_jiffies(rcfw->max_timeout * 1000)); + + if (!crsqe->is_in_used) + return 0; + /* + * Take care if interrupt miss or other cases like DBR drop + */ + bnxt_qplib_service_creq((unsigned long)rcfw); + dev_warn_ratelimited(&rcfw->pdev->dev, + "Non-Blocking QPLIB: cmdq[%#x]=%#x waited (%lu) msec bit %d\n", + cookie, crsqe->opcode, + (long)jiffies_to_msecs(jiffies - issue_time), + crsqe->is_in_used); + + if (!crsqe->is_in_used) + return 0; + + ret = bnxt_re_is_fw_stalled(rcfw, cookie); + if (ret) + return ret; + + } while (true); +}; + +/** + * __block_for_resp - hold the cpu context and wait for response + * @rcfw - rcfw channel instance of rdev + * @cookie - cookie to track the command + * + * This function will hold the cpu (non-sleepable context) and + * wait for command completion. Maximum holding interval is 8 second. + * + * Returns: + * -ETIMEOUT if command is not completed in specific time interval. + * 0 if command is completed by firmware. + */ +static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) +{ + struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq; + struct bnxt_qplib_crsqe *crsqe; + unsigned long issue_time = 0; + + issue_time = jiffies; + crsqe = &rcfw->crsqe_tbl[cookie]; + + do { + if (RCFW_NO_FW_ACCESS(rcfw)) + return bnxt_qplib_map_rc(crsqe->opcode); + if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) + return -ETIMEDOUT; + + udelay(1); + + /* Below call is must since there can be a deadlock + * if interrupt is mapped to the same cpu + */ + bnxt_qplib_service_creq((unsigned long)rcfw); + if (!crsqe->is_in_used) + return 0; + + } while (time_before(jiffies, issue_time + (8 * HZ))); + + dev_warn_ratelimited(&rcfw->pdev->dev, + "Blocking QPLIB: cmdq[%#x]=%#x taken (%lu) msec", + cookie, crsqe->opcode, + (long)jiffies_to_msecs(jiffies - issue_time)); + + return -ETIMEDOUT; +}; + +/* __send_message_no_waiter - get cookie and post the message. + * @rcfw - rcfw channel instance of rdev + * @msg - qplib message internal + * + * This function will just post and don't bother about completion. + * Current design of this function is - + * user must hold the completion queue hwq->lock. + * user must have used existing completion and free the resources. + * this function will not check queue full condition. + * this function will explicitly set is_waiter_alive=false. + * current use case is - send destroy_ah if create_ah is return + * after waiter of create_ah is lost. It can be extended for other + * use case as well. + * + * Returns: Nothing + * + */ +static void __send_message_no_waiter(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg) +{ + struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq; + struct bnxt_qplib_hwq *cmdq_hwq = &cmdq->hwq; + struct bnxt_qplib_crsqe *crsqe; + struct bnxt_qplib_cmdqe *cmdqe; + u32 sw_prod, cmdq_prod, bsize; + u16 cookie; + u8 *preq; + + cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE; + __set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie)); + crsqe = &rcfw->crsqe_tbl[cookie]; + + /* Set cmd_size in terms of 16B slots in req. */ + bsize = bnxt_qplib_set_cmd_slots(msg->req); + /* GET_CMD_SIZE would return number of slots in either case of tlv + * and non-tlv commands after call to bnxt_qplib_set_cmd_slots() + */ + crsqe->send_timestamp = jiffies; + crsqe->is_internal_cmd = true; + crsqe->is_waiter_alive = false; + crsqe->is_in_used = true; + crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz); + + preq = (u8 *)msg->req; + do { + /* Locate the next cmdq slot */ + sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq); + cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL); + /* Copy a segment of the req cmd to the cmdq */ + memset(cmdqe, 0, sizeof(*cmdqe)); + memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe))); + preq += min_t(u32, bsize, sizeof(*cmdqe)); + bsize -= min_t(u32, bsize, sizeof(*cmdqe)); + cmdq_hwq->prod++; + } while (bsize > 0); + cmdq->seq_num++; + + cmdq_prod = cmdq_hwq->prod & 0xFFFF; + atomic_inc(&rcfw->timeout_send); + /* ring CMDQ DB */ + wmb(); + writel(cmdq_prod, cmdq->cmdq_mbox.prod); + writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db); +} + +static int __send_message(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg) +{ + u32 bsize, free_slots, required_slots; + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_crsqe *crsqe; + struct bnxt_qplib_cmdqe *cmdqe; + struct bnxt_qplib_hwq *cmdq_hwq; + u32 sw_prod, cmdq_prod; + struct pci_dev *pdev; + unsigned long flags; + u16 cookie; + u8 opcode; + u8 *preq; + + cmdq = &rcfw->cmdq; + cmdq_hwq = &cmdq->hwq; + pdev = rcfw->pdev; + opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz); + + /* Cmdq are in 16-byte units, each request can consume 1 or more + cmdqe */ + spin_lock_irqsave(&cmdq_hwq->lock, flags); + required_slots = bnxt_qplib_get_cmd_slots(msg->req); + free_slots = HWQ_FREE_SLOTS(cmdq_hwq); + cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE; + crsqe = &rcfw->crsqe_tbl[cookie]; + + if (required_slots >= free_slots) { + dev_warn_ratelimited(&pdev->dev, + "QPLIB: RCFW: CMDQ is full req/free %d/%d!\n", + required_slots, free_slots); + rcfw->cmdq_full_dbg++; + spin_unlock_irqrestore(&cmdq_hwq->lock, flags); + return -EAGAIN; + } + + if (crsqe->is_in_used) + panic("QPLIB: Cookie was not requested %d\n", + cookie); + + if (msg->block) + cookie |= RCFW_CMD_IS_BLOCKING; + __set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie)); + + /* Set cmd_size in terms of 16B slots in req. */ + bsize = bnxt_qplib_set_cmd_slots(msg->req); + /* GET_CMD_SIZE would return number of slots in either case of tlv + * and non-tlv commands after call to bnxt_qplib_set_cmd_slots() + */ + crsqe->send_timestamp = jiffies; + crsqe->free_slots = free_slots; + crsqe->resp = (struct creq_qp_event *)msg->resp; + crsqe->resp->cookie = cpu_to_le16(cookie); + crsqe->is_internal_cmd = false; + crsqe->is_waiter_alive = true; + crsqe->is_in_used = true; + crsqe->opcode = opcode; + crsqe->requested_qp_state = msg->qp_state; + + crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz); + if (__get_cmdq_base_resp_size(msg->req, msg->req_sz) && msg->sb) { + struct bnxt_qplib_rcfw_sbuf *sbuf = msg->sb; + + __set_cmdq_base_resp_addr(msg->req, msg->req_sz, + cpu_to_le64(sbuf->dma_addr)); + __set_cmdq_base_resp_size(msg->req, msg->req_sz, + ALIGN(sbuf->size, BNXT_QPLIB_CMDQE_UNITS) / + BNXT_QPLIB_CMDQE_UNITS); + } + + preq = (u8 *)msg->req; + do { + /* Locate the next cmdq slot */ + sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq); + cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL); + /* Copy a segment of the req cmd to the cmdq */ + memset(cmdqe, 0, sizeof(*cmdqe)); + memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe))); + preq += min_t(u32, bsize, sizeof(*cmdqe)); + bsize -= min_t(u32, bsize, sizeof(*cmdqe)); + cmdq_hwq->prod++; + } while (bsize > 0); + cmdq->seq_num++; + + cmdq_prod = cmdq_hwq->prod & 0xFFFF; + if (test_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags)) { + /* The very first doorbell write + * is required to set this flag + * which prompts the FW to reset + * its internal pointers + */ + cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG); + clear_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags); + } + /* ring CMDQ DB */ + wmb(); + writel(cmdq_prod, cmdq->cmdq_mbox.prod); + writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db); + + dev_dbg(&pdev->dev, "QPLIB: RCFW sent request with 0x%x 0x%x 0x%x\n", + cmdq_prod, cmdq_hwq->prod, crsqe->req_size); + dev_dbg(&pdev->dev, + "QPLIB: opcode 0x%x with cookie 0x%x at cmdq/crsq 0x%p/0x%p\n", + opcode, + __get_cmdq_base_cookie(msg->req, msg->req_sz), + cmdqe, crsqe); + spin_unlock_irqrestore(&cmdq_hwq->lock, flags); + /* Return the CREQ response pointer */ + return 0; +} + +/** + * __poll_for_resp - self poll completion for rcfw command + * @rcfw - rcfw channel instance of rdev + * @cookie - cookie to track the command + * + * It works same as __wait_for_resp except this function will + * do self polling in sort interval since interrupt is disabled. + * This function can not be called from non-sleepable context. + * + * Returns: + * -ETIMEOUT if command is not completed in specific time interval. + * 0 if command is completed by firmware. + */ +static int __poll_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) +{ + struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq; + struct bnxt_qplib_crsqe *crsqe; + unsigned long issue_time; + int ret; + + issue_time = jiffies; + crsqe = &rcfw->crsqe_tbl[cookie]; + + do { + if (RCFW_NO_FW_ACCESS(rcfw)) + return bnxt_qplib_map_rc(crsqe->opcode); + if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) + return -ETIMEDOUT; + + usleep_range(1000, 1001); + + bnxt_qplib_service_creq((unsigned long)rcfw); + if (!crsqe->is_in_used) + return 0; + + if (jiffies_to_msecs(jiffies - issue_time) > + (rcfw->max_timeout * 1000)) { + dev_warn_ratelimited(&rcfw->pdev->dev, + "Self Polling QPLIB: cmdq[%#x]=%#x taken (%lu) msec", + cookie, crsqe->opcode, + (long)jiffies_to_msecs(jiffies - issue_time)); + ret = bnxt_re_is_fw_stalled(rcfw, cookie); + if (ret) + return ret; + } + } while (true); + +}; + +static int __send_message_basic_sanity(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg, u8 opcode) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + + cmdq = &rcfw->cmdq; + + /* Prevent posting if f/w is not in a state to process */ + if (RCFW_NO_FW_ACCESS(rcfw)) + return -ENXIO; + + if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) + return -ETIMEDOUT; + + if (test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) && + opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) { + dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!\n"); + return -EINVAL; + } + + if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) && + (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC && + opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW && + opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) { + dev_err(&rcfw->pdev->dev, + "QPLIB: RCFW not initialized, reject opcode 0x%x\n", + opcode); + return -ENOTSUPP; + } + + return 0; +} + +/* This function will just post and do not bother about completion */ +static void __destroy_timedout_ah(struct bnxt_qplib_rcfw *rcfw, + struct creq_create_ah_resp *create_ah_resp) +{ + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_destroy_ah req = {}; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_AH, + sizeof(req)); + req.ah_cid = create_ah_resp->xid; + msg.req = (struct cmdq_base *)&req; + msg.req_sz = sizeof(req); + __send_message_no_waiter(rcfw, &msg); + dev_warn_ratelimited(&rcfw->pdev->dev, + "From %s: ah_cid = %d timeout_send %d\n", __func__, + req.ah_cid, + atomic_read(&rcfw->timeout_send)); +} + +/** + * __bnxt_qplib_rcfw_send_message - qplib interface to send + * and complete rcfw command. + * @rcfw - rcfw channel instance of rdev + * @msg - qplib message internal + * + * This function does not account shadow queue depth. It will send + * all the command unconditionally as long as send queue is not full. + * + * Returns: + * 0 if command completed by firmware. + * Non zero if the command is not completed by firmware. + */ +static int __bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg) +{ + struct bnxt_qplib_crsqe *crsqe; + struct creq_qp_event *event; + unsigned long flags; + u16 cookie; + int rc = 0; + u8 opcode; + + opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz); + + rc = __send_message_basic_sanity(rcfw, msg, opcode); + if (rc) + return rc == -ENXIO ? bnxt_qplib_map_rc(opcode) : rc; + + rc = __send_message(rcfw, msg); + if (rc) + return rc; + + cookie = le16_to_cpu(__get_cmdq_base_cookie(msg->req, + msg->req_sz)) & RCFW_MAX_COOKIE_VALUE; + + + if (msg->block) + rc = __block_for_resp(rcfw, cookie); + else if (atomic_read(&rcfw->rcfw_intr_enabled)) + rc = __wait_for_resp(rcfw, cookie); + else + rc = __poll_for_resp(rcfw, cookie); + + if (rc) { + /* First check if it is FW stall. + * Use hwq.lock to avoid race with actual completion. + */ + spin_lock_irqsave(&rcfw->cmdq.hwq.lock, flags); + crsqe = &rcfw->crsqe_tbl[cookie]; + crsqe->is_waiter_alive = false; + if (rc == -ENODEV) + set_bit(FIRMWARE_STALL_DETECTED, &rcfw->cmdq.flags); + spin_unlock_irqrestore(&rcfw->cmdq.hwq.lock, flags); + + return -ETIMEDOUT; + } + + event = (struct creq_qp_event *)msg->resp; + if (event->status) { + /* failed with status */ + dev_err(&rcfw->pdev->dev, "QPLIB: cmdq[%#x]=%#x (%s) status %d\n", + cookie, opcode, GET_OPCODE_TYPE(opcode), event->status); + rc = -EFAULT; + /* + * Workaround to avoid errors in the stack during bond + * creation and deletion. + * Disable error returned for ADD_GID/DEL_GID + */ + if (opcode == CMDQ_BASE_OPCODE_ADD_GID || + opcode == CMDQ_BASE_OPCODE_DELETE_GID) + rc = 0; + } + + dev_dbg(&pdev->dev, "QPLIB: %s:%d - op 0x%x (%s), cookie 0x%x -- Return: e->status 0x%x, rc = 0x%x\n", + __func__, __LINE__, opcode, GET_OPCODE_TYPE(opcode), cookie, event->status, rc); + return rc; +} + +/** + * bnxt_qplib_rcfw_send_message - qplib interface to send + * and complete rcfw command. + * @rcfw - rcfw channel instance of rdev + * @msg - qplib message internal + * + * Driver interact with Firmware through rcfw channel/slow path in two ways. + * a. Blocking rcfw command send. In this path, driver cannot hold + * the context for longer period since it is holding cpu until + * command is not completed. + * b. Non-blocking rcfw command send. In this path, driver can hold the + * context for longer period. There may be many pending command waiting + * for completion because of non-blocking nature. + * + * Driver will use shadow queue depth. Current queue depth of 8K + * (due to size of rcfw message it can be actual ~4K rcfw outstanding) + * is not optimal for rcfw command processing in firmware. + * RCFW_CMD_NON_BLOCKING_SHADOW_QD is defined as 64. + * Restrict at max 64 Non-Blocking rcfw commands. + * Do not allow more than 64 non-blocking command to the Firmware. + * Allow all blocking commands until there is no queue full. + * + * Returns: + * 0 if command completed by firmware. + * Non zero if the command is not completed by firmware. + */ +int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg) +{ + int ret; + + if (!msg->block) { + down(&rcfw->rcfw_inflight); + ret = __bnxt_qplib_rcfw_send_message(rcfw, msg); + up(&rcfw->rcfw_inflight); + } else { + ret = __bnxt_qplib_rcfw_send_message(rcfw, msg); + } + + return ret; +} + +static void bnxt_re_add_perf_stats(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_crsqe *crsqe) +{ + u32 latency_msec, dest_stats_id; + u64 *dest_stats_ptr = NULL; + + latency_msec = jiffies_to_msecs(rcfw->cmdq.last_seen - + crsqe->send_timestamp); + if (latency_msec/1000 < RCFW_MAX_LATENCY_SEC_SLAB_INDEX) + rcfw->rcfw_lat_slab_sec[latency_msec/1000]++; + + if (!rcfw->sp_perf_stats_enabled) + return; + + if (latency_msec < RCFW_MAX_LATENCY_MSEC_SLAB_INDEX) + rcfw->rcfw_lat_slab_msec[latency_msec]++; + + switch (crsqe->opcode) { + case CMDQ_BASE_OPCODE_CREATE_QP: + dest_stats_id = rcfw->qp_create_stats_id++; + dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; + dest_stats_ptr = &rcfw->qp_create_stats[dest_stats_id]; + break; + case CMDQ_BASE_OPCODE_DESTROY_QP: + dest_stats_id = rcfw->qp_destroy_stats_id++; + dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; + dest_stats_ptr = &rcfw->qp_destroy_stats[dest_stats_id]; + break; + case CMDQ_BASE_OPCODE_REGISTER_MR: + dest_stats_id = rcfw->mr_create_stats_id++; + dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; + dest_stats_ptr = &rcfw->mr_create_stats[dest_stats_id]; + break; + case CMDQ_BASE_OPCODE_DEREGISTER_MR: + case CMDQ_BASE_OPCODE_DEALLOCATE_KEY: + dest_stats_id = rcfw->mr_destroy_stats_id++; + dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; + dest_stats_ptr = &rcfw->mr_destroy_stats[dest_stats_id]; + break; + case CMDQ_BASE_OPCODE_MODIFY_QP: + if (crsqe->requested_qp_state != IB_QPS_ERR) + break; + dest_stats_id = rcfw->qp_modify_stats_id++; + dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; + dest_stats_ptr = &rcfw->qp_modify_stats[dest_stats_id]; + break; + default: + break; + } + if (dest_stats_ptr) + *dest_stats_ptr = max_t(unsigned long, + (rcfw->cmdq.last_seen - crsqe->send_timestamp), 1); + +} + +/* Completions */ +static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw, + struct creq_qp_event *event, + u32 *num_wait) +{ + struct bnxt_qplib_hwq *cmdq_hwq = &rcfw->cmdq.hwq; + struct creq_cq_error_notification *cqerr; + struct creq_qp_error_notification *qperr; + struct bnxt_qplib_crsqe *crsqe; + struct bnxt_qplib_reftbl *tbl; + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_cq *cq; + u16 cookie, blocked = 0; + struct pci_dev *pdev; + bool is_waiter_alive; + unsigned long flags; + u32 wait_cmds = 0; + u32 xid, qp_idx; + u32 req_size; + int rc = 0; + + pdev = rcfw->pdev; + switch (event->event) { + case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: + tbl = &rcfw->res->reftbl.qpref; + qperr = (struct creq_qp_error_notification *)event; + xid = le32_to_cpu(qperr->xid); + qp_idx = map_qp_id_to_tbl_indx(xid, tbl); + spin_lock(&tbl->lock); + qp = tbl->rec[qp_idx].handle; + if (!qp) { + spin_unlock(&tbl->lock); + break; + } + bnxt_qplib_mark_qp_error(qp); + rc = rcfw->creq.aeq_handler(rcfw, event, qp); + spin_unlock(&tbl->lock); + /* + * Keeping these prints as debug to avoid flooding of log + * messages during modify QP to error state by applications + */ + dev_dbg(&pdev->dev, "QPLIB: QP Error encountered!\n"); + dev_dbg(&pdev->dev, + "QPLIB: qpid 0x%x, req_err=0x%x, resp_err=0x%x\n", + xid, qperr->req_err_state_reason, + qperr->res_err_state_reason); + break; + case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION: + tbl = &rcfw->res->reftbl.cqref; + cqerr = (struct creq_cq_error_notification *)event; + xid = le32_to_cpu(cqerr->xid); + spin_lock(&tbl->lock); + cq = tbl->rec[GET_TBL_INDEX(xid, tbl)].handle; + if (!cq) { + spin_unlock(&tbl->lock); + break; + } + rc = rcfw->creq.aeq_handler(rcfw, event, cq); + spin_unlock(&tbl->lock); + dev_dbg(&pdev->dev, "QPLIB: CQ error encountered!\n"); + break; + default: + /* + * Command Response + * cmdq hwq lock needs to be acquired to synchronize + * the command send and completion reaping. This function + * is always called with creq hwq lock held. So there is no + * chance of deadlock here as the locking is in correct sequence. + * Using the nested variant of spin_lock to annotate + */ + spin_lock_irqsave_nested(&cmdq_hwq->lock, flags, + SINGLE_DEPTH_NESTING); + cookie = le16_to_cpu(event->cookie); + blocked = cookie & RCFW_CMD_IS_BLOCKING; + cookie &= RCFW_MAX_COOKIE_VALUE; + + crsqe = &rcfw->crsqe_tbl[cookie]; + + bnxt_re_add_perf_stats(rcfw, crsqe); + + if (WARN_ONCE(test_bit(FIRMWARE_STALL_DETECTED, + &rcfw->cmdq.flags), + "QPLIB: Unreponsive rcfw channel detected.!!")) { + dev_info(&pdev->dev, "rcfw timedout: cookie = %#x," + " latency_msec = %ld free_slots = %d\n", cookie, + (long)jiffies_to_msecs(rcfw->cmdq.last_seen - + crsqe->send_timestamp), + crsqe->free_slots); + spin_unlock_irqrestore(&cmdq_hwq->lock, flags); + return rc; + } + + if (crsqe->is_internal_cmd && !event->status) + atomic_dec(&rcfw->timeout_send); + + if (crsqe->is_waiter_alive) { + if (crsqe->resp) + memcpy(crsqe->resp, event, sizeof(*event)); + if (!blocked) + wait_cmds++; + } + + req_size = crsqe->req_size; + is_waiter_alive = crsqe->is_waiter_alive; + + crsqe->req_size = 0; + if (!crsqe->is_waiter_alive) + crsqe->resp = NULL; + crsqe->is_in_used = false; + /* Consumer is updated so that __send_message_no_waiter + * can never see queue full. + * It is safe since we are still holding cmdq_hwq->lock. + */ + cmdq_hwq->cons += req_size; + + /* This is a case to handle below scenario - + * Create AH is completed successfully by firmware, + * but completion took more time and driver already lost + * the context of create_ah from caller. + * We have already return failure for create_ah verbs, + * so let's destroy the same address vector since it is + * no more used in stack. We don't care about completion + * in __send_message_no_waiter. + * If destroy_ah is failued by firmware, there will be AH + * resource leak and relatively not critical + unlikely + * scenario. Current design is not to handle such case. + */ + if (!is_waiter_alive && !event->status && + event->event == CREQ_QP_EVENT_EVENT_CREATE_AH) + __destroy_timedout_ah(rcfw, + (struct creq_create_ah_resp *) + event); + + spin_unlock_irqrestore(&cmdq_hwq->lock, flags); + } + *num_wait += wait_cmds; + return rc; +} + +/* SP - CREQ Completion handlers */ +static void bnxt_qplib_service_creq(unsigned long data) +{ + struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data; + struct bnxt_qplib_creq_ctx *creq = &rcfw->creq; + struct bnxt_qplib_res *res; + u32 type, budget = CREQ_ENTRY_POLL_BUDGET; + struct bnxt_qplib_hwq *creq_hwq = &creq->hwq; + struct creq_base *creqe; + struct pci_dev *pdev; + unsigned long flags; + u32 num_wakeup = 0; + int rc; + + pdev = rcfw->pdev; + res = rcfw->res; + /* Service the CREQ until empty */ + spin_lock_irqsave(&creq_hwq->lock, flags); + while (budget > 0) { + if (RCFW_NO_FW_ACCESS(rcfw)) { + spin_unlock_irqrestore(&creq_hwq->lock, flags); + return; + } + creqe = bnxt_qplib_get_qe(creq_hwq, creq_hwq->cons, NULL); + if (!CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags)) + break; + /* The valid test of the entry must be done first before + * reading any further. + */ + dma_rmb(); + type = creqe->type & CREQ_BASE_TYPE_MASK; + rcfw->cmdq.last_seen = jiffies; + + switch (type) { + case CREQ_BASE_TYPE_QP_EVENT: + bnxt_qplib_process_qp_event + (rcfw,(struct creq_qp_event *)creqe, + &num_wakeup); + creq->stats.creq_qp_event_processed++; + break; + case CREQ_BASE_TYPE_FUNC_EVENT: + rc = rcfw->creq.aeq_handler(rcfw, creqe, NULL); + if (rc) + dev_warn(&pdev->dev, + "QPLIB: async event type = 0x%x not handled", + type); + creq->stats.creq_func_event_processed++; + break; + default: + if (type != HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT) { + dev_warn(&pdev->dev, + "QPLIB: op_event = 0x%x not handled\n", + type); + } + break; + } + budget--; + bnxt_qplib_hwq_incr_cons(creq_hwq->max_elements, &creq_hwq->cons, + 1, &creq->creq_db.dbinfo.flags); + } + if (budget == CREQ_ENTRY_POLL_BUDGET && + !CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags)) { + /* No completions received during this poll. Enable interrupt now */ + bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true); + creq->stats.creq_arm_count++; + dev_dbg(&pdev->dev, "QPLIB: Num of Func (0x%llx) \n", + creq->stats.creq_func_event_processed); + dev_dbg(&pdev->dev, "QPLIB: QP (0x%llx) events processed\n", + creq->stats.creq_qp_event_processed); + dev_dbg(&pdev->dev, "QPLIB: Armed:%#llx resched:%#llx \n", + creq->stats.creq_arm_count, + creq->stats.creq_tasklet_schedule_count); + } else if (creq->requested) { + /* + * Currently there is no bottom half implementation to process + * completions, all completions are processed in interrupt context + * only. So enable interrupts. + */ + bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true); + creq->stats.creq_tasklet_schedule_count++; + } + spin_unlock_irqrestore(&creq_hwq->lock, flags); + if (num_wakeup) + wake_up_all(&rcfw->cmdq.waitq); +} + +static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance) +{ + struct bnxt_qplib_rcfw *rcfw = dev_instance; + + bnxt_qplib_service_creq((unsigned long)rcfw); + return IRQ_HANDLED; +} + +/* RCFW */ +int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw) +{ + struct creq_deinitialize_fw_resp resp = {}; + struct cmdq_deinitialize_fw req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEINITIALIZE_FW, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, + sizeof(req), sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags); + return 0; +} + +int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn) +{ + struct creq_initialize_fw_resp resp = {}; + struct cmdq_initialize_fw req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_qplib_ctx *hctx; + struct bnxt_qplib_res *res; + struct bnxt_qplib_hwq *hwq; + int rc; + + res = rcfw->res; + cctx = res->cctx; + hctx = res->hctx; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_INITIALIZE_FW, + sizeof(req)); + /* Supply (log-base-2-of-host-page-size - base-page-shift) + * to bono to adjust the doorbell page sizes. + */ + req.log2_dbr_pg_size = cpu_to_le16(PAGE_SHIFT - + RCFW_DBR_BASE_PAGE_SHIFT); + /* + * VFs need not setup the HW context area, PF + * shall setup this area for VF. Skipping the + * HW programming + */ + if (is_virtfn || _is_chip_gen_p5_p7(cctx)) + goto skip_ctx_setup; + + hwq = &hctx->qp_ctx.hwq; + req.qpc_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.number_of_qp = cpu_to_le32(hwq->max_elements); + req.qpc_pg_size_qpc_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) | + (u8)hwq->level; + + hwq = &hctx->mrw_ctx.hwq; + req.mrw_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.number_of_mrw = cpu_to_le32(hwq->max_elements); + req.mrw_pg_size_mrw_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT) | + (u8)hwq->level; + + hwq = &hctx->srq_ctx.hwq; + req.srq_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.number_of_srq = cpu_to_le32(hwq->max_elements); + req.srq_pg_size_srq_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT) | + (u8)hwq->level; + + hwq = &hctx->cq_ctx.hwq; + req.cq_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.number_of_cq = cpu_to_le32(hwq->max_elements); + req.cq_pg_size_cq_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT) | + (u8)hwq->level; + + hwq = &hctx->tim_ctx.hwq; + req.tim_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.tim_pg_size_tim_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT) | + (u8)hwq->level; + hwq = &hctx->tqm_ctx.pde; + req.tqm_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.tqm_pg_size_tqm_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT) | + (u8)hwq->level; +skip_ctx_setup: + if (BNXT_RE_HW_RETX(res->dattr->dev_cap_flags)) + req.flags |= CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED; + req.stat_ctx_id = cpu_to_le32(hctx->stats.fw_id); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, + sizeof(req), sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags); + + return 0; +} + +void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + + vfree(rcfw->rcfw_lat_slab_msec); + rcfw->rcfw_lat_slab_msec = NULL; + vfree(rcfw->qp_create_stats); + rcfw->qp_create_stats = NULL; + vfree(rcfw->qp_destroy_stats); + rcfw->qp_destroy_stats = NULL; + vfree(rcfw->mr_create_stats); + rcfw->mr_create_stats = NULL; + vfree(rcfw->mr_destroy_stats); + rcfw->mr_destroy_stats = NULL; + vfree(rcfw->qp_modify_stats); + rcfw->qp_modify_stats = NULL; + rcfw->sp_perf_stats_enabled = false; + + kfree(rcfw->crsqe_tbl); + rcfw->crsqe_tbl = NULL; + + bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq); + bnxt_qplib_free_hwq(res, &rcfw->creq.hwq); + rcfw->pdev = NULL; +} + +int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct bnxt_qplib_sg_info sginfo = {}; + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_creq_ctx *creq; + + rcfw->pdev = res->pdev; + rcfw->res = res; + cmdq = &rcfw->cmdq; + creq = &rcfw->creq; + + sginfo.pgsize = PAGE_SIZE; + sginfo.pgshft = PAGE_SHIFT; + + hwq_attr.sginfo = &sginfo; + hwq_attr.res = rcfw->res; + hwq_attr.depth = BNXT_QPLIB_CREQE_MAX_CNT; + hwq_attr.stride = BNXT_QPLIB_CREQE_UNITS; + hwq_attr.type = _get_hwq_type(res); + + if (bnxt_qplib_alloc_init_hwq(&creq->hwq, &hwq_attr)) { + dev_err(&rcfw->pdev->dev, + "QPLIB: HW channel CREQ allocation failed\n"); + return -ENOMEM; + } + + sginfo.pgsize = BNXT_QPLIB_CMDQE_PAGE_SIZE; + hwq_attr.depth = BNXT_QPLIB_CMDQE_MAX_CNT & 0x7FFFFFFF; + hwq_attr.stride = BNXT_QPLIB_CMDQE_UNITS; + hwq_attr.type = HWQ_TYPE_CTX; + if (bnxt_qplib_alloc_init_hwq(&cmdq->hwq, &hwq_attr)) { + dev_err(&rcfw->pdev->dev, + "QPLIB: HW channel CMDQ allocation failed\n"); + goto fail_free_creq_hwq; + } + + rcfw->crsqe_tbl = kcalloc(cmdq->hwq.max_elements, + sizeof(*rcfw->crsqe_tbl), GFP_KERNEL); + if (!rcfw->crsqe_tbl) { + dev_err(&rcfw->pdev->dev, + "QPLIB: HW channel CRSQ allocation failed\n"); + goto fail_free_cmdq_hwq; + } + + rcfw->max_timeout = res->cctx->hwrm_cmd_max_timeout; + + rcfw->sp_perf_stats_enabled = false; + rcfw->rcfw_lat_slab_msec = vzalloc(sizeof(u32) * + RCFW_MAX_LATENCY_MSEC_SLAB_INDEX); + rcfw->qp_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); + rcfw->qp_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); + rcfw->mr_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); + rcfw->mr_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); + rcfw->qp_modify_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); + + if (rcfw->rcfw_lat_slab_msec && + rcfw->qp_create_stats && + rcfw->qp_destroy_stats && + rcfw->mr_create_stats && + rcfw->mr_destroy_stats && + rcfw->qp_modify_stats) + rcfw->sp_perf_stats_enabled = true; + + return 0; +fail_free_cmdq_hwq: + bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq); +fail_free_creq_hwq: + bnxt_qplib_free_hwq(res, &rcfw->creq.hwq); + return -ENOMEM; +} + +void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill) +{ + struct bnxt_qplib_creq_ctx *creq; + struct bnxt_qplib_res *res; + + creq = &rcfw->creq; + res = rcfw->res; + + if (!creq->requested) + return; + + creq->requested = false; + /* Mask h/w interrupts */ + bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, false); + /* Sync with last running IRQ-handler */ + synchronize_irq(creq->msix_vec); + free_irq(creq->msix_vec, rcfw); + kfree(creq->irq_name); + creq->irq_name = NULL; + /* rcfw_intr_enabled should not be greater than 1. Debug + * print to check if that is the case + */ + if (atomic_read(&rcfw->rcfw_intr_enabled) > 1) { + dev_err(&rcfw->pdev->dev, + "%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__, + atomic_read(&rcfw->rcfw_intr_enabled)); + } + atomic_set(&rcfw->rcfw_intr_enabled, 0); + rcfw->num_irq_stopped++; +} + +void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) +{ + struct bnxt_qplib_creq_ctx *creq; + struct bnxt_qplib_cmdq_ctx *cmdq; + + creq = &rcfw->creq; + cmdq = &rcfw->cmdq; + /* Make sure the HW channel is stopped! */ + bnxt_qplib_rcfw_stop_irq(rcfw, true); + + creq->creq_db.reg.bar_reg = NULL; + creq->creq_db.db = NULL; + + if (cmdq->cmdq_mbox.reg.bar_reg) { + iounmap(cmdq->cmdq_mbox.reg.bar_reg); + cmdq->cmdq_mbox.reg.bar_reg = NULL; + cmdq->cmdq_mbox.prod = NULL; + cmdq->cmdq_mbox.db = NULL; + } + + creq->aeq_handler = NULL; + creq->msix_vec = 0; +} + +int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector, + bool need_init) +{ + struct bnxt_qplib_creq_ctx *creq; + struct bnxt_qplib_res *res; + int rc; + + creq = &rcfw->creq; + res = rcfw->res; + + if (creq->requested) + return -EFAULT; + + creq->msix_vec = msix_vector; + + creq->irq_name = kasprintf(GFP_KERNEL, "bnxt_re-creq@pci:%s\n", + pci_name(res->pdev)); + if (!creq->irq_name) + return -ENOMEM; + + rc = request_irq(creq->msix_vec, bnxt_qplib_creq_irq, 0, + creq->irq_name, rcfw); + if (rc) { + kfree(creq->irq_name); + creq->irq_name = NULL; + return rc; + } + creq->requested = true; + + bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true); + + rcfw->num_irq_started++; + /* Debug print to check rcfw interrupt enable/disable is invoked + * out of sequence + */ + if (atomic_read(&rcfw->rcfw_intr_enabled) > 0) { + dev_err(&rcfw->pdev->dev, + "%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__, + atomic_read(&rcfw->rcfw_intr_enabled)); + } + atomic_inc(&rcfw->rcfw_intr_enabled); + return 0; +} + +static int bnxt_qplib_map_cmdq_mbox(struct bnxt_qplib_rcfw *rcfw) +{ + struct bnxt_qplib_cmdq_mbox *mbox; + resource_size_t bar_reg; + struct pci_dev *pdev; + + pdev = rcfw->pdev; + mbox = &rcfw->cmdq.cmdq_mbox; + + mbox->reg.bar_id = RCFW_COMM_PCI_BAR_REGION; + mbox->reg.len = RCFW_COMM_SIZE; + mbox->reg.bar_base = pci_resource_start(pdev, mbox->reg.bar_id); + if (!mbox->reg.bar_base) { + dev_err(&pdev->dev, + "QPLIB: CMDQ BAR region %d resc start is 0!\n", + mbox->reg.bar_id); + return -ENOMEM; + } + + bar_reg = mbox->reg.bar_base + RCFW_COMM_BASE_OFFSET; + mbox->reg.len = RCFW_COMM_SIZE; + mbox->reg.bar_reg = ioremap(bar_reg, mbox->reg.len); + if (!mbox->reg.bar_reg) { + dev_err(&pdev->dev, + "QPLIB: CMDQ BAR region %d mapping failed\n", + mbox->reg.bar_id); + return -ENOMEM; + } + + mbox->prod = (void __iomem *)((char *)mbox->reg.bar_reg + + RCFW_PF_VF_COMM_PROD_OFFSET); + mbox->db = (void __iomem *)((char *)mbox->reg.bar_reg + + RCFW_COMM_TRIG_OFFSET); + return 0; +} + +static int bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw *rcfw, u32 reg_offt) +{ + struct bnxt_qplib_creq_db *creq_db; + struct bnxt_qplib_reg_desc *dbreg; + struct bnxt_qplib_res *res; + + res = rcfw->res; + creq_db = &rcfw->creq.creq_db; + dbreg = &res->dpi_tbl.ucreg; + + creq_db->reg.bar_id = dbreg->bar_id; + creq_db->reg.bar_base = dbreg->bar_base; + creq_db->reg.bar_reg = dbreg->bar_reg + reg_offt; + creq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) : + sizeof(u32); + + creq_db->dbinfo.db = creq_db->reg.bar_reg; + creq_db->dbinfo.hwq = &rcfw->creq.hwq; + creq_db->dbinfo.xid = rcfw->creq.ring_id; + creq_db->dbinfo.seed = rcfw->creq.ring_id; + creq_db->dbinfo.flags = 0; + spin_lock_init(&creq_db->dbinfo.lock); + creq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + creq_db->dbinfo.res = rcfw->res; + + return 0; +} + +static void bnxt_qplib_start_rcfw(struct bnxt_qplib_rcfw *rcfw) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_creq_ctx *creq; + struct bnxt_qplib_cmdq_mbox *mbox; + struct cmdq_init init = {0}; + + cmdq = &rcfw->cmdq; + creq = &rcfw->creq; + mbox = &cmdq->cmdq_mbox; + + init.cmdq_pbl = cpu_to_le64(cmdq->hwq.pbl[PBL_LVL_0].pg_map_arr[0]); + init.cmdq_size_cmdq_lvl = cpu_to_le16( + ((BNXT_QPLIB_CMDQE_MAX_CNT << CMDQ_INIT_CMDQ_SIZE_SFT) & + CMDQ_INIT_CMDQ_SIZE_MASK) | + ((cmdq->hwq.level << CMDQ_INIT_CMDQ_LVL_SFT) & + CMDQ_INIT_CMDQ_LVL_MASK)); + init.creq_ring_id = cpu_to_le16(creq->ring_id); + /* Write to the Bono mailbox register */ + __iowrite32_copy(mbox->reg.bar_reg, &init, sizeof(init) / 4); +} + +int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw, + int msix_vector, + int cp_bar_reg_off, + aeq_handler_t aeq_handler) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_creq_ctx *creq; + int rc; + + cmdq = &rcfw->cmdq; + creq = &rcfw->creq; + + /* Clear to defaults */ + cmdq->seq_num = 0; + set_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags); + init_waitqueue_head(&cmdq->waitq); + + creq->stats.creq_qp_event_processed = 0; + creq->stats.creq_func_event_processed = 0; + creq->aeq_handler = aeq_handler; + + rc = bnxt_qplib_map_cmdq_mbox(rcfw); + if (rc) + return rc; + + rc = bnxt_qplib_map_creq_db(rcfw, cp_bar_reg_off); + if (rc) + return rc; + + rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true); + if (rc) { + dev_err(&rcfw->pdev->dev, + "QPLIB: Failed to request IRQ for CREQ rc = 0x%x\n", rc); + bnxt_qplib_disable_rcfw_channel(rcfw); + return rc; + } + + rcfw->curr_shadow_qd = min_not_zero(cmdq_shadow_qd, + (unsigned int)RCFW_CMD_NON_BLOCKING_SHADOW_QD); + sema_init(&rcfw->rcfw_inflight, rcfw->curr_shadow_qd); + dev_dbg(&rcfw->pdev->dev, + "Perf Debug: shadow qd %d\n", rcfw->curr_shadow_qd); + bnxt_qplib_start_rcfw(rcfw); + + return 0; +} diff --git a/sys/dev/bnxt/bnxt_re/qplib_rcfw.h b/sys/dev/bnxt/bnxt_re/qplib_rcfw.h new file mode 100644 index 00000000000..f117525daac --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_rcfw.h @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: RDMA Controller HW interface (header) + */ + +#ifndef __BNXT_QPLIB_RCFW_H__ +#define __BNXT_QPLIB_RCFW_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qplib_tlv.h" + +#define RCFW_CMDQ_TRIG_VAL 1 +#define RCFW_COMM_PCI_BAR_REGION 0 +#define RCFW_COMM_CONS_PCI_BAR_REGION 2 +#define RCFW_COMM_BASE_OFFSET 0x600 +#define RCFW_PF_VF_COMM_PROD_OFFSET 0xc +#define RCFW_COMM_TRIG_OFFSET 0x100 +#define RCFW_COMM_SIZE 0x104 + +#define RCFW_DBR_PCI_BAR_REGION 2 +#define RCFW_DBR_BASE_PAGE_SHIFT 12 +#define RCFW_MAX_LATENCY_SEC_SLAB_INDEX 128 +#define RCFW_MAX_LATENCY_MSEC_SLAB_INDEX 3000 +#define RCFW_MAX_STAT_INDEX 0xFFFF +#define RCFW_FW_STALL_MAX_TIMEOUT 40 + +#define GET_OPCODE_TYPE(x) \ + ((x) == 0x1 ? "CREATE_QP": \ + ((x) == 0x2 ? "DESTROY_QP": \ + ((x) == 0x3 ? "MODIFY_QP": \ + ((x) == 0x4 ? "QUERY_QP": \ + ((x) == 0x5 ? "CREATE_SRQ": \ + ((x) == 0x6 ? "DESTROY_SRQ": \ + ((x) == 0x8 ? "QUERY_SRQ": \ + ((x) == 0x9 ? "CREATE_CQ": \ + ((x) == 0xa ? "DESTROY_CQ": \ + ((x) == 0xc ? "RESIZE_CQ": \ + ((x) == 0xd ? "ALLOCATE_MRW": \ + ((x) == 0xe ? "DEALLOCATE_KEY": \ + ((x) == 0xf ? "REGISTER_MR": \ + ((x) == 0x10 ? "DEREGISTER_MR": \ + ((x) == 0x11 ? "ADD_GID": \ + ((x) == 0x12 ? "DELETE_GID": \ + ((x) == 0x17 ? "MODIFY_GID": \ + ((x) == 0x18 ? "QUERY_GID": \ + ((x) == 0x13 ? "CREATE_QP1": \ + ((x) == 0x14 ? "DESTROY_QP1": \ + ((x) == 0x15 ? "CREATE_AH": \ + ((x) == 0x16 ? "DESTROY_AH": \ + ((x) == 0x80 ? "INITIALIZE_FW": \ + ((x) == 0x81 ? "DEINITIALIZE_FW": \ + ((x) == 0x82 ? "STOP_FUNC": \ + ((x) == 0x83 ? "QUERY_FUNC": \ + ((x) == 0x84 ? "SET_FUNC_RESOURCES": \ + ((x) == 0x85 ? "READ_CONTEXT": \ + ((x) == 0x86 ? "VF_BACKCHANNEL_REQUEST": \ + ((x) == 0x87 ? "READ_VF_MEMORY": \ + ((x) == 0x88 ? "COMPLETE_VF_REQUEST": \ + ((x) == 0x89 ? "EXTEND_CONTEXT_ARRRAY": \ + ((x) == 0x8a ? "MAP_TC_TO_COS": \ + ((x) == 0x8b ? "QUERY_VERSION": \ + ((x) == 0x8c ? "MODIFY_ROCE_CC": \ + ((x) == 0x8d ? "QUERY_ROCE_CC": \ + ((x) == 0x8e ? "QUERY_ROCE_STATS": \ + ((x) == 0x8f ? "SET_LINK_AGGR_MODE": \ + ((x) == 0x90 ? "MODIFY_CQ": \ + ((x) == 0x91 ? "QUERY_QP_EXTEND": \ + ((x) == 0x92 ? "QUERY_ROCE_STATS_EXT": \ + "Unknown OPCODE" \ + ))))))))))))))))))))))))))))))))))))))))) + +extern unsigned int cmdq_shadow_qd; +/* Cmdq contains a fix number of a 16-Byte slots */ +struct bnxt_qplib_cmdqe { + u8 data[16]; +}; +#define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe) + +static inline void bnxt_qplib_rcfw_cmd_prep(void *r, u8 opcode, u8 cmd_size) +{ + struct cmdq_base *req = r; + + req->opcode = opcode; + req->cmd_size = cmd_size; +} + +/* Shadow queue depth for non blocking command */ +#define RCFW_CMD_NON_BLOCKING_SHADOW_QD 64 +#define RCFW_CMD_DEV_ERR_CHECK_TIME_MS 1000 /* 1 Second time out*/ +#define RCFW_ERR_RETRY_COUNT (RCFW_CMD_WAIT_TIME_MS / RCFW_CMD_DEV_ERR_CHECK_TIME_MS) + +/* CMDQ elements */ +#define BNXT_QPLIB_CMDQE_MAX_CNT 8192 +#define BNXT_QPLIB_CMDQE_BYTES (BNXT_QPLIB_CMDQE_MAX_CNT * \ + BNXT_QPLIB_CMDQE_UNITS) +#define BNXT_QPLIB_CMDQE_NPAGES ((BNXT_QPLIB_CMDQE_BYTES % \ + PAGE_SIZE) ? \ + ((BNXT_QPLIB_CMDQE_BYTES / \ + PAGE_SIZE) + 1) : \ + (BNXT_QPLIB_CMDQE_BYTES / \ + PAGE_SIZE)) +#define BNXT_QPLIB_CMDQE_PAGE_SIZE (BNXT_QPLIB_CMDQE_NPAGES * \ + PAGE_SIZE) + +#define RCFW_MAX_OUTSTANDING_CMD BNXT_QPLIB_CMDQE_MAX_CNT +#define RCFW_MAX_COOKIE_VALUE (BNXT_QPLIB_CMDQE_MAX_CNT - 1) +#define RCFW_CMD_IS_BLOCKING 0x8000 +#define RCFW_NO_FW_ACCESS(rcfw) \ + (test_bit(ERR_DEVICE_DETACHED, &(rcfw)->cmdq.flags) || \ + pci_channel_offline((rcfw)->pdev)) + +/* Crsq buf is 1024-Byte */ +struct bnxt_qplib_crsbe { + u8 data[1024]; +}; + +/* Get the number of command units required for the req. The + * function returns correct value only if called before + * setting using bnxt_qplib_set_cmd_slots + */ +static inline u32 bnxt_qplib_get_cmd_slots(struct cmdq_base *req) +{ + u32 cmd_units = 0; + + if (HAS_TLV_HEADER(req)) { + struct roce_tlv *tlv_req = (struct roce_tlv *)req; + cmd_units = tlv_req->total_size; + } else { + cmd_units = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) / + BNXT_QPLIB_CMDQE_UNITS; + } + return cmd_units; +} + +/* Set the cmd_size to a factor of CMDQE unit */ +static inline u32 bnxt_qplib_set_cmd_slots(struct cmdq_base *req) +{ + u32 cmd_byte = 0; + + if (HAS_TLV_HEADER(req)) { + struct roce_tlv *tlv_req = (struct roce_tlv *)req; + cmd_byte = tlv_req->total_size * BNXT_QPLIB_CMDQE_UNITS; + } else { + cmd_byte = req->cmd_size; + req->cmd_size = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) / + BNXT_QPLIB_CMDQE_UNITS; + } + + return cmd_byte; +} + +/* CREQ */ +/* Allocate 1 per QP for async error notification for now */ +#define BNXT_QPLIB_CREQE_MAX_CNT (64 * 1024) +#define BNXT_QPLIB_CREQE_UNITS 16 /* 16-Bytes per prod unit */ + +#define CREQ_CMP_VALID(hdr, pass) \ + (!!((hdr)->v & CREQ_BASE_V) == \ + !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK)) + +#define CREQ_ENTRY_POLL_BUDGET 8 + +typedef int (*aeq_handler_t)(struct bnxt_qplib_rcfw *, void *, void *); + +struct bnxt_qplib_crsqe { + struct creq_qp_event *resp; + u32 req_size; + bool is_waiter_alive; + bool is_internal_cmd; + bool is_in_used; + + /* Free slots at the time of submission */ + u32 free_slots; + unsigned long send_timestamp; + u8 opcode; + u8 requested_qp_state; +}; + +struct bnxt_qplib_rcfw_sbuf { + void *sb; + dma_addr_t dma_addr; + u32 size; +}; + +#define BNXT_QPLIB_OOS_COUNT_MASK 0xFFFFFFFF + +#define FIRMWARE_INITIALIZED_FLAG (0) +#define FIRMWARE_FIRST_FLAG (31) +#define FIRMWARE_STALL_DETECTED (3) +#define ERR_DEVICE_DETACHED (4) +struct bnxt_qplib_cmdq_mbox { + struct bnxt_qplib_reg_desc reg; + void __iomem *prod; + void __iomem *db; +}; + +struct bnxt_qplib_cmdq_ctx { + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_cmdq_mbox cmdq_mbox; + wait_queue_head_t waitq; + unsigned long flags; + unsigned long last_seen; + u32 seq_num; +}; + +struct bnxt_qplib_creq_db { + struct bnxt_qplib_reg_desc reg; + void __iomem *db; + struct bnxt_qplib_db_info dbinfo; +}; + +struct bnxt_qplib_creq_stat { + u64 creq_arm_count; + u64 creq_tasklet_schedule_count; + u64 creq_qp_event_processed; + u64 creq_func_event_processed; +}; + +struct bnxt_qplib_creq_ctx { + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_creq_db creq_db; + struct bnxt_qplib_creq_stat stats; + aeq_handler_t aeq_handler; + u16 ring_id; + int msix_vec; + bool requested; + char *irq_name; +}; + +/* RCFW Communication Channels */ +#define BNXT_QPLIB_RCFW_SEND_RETRY_COUNT 4000 +struct bnxt_qplib_rcfw { + struct pci_dev *pdev; + struct bnxt_qplib_res *res; + struct bnxt_qplib_cmdq_ctx cmdq; + struct bnxt_qplib_creq_ctx creq; + struct bnxt_qplib_crsqe *crsqe_tbl; + u32 rcfw_lat_slab_sec[RCFW_MAX_LATENCY_SEC_SLAB_INDEX]; + + /* Slow path Perf Stats */ + bool sp_perf_stats_enabled; + u32 *rcfw_lat_slab_msec; + u64 *qp_create_stats; + u64 *qp_destroy_stats; + u64 *qp_modify_stats; + u64 *mr_create_stats; + u64 *mr_destroy_stats; + u32 qp_create_stats_id; + u32 qp_destroy_stats_id; + u32 qp_modify_stats_id; + u32 mr_create_stats_id; + u32 mr_destroy_stats_id; + bool init_oos_stats; + u64 oos_prev; + u32 num_irq_stopped; + u32 num_irq_started; + u32 poll_in_intr_en; + u32 poll_in_intr_dis; + atomic_t rcfw_intr_enabled; + u32 cmdq_full_dbg; + struct semaphore rcfw_inflight; + unsigned int curr_shadow_qd; + atomic_t timeout_send; + /* cached from chip cctx for quick reference in slow path */ + u16 max_timeout; +}; + +struct bnxt_qplib_cmdqmsg { + struct cmdq_base *req; + struct creq_base *resp; + void *sb; + u32 req_sz; + u32 res_sz; + u8 block; + u8 qp_state; +}; + +static inline void bnxt_qplib_fill_cmdqmsg(struct bnxt_qplib_cmdqmsg *msg, + void *req, void *resp, void *sb, + u32 req_sz, u32 res_sz, u8 block) +{ + msg->req = req; + msg->resp = resp; + msg->sb = sb; + msg->req_sz = req_sz; + msg->res_sz = res_sz; + msg->block = block; +} + +void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res); +int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res); +void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill); +void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw); +int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector, + bool need_init); +int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw, + int msix_vector, + int cp_bar_reg_off, + aeq_handler_t aeq_handler); + +struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf( + struct bnxt_qplib_rcfw *rcfw, + u32 size); +void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_rcfw_sbuf *sbuf); +int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg); + +int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw); +int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn); +void bnxt_qplib_mark_qp_error(void *qp_handle); +int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw, + u32 *cur_prod, u32 *cur_cons); +#endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.c b/sys/dev/bnxt/bnxt_re/qplib_res.c new file mode 100644 index 00000000000..69661c67708 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_res.c @@ -0,0 +1,1226 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: QPLib resource manager + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "hsi_struct_def.h" +#include "qplib_res.h" +#include "qplib_sp.h" +#include "qplib_rcfw.h" +#include "bnxt.h" +#include "bnxt_ulp.h" + +uint8_t _get_chip_gen_p5_type(struct bnxt_qplib_chip_ctx *cctx) +{ + /* Extend this for granular type */ + return(BNXT_RE_DEFAULT); +} + +inline bool _is_alloc_mr_unified(struct bnxt_qplib_dev_attr *dattr) +{ + return dattr->dev_cap_flags & + CREQ_QUERY_FUNC_RESP_SB_MR_REGISTER_ALLOC; +} + +/* PBL */ +static void __free_pbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_pbl *pbl, bool is_umem) +{ + struct pci_dev *pdev; + int i; + + pdev = res->pdev; + if (is_umem == false) { + for (i = 0; i < pbl->pg_count; i++) { + if (pbl->pg_arr[i]) { + dma_free_coherent(&pdev->dev, pbl->pg_size, + (void *)((u64)pbl->pg_arr[i] & + PAGE_MASK), + pbl->pg_map_arr[i]); + } + else + dev_warn(&pdev->dev, + "QPLIB: PBL free pg_arr[%d] empty?!\n", + i); + pbl->pg_arr[i] = NULL; + } + } + + if (pbl->pg_arr) { + vfree(pbl->pg_arr); + pbl->pg_arr = NULL; + } + if (pbl->pg_map_arr) { + vfree(pbl->pg_map_arr); + pbl->pg_map_arr = NULL; + } + pbl->pg_count = 0; + pbl->pg_size = 0; +} + +struct qplib_sg { + dma_addr_t pg_map_arr; + u32 size; +}; + +static int __fill_user_dma_pages(struct bnxt_qplib_pbl *pbl, + struct bnxt_qplib_sg_info *sginfo) +{ + int sg_indx, pg_indx, tmp_size, offset; + struct qplib_sg *tmp_sg = NULL; + struct scatterlist *sg; + u64 pmask, addr; + + tmp_sg = vzalloc(sginfo->nmap * sizeof(struct qplib_sg)); + if (!tmp_sg) + return -ENOMEM; + + pmask = BIT_ULL(sginfo->pgshft) - 1; + sg_indx = 0; + for_each_sg(sginfo->sghead, sg, sginfo->nmap, sg_indx) { + tmp_sg[sg_indx].pg_map_arr = sg_dma_address(sg); + tmp_sg[sg_indx].size = sg_dma_len(sg); + } + pg_indx = 0; + for (sg_indx = 0; sg_indx < sginfo->nmap; sg_indx++) { + tmp_size = tmp_sg[sg_indx].size; + offset = 0; + while (tmp_size > 0) { + addr = tmp_sg[sg_indx].pg_map_arr + offset; + if ((!sg_indx && !pg_indx) || !(addr & pmask)) { + pbl->pg_map_arr[pg_indx] = addr &(~pmask); + pbl->pg_count++; + pg_indx++; + } + offset += sginfo->pgsize; + tmp_size -= sginfo->pgsize; + } + } + + vfree(tmp_sg); + return 0; +} + +static int bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl, + struct bnxt_qplib_sg_info *sginfo) +{ + int rc = 0; + + rc = __fill_user_dma_pages(pbl, sginfo); + + return rc; +} + +static int __alloc_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl, + struct bnxt_qplib_sg_info *sginfo) +{ + struct pci_dev *pdev; + bool is_umem = false; + int i; + + if (sginfo->nopte) + return 0; + + pdev = res->pdev; + /* page ptr arrays */ + pbl->pg_arr = vmalloc(sginfo->npages * sizeof(void *)); + if (!pbl->pg_arr) + return -ENOMEM; + + pbl->pg_map_arr = vmalloc(sginfo->npages * sizeof(dma_addr_t)); + if (!pbl->pg_map_arr) { + vfree(pbl->pg_arr); + return -ENOMEM; + } + pbl->pg_count = 0; + pbl->pg_size = sginfo->pgsize; + if (!sginfo->sghead) { + for (i = 0; i < sginfo->npages; i++) { + pbl->pg_arr[i] = dma_zalloc_coherent(&pdev->dev, + pbl->pg_size, + &pbl->pg_map_arr[i], + GFP_KERNEL); + if (!pbl->pg_arr[i]) + goto fail; + pbl->pg_count++; + } + } else { + is_umem = true; + if (bnxt_qplib_fill_user_dma_pages(pbl, sginfo)) + goto fail; + } + + return 0; +fail: + __free_pbl(res, pbl, is_umem); + return -ENOMEM; +} + +/* HWQ */ +void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res, + struct bnxt_qplib_hwq *hwq) +{ + int i; + + if (!hwq->max_elements) + return; + if (hwq->level >= PBL_LVL_MAX) + return; + + for (i = 0; i < hwq->level + 1; i++) { + if (i == hwq->level) + __free_pbl(res, &hwq->pbl[i], hwq->is_user); + else + __free_pbl(res, &hwq->pbl[i], false); + } + + hwq->level = PBL_LVL_MAX; + hwq->max_elements = 0; + hwq->element_size = 0; + hwq->prod = hwq->cons = 0; + hwq->cp_bit = 0; +} + +/* All HWQs are power of 2 in size */ +int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, + struct bnxt_qplib_hwq_attr *hwq_attr) +{ + u32 npages = 0, depth, stride, aux_pages = 0; + dma_addr_t *src_phys_ptr, **dst_virt_ptr; + struct bnxt_qplib_sg_info sginfo = {}; + u32 aux_size = 0, npbl, npde; + void *umem; + struct bnxt_qplib_res *res; + u32 aux_slots, pg_size; + struct pci_dev *pdev; + int i, rc, lvl; + + res = hwq_attr->res; + pdev = res->pdev; + umem = hwq_attr->sginfo->sghead; + pg_size = hwq_attr->sginfo->pgsize; + hwq->level = PBL_LVL_MAX; + + depth = roundup_pow_of_two(hwq_attr->depth); + stride = roundup_pow_of_two(hwq_attr->stride); + if (hwq_attr->aux_depth) { + aux_slots = hwq_attr->aux_depth; + aux_size = roundup_pow_of_two(hwq_attr->aux_stride); + aux_pages = (aux_slots * aux_size) / pg_size; + if ((aux_slots * aux_size) % pg_size) + aux_pages++; + } + + if (!umem) { + hwq->is_user = false; + npages = (depth * stride) / pg_size + aux_pages; + if ((depth * stride) % pg_size) + npages++; + if (!npages) + return -EINVAL; + hwq_attr->sginfo->npages = npages; + } else { + hwq->is_user = true; + npages = hwq_attr->sginfo->npages; + npages = (npages * (u64)pg_size) / + BIT_ULL(hwq_attr->sginfo->pgshft); + if ((hwq_attr->sginfo->npages * (u64)pg_size) % + BIT_ULL(hwq_attr->sginfo->pgshft)) + npages++; + } + if (npages == MAX_PBL_LVL_0_PGS && !hwq_attr->sginfo->nopte) { + /* This request is Level 0, map PTE */ + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], hwq_attr->sginfo); + if (rc) + goto fail; + hwq->level = PBL_LVL_0; + goto done; + } + + if (npages >= MAX_PBL_LVL_0_PGS) { + if (npages > MAX_PBL_LVL_1_PGS) { + u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ? + 0 : PTU_PTE_VALID; + /* 2 levels of indirection */ + npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT; + if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT)) + npbl++; + npde = npbl >> MAX_PDL_LVL_SHIFT; + if(npbl % BIT(MAX_PDL_LVL_SHIFT)) + npde++; + /* Alloc PDE pages */ + sginfo.pgsize = npde * PAGE_SIZE; + sginfo.npages = 1; + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo); + + /* Alloc PBL pages */ + sginfo.npages = npbl; + sginfo.pgsize = PAGE_SIZE; + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo); + if (rc) + goto fail; + /* Fill PDL with PBL page pointers */ + dst_virt_ptr = + (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; + src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; + if (hwq_attr->type == HWQ_TYPE_MR) { + /* For MR it is expected that we supply only 1 contigous + * page i.e only 1 entry in the PDL that will contain + * all the PBLs for the user supplied memory region + */ + for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) + dst_virt_ptr[0][i] = src_phys_ptr[i] | + flag; + } else { + for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = + src_phys_ptr[i] | PTU_PDE_VALID; + } + /* Alloc or init PTEs */ + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_2], + hwq_attr->sginfo); + if (rc) + goto fail; + hwq->level = PBL_LVL_2; + if (hwq_attr->sginfo->nopte) + goto done; + /* Fill PBLs with PTE pointers */ + dst_virt_ptr = + (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr; + src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr; + for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) { + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = + src_phys_ptr[i] | PTU_PTE_VALID; + } + if (hwq_attr->type == HWQ_TYPE_QUEUE) { + /* Find the last pg of the size */ + i = hwq->pbl[PBL_LVL_2].pg_count; + dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= + PTU_PTE_LAST; + if (i > 1) + dst_virt_ptr[PTR_PG(i - 2)] + [PTR_IDX(i - 2)] |= + PTU_PTE_NEXT_TO_LAST; + } + } else { /* pages < 512 npbl = 1, npde = 0 */ + u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ? + 0 : PTU_PTE_VALID; + + /* 1 level of indirection */ + npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT; + if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT)) + npbl++; + sginfo.npages = npbl; + sginfo.pgsize = PAGE_SIZE; + /* Alloc PBL page */ + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo); + if (rc) + goto fail; + /* Alloc or init PTEs */ + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], + hwq_attr->sginfo); + if (rc) + goto fail; + hwq->level = PBL_LVL_1; + if (hwq_attr->sginfo->nopte) + goto done; + /* Fill PBL with PTE pointers */ + dst_virt_ptr = + (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; + src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; + for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = + src_phys_ptr[i] | flag; + if (hwq_attr->type == HWQ_TYPE_QUEUE) { + /* Find the last pg of the size */ + i = hwq->pbl[PBL_LVL_1].pg_count; + dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= + PTU_PTE_LAST; + if (i > 1) + dst_virt_ptr[PTR_PG(i - 2)] + [PTR_IDX(i - 2)] |= + PTU_PTE_NEXT_TO_LAST; + } + } + } +done: + hwq->prod = 0; + hwq->cons = 0; + hwq->pdev = pdev; + hwq->depth = hwq_attr->depth; + hwq->max_elements = depth; + hwq->element_size = stride; + hwq->qe_ppg = (pg_size/stride); + + if (hwq->level >= PBL_LVL_MAX) + goto fail; + /* For direct access to the elements */ + lvl = hwq->level; + if (hwq_attr->sginfo->nopte && hwq->level) + lvl = hwq->level - 1; + hwq->pbl_ptr = hwq->pbl[lvl].pg_arr; + hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr; + spin_lock_init(&hwq->lock); + + return 0; +fail: + bnxt_qplib_free_hwq(res, hwq); + return -ENOMEM; +} + +/* Context Tables */ +void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_ctx *hctx; + int i; + + hctx = res->hctx; + bnxt_qplib_free_hwq(res, &hctx->qp_ctx.hwq); + bnxt_qplib_free_hwq(res, &hctx->mrw_ctx.hwq); + bnxt_qplib_free_hwq(res, &hctx->srq_ctx.hwq); + bnxt_qplib_free_hwq(res, &hctx->cq_ctx.hwq); + bnxt_qplib_free_hwq(res, &hctx->tim_ctx.hwq); + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) + bnxt_qplib_free_hwq(res, &hctx->tqm_ctx.qtbl[i]); + /* restor original pde level before destroy */ + hctx->tqm_ctx.pde.level = hctx->tqm_ctx.pde_level; + bnxt_qplib_free_hwq(res, &hctx->tqm_ctx.pde); +} + +static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res, + struct bnxt_qplib_ctx *hctx) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_sg_info sginfo = {}; + struct bnxt_qplib_tqm_ctx *tqmctx; + int rc = 0; + int i; + + tqmctx = &hctx->tqm_ctx; + + sginfo.pgsize = PAGE_SIZE; + sginfo.pgshft = PAGE_SHIFT; + hwq_attr.sginfo = &sginfo; + hwq_attr.res = res; + hwq_attr.type = HWQ_TYPE_CTX; + hwq_attr.depth = 512; + hwq_attr.stride = sizeof(u64); + /* Alloc pdl buffer */ + rc = bnxt_qplib_alloc_init_hwq(&tqmctx->pde, &hwq_attr); + if (rc) + goto out; + /* Save original pdl level */ + tqmctx->pde_level = tqmctx->pde.level; + + hwq_attr.stride = 1; + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) { + if (!tqmctx->qcount[i]) + continue; + hwq_attr.depth = hctx->qp_ctx.max * tqmctx->qcount[i]; + rc = bnxt_qplib_alloc_init_hwq(&tqmctx->qtbl[i], &hwq_attr); + if (rc) + goto out; + } +out: + return rc; +} + +static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx) +{ + struct bnxt_qplib_hwq *qtbl_hwq; + dma_addr_t *dma_ptr; + __le64 **pbl_ptr, *ptr; + int i, j, k; + int fnz_idx = -1; + int pg_count; + + pbl_ptr = (__le64 **)ctx->pde.pbl_ptr; + + for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ; + i++, j += MAX_TQM_ALLOC_BLK_SIZE) { + qtbl_hwq = &ctx->qtbl[i]; + if (!qtbl_hwq->max_elements) + continue; + if (fnz_idx == -1) + fnz_idx = i; /* first non-zero index */ + switch (qtbl_hwq->level) { + case PBL_LVL_2: + pg_count = qtbl_hwq->pbl[PBL_LVL_1].pg_count; + for (k = 0; k < pg_count; k++) { + ptr = &pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)]; + dma_ptr = &qtbl_hwq->pbl[PBL_LVL_1].pg_map_arr[k]; + *ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID); + } + break; + case PBL_LVL_1: + case PBL_LVL_0: + default: + ptr = &pbl_ptr[PTR_PG(j)][PTR_IDX(j)]; + *ptr = cpu_to_le64(qtbl_hwq->pbl[PBL_LVL_0].pg_map_arr[0] | + PTU_PTE_VALID); + break; + } + } + if (fnz_idx == -1) + fnz_idx = 0; + /* update pde level as per page table programming */ + ctx->pde.level = (ctx->qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 : + ctx->qtbl[fnz_idx].level + 1; +} + +static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res, + struct bnxt_qplib_ctx *hctx) +{ + int rc = 0; + + rc = bnxt_qplib_alloc_tqm_rings(res, hctx); + if (rc) + goto fail; + + bnxt_qplib_map_tqm_pgtbl(&hctx->tqm_ctx); +fail: + return rc; +} + +/* + * Routine: bnxt_qplib_alloc_hwctx + * Description: + * Context tables are memories which are used by the chip. + * The 6 tables defined are: + * QPC ctx - holds QP states + * MRW ctx - holds memory region and window + * SRQ ctx - holds shared RQ states + * CQ ctx - holds completion queue states + * TQM ctx - holds Tx Queue Manager context + * TIM ctx - holds timer context + * Depending on the size of the tbl requested, either a 1 Page Buffer List + * or a 1-to-2-stage indirection Page Directory List + 1 PBL is used + * instead. + * Table might be employed as follows: + * For 0 < ctx size <= 1 PAGE, 0 level of ind is used + * For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used + * For 512 < ctx size <= MAX, 2 levels of ind is used + * Returns: + * 0 if success, else -ERRORS + */ +int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_sg_info sginfo = {}; + struct bnxt_qplib_ctx *hctx; + struct bnxt_qplib_hwq *hwq; + int rc = 0; + + hctx = res->hctx; + /* QPC Tables */ + sginfo.pgsize = PAGE_SIZE; + sginfo.pgshft = PAGE_SHIFT; + hwq_attr.sginfo = &sginfo; + + hwq_attr.res = res; + hwq_attr.depth = hctx->qp_ctx.max; + hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE; + hwq_attr.type = HWQ_TYPE_CTX; + hwq = &hctx->qp_ctx.hwq; + rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); + if (rc) + goto fail; + + /* MRW Tables */ + hwq_attr.depth = hctx->mrw_ctx.max; + hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE; + hwq = &hctx->mrw_ctx.hwq; + rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); + if (rc) + goto fail; + + /* SRQ Tables */ + hwq_attr.depth = hctx->srq_ctx.max; + hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE; + hwq = &hctx->srq_ctx.hwq; + rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); + if (rc) + goto fail; + + /* CQ Tables */ + hwq_attr.depth = hctx->cq_ctx.max; + hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE; + hwq = &hctx->cq_ctx.hwq; + rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); + if (rc) + goto fail; + + /* TQM Buffer */ + rc = bnxt_qplib_setup_tqm_rings(res, hctx); + if (rc) + goto fail; + /* TIM Buffer */ + hwq_attr.depth = hctx->qp_ctx.max * 16; + hwq_attr.stride = 1; + hwq = &hctx->tim_ctx.hwq; + rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); + if (rc) + goto fail; + + return 0; +fail: + bnxt_qplib_free_hwctx(res); + return rc; +} + +/* GUID */ +void bnxt_qplib_get_guid(const u8 *dev_addr, u8 *guid) +{ + u8 mac[ETH_ALEN]; + + /* MAC-48 to EUI-64 mapping */ + memcpy(mac, dev_addr, ETH_ALEN); + guid[0] = mac[0] ^ 2; + guid[1] = mac[1]; + guid[2] = mac[2]; + guid[3] = 0xff; + guid[4] = 0xfe; + guid[5] = mac[3]; + guid[6] = mac[4]; + guid[7] = mac[5]; +} + +static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_sgid_tbl *sgid_tbl; + + sgid_tbl = &res->sgid_tbl; + + if (sgid_tbl->tbl) { + kfree(sgid_tbl->tbl); + sgid_tbl->tbl = NULL; + kfree(sgid_tbl->hw_id); + sgid_tbl->hw_id = NULL; + kfree(sgid_tbl->ctx); + sgid_tbl->ctx = NULL; + kfree(sgid_tbl->vlan); + sgid_tbl->vlan = NULL; + } else { + dev_dbg(&res->pdev->dev, "QPLIB: SGID tbl not present"); + } + sgid_tbl->max = 0; + sgid_tbl->active = 0; +} + +static void bnxt_qplib_free_reftbls(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_reftbl *tbl; + + tbl = &res->reftbl.srqref; + vfree(tbl->rec); + + tbl = &res->reftbl.cqref; + vfree(tbl->rec); + + tbl = &res->reftbl.qpref; + vfree(tbl->rec); +} + +static int bnxt_qplib_alloc_reftbl(struct bnxt_qplib_reftbl *tbl, u32 max) +{ + tbl->max = max; + tbl->rec = vzalloc(sizeof(*tbl->rec) * max); + if (!tbl->rec) + return -ENOMEM; + spin_lock_init(&tbl->lock); + return 0; +} + +static int bnxt_qplib_alloc_reftbls(struct bnxt_qplib_res *res, + struct bnxt_qplib_dev_attr *dattr) +{ + u32 max_cq = BNXT_QPLIB_MAX_CQ_COUNT; + struct bnxt_qplib_reftbl *tbl; + u32 res_cnt; + int rc; + + /* + * Allocating one extra entry to hold QP1 info. + * Store QP1 info at the last entry of the table. + * Decrement the tbl->max by one so that modulo + * operation to get the qp table index from qp id + * returns any value between 0 and max_qp-1 + */ + res_cnt = max_t(u32, BNXT_QPLIB_MAX_QPC_COUNT + 1, dattr->max_qp); + tbl = &res->reftbl.qpref; + rc = bnxt_qplib_alloc_reftbl(tbl, res_cnt); + if (rc) + goto fail; + tbl->max--; + + if (_is_chip_gen_p5_p7(res->cctx)) + max_cq = BNXT_QPLIB_MAX_CQ_COUNT_P5; + res_cnt = max_t(u32, max_cq, dattr->max_cq); + tbl = &res->reftbl.cqref; + rc = bnxt_qplib_alloc_reftbl(tbl, res_cnt); + if (rc) + goto fail; + + res_cnt = max_t(u32, BNXT_QPLIB_MAX_SRQC_COUNT, dattr->max_cq); + tbl = &res->reftbl.srqref; + rc = bnxt_qplib_alloc_reftbl(tbl, BNXT_QPLIB_MAX_SRQC_COUNT); + if (rc) + goto fail; + + return 0; +fail: + return rc; +} + +static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res, u16 max) +{ + struct bnxt_qplib_sgid_tbl *sgid_tbl; + + sgid_tbl = &res->sgid_tbl; + + sgid_tbl->tbl = kcalloc(max, sizeof(*sgid_tbl->tbl), GFP_KERNEL); + if (!sgid_tbl->tbl) + return -ENOMEM; + + sgid_tbl->hw_id = kcalloc(max, sizeof(u32), GFP_KERNEL); + if (!sgid_tbl->hw_id) + goto free_tbl; + + sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL); + if (!sgid_tbl->ctx) + goto free_hw_id; + + sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL); + if (!sgid_tbl->vlan) + goto free_ctx; + + sgid_tbl->max = max; + return 0; +free_ctx: + kfree(sgid_tbl->ctx); +free_hw_id: + kfree(sgid_tbl->hw_id); +free_tbl: + kfree(sgid_tbl->tbl); + return -ENOMEM; +}; + +static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_sgid_tbl *sgid_tbl) +{ + int i; + + for (i = 0; i < sgid_tbl->max; i++) { + if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero))) + bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i].gid, + sgid_tbl->tbl[i].vlan_id, true); + } + memset(sgid_tbl->tbl, 0, sizeof(*sgid_tbl->tbl) * sgid_tbl->max); + memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); + memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max); + sgid_tbl->active = 0; +} + +static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct ifnet *netdev) +{ + u32 i; + + for (i = 0; i < sgid_tbl->max; i++) + sgid_tbl->tbl[i].vlan_id = 0xffff; + memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); +} + +/* PDs */ +int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd *pd) +{ + u32 bit_num; + int rc = 0; + struct bnxt_qplib_pd_tbl *pdt = &res->pd_tbl; + + mutex_lock(&res->pd_tbl_lock); + bit_num = find_first_bit(pdt->tbl, pdt->max); + if (bit_num == pdt->max - 1) {/* Last bit is reserved */ + rc = -ENOMEM; + goto fail; + } + + /* Found unused PD */ + clear_bit(bit_num, pdt->tbl); + pd->id = bit_num; +fail: + mutex_unlock(&res->pd_tbl_lock); + return rc; +} + +int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, + struct bnxt_qplib_pd_tbl *pdt, + struct bnxt_qplib_pd *pd) +{ + mutex_lock(&res->pd_tbl_lock); + if (test_and_set_bit(pd->id, pdt->tbl)) { + dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n", + pd->id); + mutex_unlock(&res->pd_tbl_lock); + return -EINVAL; + } + /* Reset to reserved pdid. */ + pd->id = pdt->max - 1; + + mutex_unlock(&res->pd_tbl_lock); + return 0; +} + +static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt) +{ + if (pdt->tbl) { + kfree(pdt->tbl); + pdt->tbl = NULL; + } + pdt->max = 0; +} + +static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res, u32 max) +{ + struct bnxt_qplib_pd_tbl *pdt; + u32 bytes; + + pdt = &res->pd_tbl; + + max++; /* One extra for reserved pdid. */ + bytes = DIV_ROUND_UP(max, 8); + + if (!bytes) + bytes = 1; + pdt->tbl = kmalloc(bytes, GFP_KERNEL); + if (!pdt->tbl) { + dev_err(&res->pdev->dev, + "QPLIB: PD tbl allocation failed for size = %d\n", bytes); + return -ENOMEM; + } + pdt->max = max; + memset((u8 *)pdt->tbl, 0xFF, bytes); + mutex_init(&res->pd_tbl_lock); + + return 0; +} + +/* DPIs */ +int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi *dpi, + void *app, u8 type) +{ + struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl; + struct bnxt_qplib_reg_desc *reg; + u32 bit_num; + u64 umaddr; + int rc = 0; + + reg = &dpit->wcreg; + mutex_lock(&res->dpi_tbl_lock); + if (type == BNXT_QPLIB_DPI_TYPE_WC && _is_chip_p7(res->cctx) && + !dpit->avail_ppp) { + rc = -ENOMEM; + goto fail; + } + bit_num = find_first_bit(dpit->tbl, dpit->max); + if (bit_num == dpit->max) { + rc = -ENOMEM; + goto fail; + } + /* Found unused DPI */ + clear_bit(bit_num, dpit->tbl); + dpit->app_tbl[bit_num] = app; + dpi->bit = bit_num; + dpi->dpi = bit_num + (reg->offset - dpit->ucreg.offset) / PAGE_SIZE; + + umaddr = reg->bar_base + reg->offset + bit_num * PAGE_SIZE; + dpi->umdbr = umaddr; + switch (type) { + case BNXT_QPLIB_DPI_TYPE_KERNEL: + /* priviledged dbr was already mapped just initialize it. */ + dpi->umdbr = dpit->ucreg.bar_base + + dpit->ucreg.offset + bit_num * PAGE_SIZE; + dpi->dbr = dpit->priv_db; + dpi->dpi = dpi->bit; + break; + case BNXT_QPLIB_DPI_TYPE_WC: + dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE); + if (_is_chip_p7(res->cctx) && dpi->dbr) + dpit->avail_ppp--; + break; + default: + dpi->dbr = ioremap(umaddr, PAGE_SIZE); + } + if (!dpi->dbr) { + dev_err(&res->pdev->dev, "QPLIB: DB remap failed, type = %d\n", + type); + rc = -ENOMEM; + } + dpi->type = type; +fail: + mutex_unlock(&res->dpi_tbl_lock); + return rc; +} + +int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi *dpi) +{ + struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl; + int rc = 0; + + mutex_lock(&res->dpi_tbl_lock); + if (dpi->bit >= dpit->max) { + dev_warn(&res->pdev->dev, + "Invalid DPI? dpi = %d, bit = %d\n", + dpi->dpi, dpi->bit); + rc = -EINVAL; + goto fail; + } + + if (dpi->dpi && dpi->type != BNXT_QPLIB_DPI_TYPE_KERNEL) { + if (dpi->type == BNXT_QPLIB_DPI_TYPE_WC && + _is_chip_p7(res->cctx) && dpi->dbr) + dpit->avail_ppp++; + pci_iounmap(res->pdev, dpi->dbr); + } + + if (test_and_set_bit(dpi->bit, dpit->tbl)) { + dev_warn(&res->pdev->dev, + "Freeing an unused DPI? dpi = %d, bit = %d\n", + dpi->dpi, dpi->bit); + rc = -EINVAL; + goto fail; + } + if (dpit->app_tbl) + dpit->app_tbl[dpi->bit] = NULL; + memset(dpi, 0, sizeof(*dpi)); +fail: + mutex_unlock(&res->dpi_tbl_lock); + return rc; +} + +static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_dpi_tbl *dpit) +{ + kfree(dpit->tbl); + kfree(dpit->app_tbl); + dpit->tbl = NULL; + dpit->app_tbl = NULL; + dpit->max = 0; +} + +static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_dev_attr *dev_attr, + u8 pppp_factor) +{ + struct bnxt_qplib_dpi_tbl *dpit; + struct bnxt_qplib_reg_desc *reg; + unsigned long bar_len; + u32 dbr_offset; + u32 bytes; + + dpit = &res->dpi_tbl; + reg = &dpit->wcreg; + + if (!_is_chip_gen_p5_p7(res->cctx)) { + /* Offest should come from L2 driver */ + dbr_offset = dev_attr->l2_db_size; + dpit->ucreg.offset = dbr_offset; + dpit->wcreg.offset = dbr_offset; + } + + bar_len = pci_resource_len(res->pdev, reg->bar_id); + dpit->max = (bar_len - reg->offset) / PAGE_SIZE; + if (dev_attr->max_dpi) + dpit->max = min_t(u32, dpit->max, dev_attr->max_dpi); + + dpit->app_tbl = kzalloc(dpit->max * sizeof(void*), GFP_KERNEL); + if (!dpit->app_tbl) { + dev_err(&res->pdev->dev, + "QPLIB: DPI app tbl allocation failed"); + return -ENOMEM; + } + + bytes = dpit->max >> 3; + if (!bytes) + bytes = 1; + + dpit->tbl = kmalloc(bytes, GFP_KERNEL); + if (!dpit->tbl) { + kfree(dpit->app_tbl); + dev_err(&res->pdev->dev, + "QPLIB: DPI tbl allocation failed for size = %d\n", + bytes); + return -ENOMEM; + } + + memset((u8 *)dpit->tbl, 0xFF, bytes); + /* + * On SR2, 2nd doorbell page of each function + * is reserved for L2 PPP. Now that the tbl is + * initialized, mark it as unavailable. By default + * RoCE can make use of the 512 extended pages for + * PPP. + */ + if (_is_chip_p7(res->cctx)) { + clear_bit(1, dpit->tbl); + if (pppp_factor) + dpit->avail_ppp = + BNXT_QPLIB_MAX_EXTENDED_PPP_PAGES / pppp_factor; + } + mutex_init(&res->dpi_tbl_lock); + dpit->priv_db = dpit->ucreg.bar_reg + dpit->ucreg.offset; + + return 0; +} + +/* Stats */ +void bnxt_qplib_free_stat_mem(struct bnxt_qplib_res *res, + struct bnxt_qplib_stats *stats) +{ + struct pci_dev *pdev; + + pdev = res->pdev; + if (stats->dma) + dma_free_coherent(&pdev->dev, stats->size, + stats->dma, stats->dma_map); + + memset(stats, 0, sizeof(*stats)); + stats->fw_id = -1; +} + +int bnxt_qplib_alloc_stat_mem(struct pci_dev *pdev, + struct bnxt_qplib_chip_ctx *cctx, + struct bnxt_qplib_stats *stats) +{ + cctx->hw_stats_size = 168; + + memset(stats, 0, sizeof(*stats)); + stats->fw_id = -1; + stats->size = cctx->hw_stats_size; + stats->dma = dma_alloc_coherent(&pdev->dev, stats->size, + &stats->dma_map, GFP_KERNEL); + if (!stats->dma) { + dev_err(&pdev->dev, "QPLIB: Stats DMA allocation failed"); + return -ENOMEM; + } + return 0; +} + +/* Resource */ +int bnxt_qplib_stop_res(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_stop_func_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_stop_func req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_STOP_FUNC, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + return rc; +} + +void bnxt_qplib_clear_tbls(struct bnxt_qplib_res *res) +{ + bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl); +} + +int bnxt_qplib_init_tbls(struct bnxt_qplib_res *res) +{ + bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev); + + return 0; +} + +void bnxt_qplib_free_tbls(struct bnxt_qplib_res *res) +{ + bnxt_qplib_free_sgid_tbl(res); + bnxt_qplib_free_pd_tbl(&res->pd_tbl); + bnxt_qplib_free_dpi_tbl(&res->dpi_tbl); + bnxt_qplib_free_reftbls(res); +} + +int bnxt_qplib_alloc_tbls(struct bnxt_qplib_res *res, u8 pppp_factor) +{ + struct bnxt_qplib_dev_attr *dev_attr; + int rc = 0; + + dev_attr = res->dattr; + + rc = bnxt_qplib_alloc_reftbls(res, dev_attr); + if (rc) + goto fail; + + rc = bnxt_qplib_alloc_sgid_tbl(res, dev_attr->max_sgid); + if (rc) + goto fail; + + rc = bnxt_qplib_alloc_pd_tbl(res, dev_attr->max_pd); + if (rc) + goto fail; + + rc = bnxt_qplib_alloc_dpi_tbl(res, dev_attr, pppp_factor); + if (rc) + goto fail; + + return 0; +fail: + bnxt_qplib_free_tbls(res); + return rc; +} + +void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_reg_desc *reg; + + reg = &res->dpi_tbl.ucreg; + if (reg->bar_reg) + pci_iounmap(res->pdev, reg->bar_reg); + reg->bar_reg = NULL; + reg->bar_base = 0; + reg->len = 0; + reg->bar_id = 0; /* Zero? or ff */ +} + +int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_reg_desc *ucreg; + struct bnxt_qplib_reg_desc *wcreg; + + wcreg = &res->dpi_tbl.wcreg; + wcreg->bar_id = RCFW_DBR_PCI_BAR_REGION; + if (!res || !res->pdev || !wcreg) + return -1; + wcreg->bar_base = pci_resource_start(res->pdev, wcreg->bar_id); + /* No need to set the wcreg->len here */ + + ucreg = &res->dpi_tbl.ucreg; + ucreg->bar_id = RCFW_DBR_PCI_BAR_REGION; + ucreg->bar_base = pci_resource_start(res->pdev, ucreg->bar_id); + + ucreg->offset = 65536; + + ucreg->len = ucreg->offset + PAGE_SIZE; + + if (!ucreg->len || ((ucreg->len & (PAGE_SIZE - 1)) != 0)) { + dev_err(&res->pdev->dev, "QPLIB: invalid dbr length %d\n", + (int)ucreg->len); + return -EINVAL; + } + ucreg->bar_reg = ioremap(ucreg->bar_base, ucreg->len); + if (!ucreg->bar_reg) { + dev_err(&res->pdev->dev, "priviledged dpi map failed!\n"); + return -ENOMEM; + } + + return 0; +} + +/** + * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port + * @dev: the PCI device + * @cap_mask: mask of desired AtomicOp sizes, including one or more of: + * PCI_EXP_DEVCAP2_ATOMIC_COMP32 + * PCI_EXP_DEVCAP2_ATOMIC_COMP64 + * PCI_EXP_DEVCAP2_ATOMIC_COMP128 + * + * Return 0 if all upstream bridges support AtomicOp routing, egress + * blocking is disabled on all upstream ports, and the root port supports + * the requested completion capabilities (32-bit, 64-bit and/or 128-bit + * AtomicOp completion), or negative otherwise. + */ +int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) +{ + struct pci_bus *bus = dev->bus; + struct pci_dev *bridge; + u32 cap; + + if (!pci_is_pcie(dev)) + return -EINVAL; + + /* + * Per PCIe r4.0, sec 6.15, endpoints and root ports may be + * AtomicOp requesters. For now, we only support endpoints as + * requesters and root ports as completers. No endpoints as + * completers, and no peer-to-peer. + */ + + switch (pci_pcie_type(dev)) { + case PCI_EXP_TYPE_ENDPOINT: + case PCI_EXP_TYPE_LEG_END: + break; + default: + return -EINVAL; + } + + bridge = bus->self; + + pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); + + switch (pci_pcie_type(bridge)) { + case PCI_EXP_TYPE_DOWNSTREAM: + if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) + return -EINVAL; + break; + + /* Ensure root port supports all the sizes we care about */ + case PCI_EXP_TYPE_ROOT_PORT: + if ((cap & cap_mask) != cap_mask) + return -EINVAL; + break; + } + return 0; +} + +int bnxt_qplib_enable_atomic_ops_to_root(struct pci_dev *dev) +{ + u16 ctl2; + + if(pci_enable_atomic_ops_to_root(dev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) && + pci_enable_atomic_ops_to_root(dev, PCI_EXP_DEVCAP2_ATOMIC_COMP64)) + return -EOPNOTSUPP; + + pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &ctl2); + return !(ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ); +} diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.h b/sys/dev/bnxt/bnxt_re/qplib_res.h new file mode 100644 index 00000000000..6468207a49a --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_res.h @@ -0,0 +1,840 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: QPLib resource manager (header) + */ + +#ifndef __BNXT_QPLIB_RES_H__ +#define __BNXT_QPLIB_RES_H__ + +#include "hsi_struct_def.h" + +extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero; + +#define CHIP_NUM_57508 0x1750 +#define CHIP_NUM_57504 0x1751 +#define CHIP_NUM_57502 0x1752 +#define CHIP_NUM_58818 0xd818 +#define CHIP_NUM_57608 0x1760 + +#define BNXT_QPLIB_MAX_QPC_COUNT (64 * 1024) +#define BNXT_QPLIB_MAX_SRQC_COUNT (64 * 1024) +#define BNXT_QPLIB_MAX_CQ_COUNT (64 * 1024) +#define BNXT_QPLIB_MAX_CQ_COUNT_P5 (128 * 1024) + +#define BNXT_QPLIB_DBR_VALID (0x1UL << 26) +#define BNXT_QPLIB_DBR_EPOCH_SHIFT 24 +#define BNXT_QPLIB_DBR_TOGGLE_SHIFT 25 + +#define BNXT_QPLIB_DBR_PF_DB_OFFSET 0x10000 +#define BNXT_QPLIB_DBR_VF_DB_OFFSET 0x4000 + +#define BNXT_QPLIB_DBR_KEY_INVALID -1 + +/* chip gen type */ +#define BNXT_RE_DEFAULT 0xf + +enum bnxt_qplib_wqe_mode { + BNXT_QPLIB_WQE_MODE_STATIC = 0x00, + BNXT_QPLIB_WQE_MODE_VARIABLE = 0x01, + BNXT_QPLIB_WQE_MODE_INVALID = 0x02 +}; + +#define BNXT_RE_PUSH_MODE_NONE 0 +#define BNXT_RE_PUSH_MODE_WCB 1 +#define BNXT_RE_PUSH_MODE_PPP 2 +#define BNXT_RE_PUSH_ENABLED(mode) ((mode) == BNXT_RE_PUSH_MODE_WCB ||\ + (mode) == BNXT_RE_PUSH_MODE_PPP) +#define BNXT_RE_PPP_ENABLED(cctx) ((cctx)->modes.db_push_mode ==\ + BNXT_RE_PUSH_MODE_PPP) +#define PCI_EXP_DEVCAP2_ATOMIC_ROUTE 0x00000040 /* Atomic Op routing */ +#define PCI_EXP_DEVCAP2_ATOMIC_COMP32 0x00000080 /* 32b AtomicOp completion */ +#define PCI_EXP_DEVCAP2_ATOMIC_COMP64 0x00000100 /* 64b AtomicOp completion */ +#define PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK 0x0080 /* Block atomic egress */ +#define PCI_EXP_DEVCTL2_ATOMIC_REQ 0x0040 /* Set Atomic requests */ + +int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask); + +struct bnxt_qplib_drv_modes { + u8 wqe_mode; + u8 te_bypass; + u8 db_push; + /* To control advanced cc params display in configfs */ + u8 cc_pr_mode; + /* Other modes to follow here e.g. GSI QP mode */ + u8 dbr_pacing; + u8 dbr_pacing_ext; + u8 dbr_drop_recov; + u8 dbr_primary_pf; + u8 dbr_pacing_v0; +}; + +struct bnxt_qplib_chip_ctx { + u16 chip_num; + u8 chip_rev; + u8 chip_metal; + u64 hwrm_intf_ver; + struct bnxt_qplib_drv_modes modes; + u32 dbr_stat_db_fifo; + u32 dbr_aeq_arm_reg; + u32 dbr_throttling_reg; + u16 hw_stats_size; + u16 hwrm_cmd_max_timeout; +}; + +static inline bool _is_chip_num_p7(u16 chip_num) +{ + return (chip_num == CHIP_NUM_58818 || + chip_num == CHIP_NUM_57608); +} + +static inline bool _is_chip_p7(struct bnxt_qplib_chip_ctx *cctx) +{ + return _is_chip_num_p7(cctx->chip_num); +} + +/* SR2 is Gen P5 */ +static inline bool _is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx) +{ + return (cctx->chip_num == CHIP_NUM_57508 || + cctx->chip_num == CHIP_NUM_57504 || + cctx->chip_num == CHIP_NUM_57502); +} + +static inline bool _is_chip_gen_p5_p7(struct bnxt_qplib_chip_ctx *cctx) +{ + return (_is_chip_gen_p5(cctx) || _is_chip_p7(cctx)); +} + +static inline bool _is_wqe_mode_variable(struct bnxt_qplib_chip_ctx *cctx) +{ + return cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE; +} + +struct bnxt_qplib_db_pacing_data { + u32 do_pacing; + u32 pacing_th; + u32 dev_err_state; + u32 alarm_th; + u32 grc_reg_offset; + u32 fifo_max_depth; + u32 fifo_room_mask; + u8 fifo_room_shift; +}; + +static inline u8 bnxt_qplib_dbr_pacing_en(struct bnxt_qplib_chip_ctx *cctx) +{ + return cctx->modes.dbr_pacing; +} + +static inline u8 bnxt_qplib_dbr_pacing_ext_en(struct bnxt_qplib_chip_ctx *cctx) +{ + return cctx->modes.dbr_pacing_ext; +} + +static inline u8 bnxt_qplib_dbr_pacing_is_primary_pf(struct bnxt_qplib_chip_ctx *cctx) +{ + return cctx->modes.dbr_primary_pf; +} + +static inline void bnxt_qplib_dbr_pacing_set_primary_pf + (struct bnxt_qplib_chip_ctx *cctx, u8 val) +{ + cctx->modes.dbr_primary_pf = val; +} + +/* Defines for handling the HWRM version check */ +#define HWRM_VERSION_DEV_ATTR_MAX_DPI 0x1000A0000000D +#define HWRM_VERSION_ROCE_STATS_FN_ID 0x1000A00000045 + +#define PTR_CNT_PER_PG (PAGE_SIZE / sizeof(void *)) +#define PTR_MAX_IDX_PER_PG (PTR_CNT_PER_PG - 1) +#define PTR_PG(x) (((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG) +#define PTR_IDX(x) ((x) & PTR_MAX_IDX_PER_PG) + +#define HWQ_CMP(idx, hwq) ((idx) & ((hwq)->max_elements - 1)) +#define HWQ_FREE_SLOTS(hwq) (hwq->max_elements - \ + ((HWQ_CMP(hwq->prod, hwq)\ + - HWQ_CMP(hwq->cons, hwq))\ + & (hwq->max_elements - 1))) +enum bnxt_qplib_hwq_type { + HWQ_TYPE_CTX, + HWQ_TYPE_QUEUE, + HWQ_TYPE_L2_CMPL, + HWQ_TYPE_MR +}; + +#define MAX_PBL_LVL_0_PGS 1 +#define MAX_PBL_LVL_1_PGS 512 +#define MAX_PBL_LVL_1_PGS_SHIFT 9 +#define MAX_PDL_LVL_SHIFT 9 + +enum bnxt_qplib_pbl_lvl { + PBL_LVL_0, + PBL_LVL_1, + PBL_LVL_2, + PBL_LVL_MAX +}; + +#define ROCE_PG_SIZE_4K (4 * 1024) +#define ROCE_PG_SIZE_8K (8 * 1024) +#define ROCE_PG_SIZE_64K (64 * 1024) +#define ROCE_PG_SIZE_2M (2 * 1024 * 1024) +#define ROCE_PG_SIZE_8M (8 * 1024 * 1024) +#define ROCE_PG_SIZE_1G (1024 * 1024 * 1024) +enum bnxt_qplib_hwrm_pg_size { + BNXT_QPLIB_HWRM_PG_SIZE_4K = 0, + BNXT_QPLIB_HWRM_PG_SIZE_8K = 1, + BNXT_QPLIB_HWRM_PG_SIZE_64K = 2, + BNXT_QPLIB_HWRM_PG_SIZE_2M = 3, + BNXT_QPLIB_HWRM_PG_SIZE_8M = 4, + BNXT_QPLIB_HWRM_PG_SIZE_1G = 5, +}; + +struct bnxt_qplib_reg_desc { + u8 bar_id; + resource_size_t bar_base; + unsigned long offset; + void __iomem *bar_reg; + size_t len; +}; + +struct bnxt_qplib_pbl { + u32 pg_count; + u32 pg_size; + void **pg_arr; + dma_addr_t *pg_map_arr; +}; + +struct bnxt_qplib_sg_info { + struct scatterlist *sghead; + u32 nmap; + u32 npages; + u32 pgshft; + u32 pgsize; + bool nopte; +}; + +struct bnxt_qplib_hwq_attr { + struct bnxt_qplib_res *res; + struct bnxt_qplib_sg_info *sginfo; + enum bnxt_qplib_hwq_type type; + u32 depth; + u32 stride; + u32 aux_stride; + u32 aux_depth; +}; + +struct bnxt_qplib_hwq { + struct pci_dev *pdev; + spinlock_t lock; + struct bnxt_qplib_pbl pbl[PBL_LVL_MAX]; + enum bnxt_qplib_pbl_lvl level; /* 0, 1, or 2 */ + void **pbl_ptr; /* ptr for easy access + to the PBL entries */ + dma_addr_t *pbl_dma_ptr; /* ptr for easy access + to the dma_addr */ + u32 max_elements; + u32 depth; /* original requested depth */ + u16 element_size; /* Size of each entry */ + u16 qe_ppg; /* queue entry per page */ + + u32 prod; /* raw */ + u32 cons; /* raw */ + u8 cp_bit; + u8 is_user; + u64 *pad_pg; + u32 pad_stride; + u32 pad_pgofft; +}; + +struct bnxt_qplib_db_info { + void __iomem *db; + void __iomem *priv_db; + struct bnxt_qplib_hwq *hwq; + struct bnxt_qplib_res *res; + u32 xid; + u32 max_slot; + u32 flags; + u8 toggle; + spinlock_t lock; + u64 shadow_key; + u64 shadow_key_arm_ena; + u32 seed; /* For DB pacing */ +}; + +enum bnxt_qplib_db_info_flags_mask { + BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT = 0x0UL, + BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT = 0x1UL, + BNXT_QPLIB_FLAG_EPOCH_CONS_MASK = 0x1UL, + BNXT_QPLIB_FLAG_EPOCH_PROD_MASK = 0x2UL, +}; + +enum bnxt_qplib_db_epoch_flag_shift { + BNXT_QPLIB_DB_EPOCH_CONS_SHIFT = BNXT_QPLIB_DBR_EPOCH_SHIFT, + BNXT_QPLIB_DB_EPOCH_PROD_SHIFT = (BNXT_QPLIB_DBR_EPOCH_SHIFT - 1) +}; + +/* Tables */ +struct bnxt_qplib_pd_tbl { + unsigned long *tbl; + u32 max; +}; + +struct bnxt_qplib_sgid_tbl { + struct bnxt_qplib_gid_info *tbl; + u16 *hw_id; + u16 max; + u16 active; + void *ctx; + bool *vlan; +}; + +enum { + BNXT_QPLIB_DPI_TYPE_KERNEL = 0, + BNXT_QPLIB_DPI_TYPE_UC = 1, + BNXT_QPLIB_DPI_TYPE_WC = 2 +}; + +struct bnxt_qplib_dpi { + u32 dpi; + u32 bit; + void __iomem *dbr; + u64 umdbr; + u8 type; +}; + +#define BNXT_QPLIB_MAX_EXTENDED_PPP_PAGES 512 +struct bnxt_qplib_dpi_tbl { + void **app_tbl; + unsigned long *tbl; + u16 max; + u16 avail_ppp; + struct bnxt_qplib_reg_desc ucreg; /* Hold entire DB bar. */ + struct bnxt_qplib_reg_desc wcreg; + void __iomem *priv_db; +}; + +struct bnxt_qplib_stats { + dma_addr_t dma_map; + void *dma; + u32 size; + u32 fw_id; +}; + +struct bnxt_qplib_vf_res { + u32 max_qp; + u32 max_mrw; + u32 max_srq; + u32 max_cq; + u32 max_gid; +}; + +#define BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE 448 +#define BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE 64 +#define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE 64 +#define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE 128 + +#define MAX_TQM_ALLOC_REQ 48 +#define MAX_TQM_ALLOC_BLK_SIZE 8 +struct bnxt_qplib_tqm_ctx { + struct bnxt_qplib_hwq pde; + enum bnxt_qplib_pbl_lvl pde_level; /* Original level */ + struct bnxt_qplib_hwq qtbl[MAX_TQM_ALLOC_REQ]; + u8 qcount[MAX_TQM_ALLOC_REQ]; +}; + +struct bnxt_qplib_hctx { + struct bnxt_qplib_hwq hwq; + u32 max; +}; + +struct bnxt_qplib_refrec { + void *handle; + u32 xid; +}; + +struct bnxt_qplib_reftbl { + struct bnxt_qplib_refrec *rec; + u32 max; + spinlock_t lock; /* reftbl lock */ +}; + +struct bnxt_qplib_reftbls { + struct bnxt_qplib_reftbl qpref; + struct bnxt_qplib_reftbl cqref; + struct bnxt_qplib_reftbl srqref; +}; + +#define GET_TBL_INDEX(id, tbl) ((id) % (((tbl)->max) - 1)) +static inline u32 map_qp_id_to_tbl_indx(u32 qid, struct bnxt_qplib_reftbl *tbl) +{ + return (qid == 1) ? tbl->max : GET_TBL_INDEX(qid, tbl); +} + +/* + * This structure includes the number of various roce resource table sizes + * actually allocated by the driver. May be less than the maximums the firmware + * allows if the driver imposes lower limits than the firmware. + */ +struct bnxt_qplib_ctx { + struct bnxt_qplib_hctx qp_ctx; + struct bnxt_qplib_hctx mrw_ctx; + struct bnxt_qplib_hctx srq_ctx; + struct bnxt_qplib_hctx cq_ctx; + struct bnxt_qplib_hctx tim_ctx; + struct bnxt_qplib_tqm_ctx tqm_ctx; + + struct bnxt_qplib_stats stats; + struct bnxt_qplib_stats stats2; + struct bnxt_qplib_vf_res vf_res; +}; + +struct bnxt_qplib_res { + struct pci_dev *pdev; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_qplib_dev_attr *dattr; + struct bnxt_qplib_ctx *hctx; + struct ifnet *netdev; + struct bnxt_en_dev *en_dev; + + struct bnxt_qplib_rcfw *rcfw; + + struct bnxt_qplib_pd_tbl pd_tbl; + struct mutex pd_tbl_lock; + struct bnxt_qplib_sgid_tbl sgid_tbl; + struct bnxt_qplib_dpi_tbl dpi_tbl; + struct mutex dpi_tbl_lock; + struct bnxt_qplib_reftbls reftbl; + bool prio; + bool is_vf; + struct bnxt_qplib_db_pacing_data *pacing_data; +}; + +struct bnxt_qplib_query_stats_info { + u32 function_id; + u8 collection_id; + bool vf_valid; +}; + +struct bnxt_qplib_query_qp_info { + u32 function_id; + u32 num_qps; + u32 start_index; + bool vf_valid; +}; + +struct bnxt_qplib_query_fn_info { + bool vf_valid; + u32 host; + u32 filter; +}; + + +#define to_bnxt_qplib(ptr, type, member) \ + container_of(ptr, type, member) + +struct bnxt_qplib_pd; +struct bnxt_qplib_dev_attr; + +bool _is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx); +bool _is_chip_gen_p5_p7(struct bnxt_qplib_chip_ctx *cctx); +bool _is_chip_a0(struct bnxt_qplib_chip_ctx *cctx); +bool _is_chip_p7(struct bnxt_qplib_chip_ctx *cctx); +bool _is_alloc_mr_unified(struct bnxt_qplib_dev_attr *dattr); +void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res, + struct bnxt_qplib_hwq *hwq); +int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, + struct bnxt_qplib_hwq_attr *hwq_attr); +void bnxt_qplib_get_guid(const u8 *dev_addr, u8 *guid); +int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res, + struct bnxt_qplib_pd *pd); +int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, + struct bnxt_qplib_pd_tbl *pd_tbl, + struct bnxt_qplib_pd *pd); +int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi *dpi, + void *app, u8 type); +int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi *dpi); +int bnxt_qplib_stop_res(struct bnxt_qplib_res *res); +void bnxt_qplib_clear_tbls(struct bnxt_qplib_res *res); +int bnxt_qplib_init_tbls(struct bnxt_qplib_res *res); +void bnxt_qplib_free_tbls(struct bnxt_qplib_res *res); +int bnxt_qplib_alloc_tbls(struct bnxt_qplib_res *res, u8 pppp_factor); +void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res); +int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res); +int bnxt_qplib_alloc_stat_mem(struct pci_dev *pdev, + struct bnxt_qplib_chip_ctx *cctx, + struct bnxt_qplib_stats *stats); +void bnxt_qplib_free_stat_mem(struct bnxt_qplib_res *res, + struct bnxt_qplib_stats *stats); + +int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res); +void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res); +int bnxt_qplib_enable_atomic_ops_to_root(struct pci_dev *dev); +u8 _get_chip_gen_p5_type(struct bnxt_qplib_chip_ctx *cctx); + +static inline void *bnxt_qplib_get_qe(struct bnxt_qplib_hwq *hwq, + u32 indx, u64 *pg) +{ + u32 pg_num, pg_idx; + + pg_num = (indx / hwq->qe_ppg); + pg_idx = (indx % hwq->qe_ppg); + if (pg) + *pg = (u64)&hwq->pbl_ptr[pg_num]; + return (void *)((u8 *)hwq->pbl_ptr[pg_num] + hwq->element_size * pg_idx); +} + +static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_db_info *dbinfo, + struct bnxt_qplib_hwq *hwq, u32 cnt) +{ + /* move prod and update toggle/epoch if wrap around */ + hwq->prod += cnt; + if (hwq->prod >= hwq->depth) { + hwq->prod %= hwq->depth; + dbinfo->flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT; + } +} + +static inline void bnxt_qplib_hwq_incr_cons(u32 max_elements, u32 *cons, + u32 cnt, u32 *dbinfo_flags) +{ + /* move cons and update toggle/epoch if wrap around */ + *cons += cnt; + if (*cons >= max_elements) { + *cons %= max_elements; + *dbinfo_flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT; + } +} + +static inline u8 _get_pte_pg_size(struct bnxt_qplib_hwq *hwq) +{ + u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; + struct bnxt_qplib_pbl *pbl; + + pbl = &hwq->pbl[hwq->level]; + switch (pbl->pg_size) { + case ROCE_PG_SIZE_4K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; + break; + case ROCE_PG_SIZE_8K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K; + break; + case ROCE_PG_SIZE_64K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K; + break; + case ROCE_PG_SIZE_2M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M; + break; + case ROCE_PG_SIZE_8M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M; + break; + case ROCE_PG_SIZE_1G: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G; + break; + default: + break; + } + return pg_size; +} + +static inline u64 _get_base_addr(struct bnxt_qplib_hwq *hwq) +{ + return hwq->pbl[PBL_LVL_0].pg_map_arr[0]; +} + +static inline u8 _get_base_pg_size(struct bnxt_qplib_hwq *hwq) +{ + u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; + struct bnxt_qplib_pbl *pbl; + + pbl = &hwq->pbl[PBL_LVL_0]; + switch (pbl->pg_size) { + case ROCE_PG_SIZE_4K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; + break; + case ROCE_PG_SIZE_8K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K; + break; + case ROCE_PG_SIZE_64K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K; + break; + case ROCE_PG_SIZE_2M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M; + break; + case ROCE_PG_SIZE_8M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M; + break; + case ROCE_PG_SIZE_1G: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G; + break; + default: + break; + } + return pg_size; +} + +static inline enum bnxt_qplib_hwq_type _get_hwq_type(struct bnxt_qplib_res *res) +{ + return _is_chip_gen_p5_p7(res->cctx) ? HWQ_TYPE_QUEUE : HWQ_TYPE_L2_CMPL; +} + +static inline bool _is_ext_stats_supported(u16 dev_cap_flags) +{ + return dev_cap_flags & + CREQ_QUERY_FUNC_RESP_SB_EXT_STATS; +} + +static inline int bnxt_ext_stats_supported(struct bnxt_qplib_chip_ctx *ctx, + u16 flags, bool virtfn) +{ + return (_is_ext_stats_supported(flags) && + ((virtfn && _is_chip_p7(ctx)) || (!virtfn))); +} + +static inline bool _is_hw_retx_supported(u16 dev_cap_flags) +{ + return dev_cap_flags & + (CREQ_QUERY_FUNC_RESP_SB_HW_REQUESTER_RETX_ENABLED | + CREQ_QUERY_FUNC_RESP_SB_HW_RESPONDER_RETX_ENABLED); +} + +/* Disable HW_RETX */ +#define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a)) + +static inline bool _is_cqe_v2_supported(u16 dev_cap_flags) +{ + return dev_cap_flags & + CREQ_QUERY_FUNC_RESP_SB_CQE_V2; +} + +#define BNXT_DB_FIFO_ROOM_MASK 0x1fff8000 +#define BNXT_DB_FIFO_ROOM_SHIFT 15 +#define BNXT_MAX_FIFO_DEPTH 0x2c00 + +#define BNXT_DB_PACING_ALGO_THRESHOLD 250 +#define BNXT_DEFAULT_PACING_PROBABILITY 0xFFFF + +#define BNXT_DBR_PACING_WIN_BASE 0x2000 +#define BNXT_DBR_PACING_WIN_MAP_OFF 4 +#define BNXT_DBR_PACING_WIN_OFF(reg) (BNXT_DBR_PACING_WIN_BASE + \ + +static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info, + bool arm) +{ + u32 key = 0; + + key = info->hwq->cons | (CMPL_DOORBELL_IDX_VALID | + (CMPL_DOORBELL_KEY_CMPL & CMPL_DOORBELL_KEY_MASK)); + if (!arm) + key |= CMPL_DOORBELL_MASK; + /* memory barrier */ + wmb(); + writel(key, info->db); +} + +#define BNXT_QPLIB_INIT_DBHDR(xid, type, indx, toggle) \ + (((u64)(((xid) & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | \ + (type) | BNXT_QPLIB_DBR_VALID) << 32) | (indx) | \ + ((toggle) << (BNXT_QPLIB_DBR_TOGGLE_SHIFT))) + +static inline void bnxt_qplib_write_db(struct bnxt_qplib_db_info *info, + u64 key, void __iomem *db, + u64 *shadow_key) +{ + unsigned long flags; + + spin_lock_irqsave(&info->lock, flags); + *shadow_key = key; + writeq(key, db); + spin_unlock_irqrestore(&info->lock, flags); +} + +static inline void __replay_writeq(u64 key, void __iomem *db) +{ + /* No need to replay uninitialised shadow_keys */ + if (key != BNXT_QPLIB_DBR_KEY_INVALID) + writeq(key, db); +} + +static inline void bnxt_qplib_replay_db(struct bnxt_qplib_db_info *info, + bool is_arm_ena) + +{ + if (!spin_trylock_irq(&info->lock)) + return; + + if (is_arm_ena) + __replay_writeq(info->shadow_key_arm_ena, info->priv_db); + else + __replay_writeq(info->shadow_key, info->db); + + spin_unlock_irq(&info->lock); +} + +static inline void bnxt_qplib_ring_db(struct bnxt_qplib_db_info *info, + u32 type) +{ + u64 key = 0; + u32 indx; + u8 toggle = 0; + + if (type == DBC_DBC_TYPE_CQ_ARMALL || + type == DBC_DBC_TYPE_CQ_ARMSE) + toggle = info->toggle; + + indx = ((info->hwq->cons & DBC_DBC_INDEX_MASK) | + ((info->flags & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK) << + BNXT_QPLIB_DB_EPOCH_CONS_SHIFT)); + + key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, indx, toggle); + bnxt_qplib_write_db(info, key, info->db, &info->shadow_key); +} + +static inline void bnxt_qplib_ring_prod_db(struct bnxt_qplib_db_info *info, + u32 type) +{ + u64 key = 0; + u32 indx; + + indx = (((info->hwq->prod / info->max_slot) & DBC_DBC_INDEX_MASK) | + ((info->flags & BNXT_QPLIB_FLAG_EPOCH_PROD_MASK) << + BNXT_QPLIB_DB_EPOCH_PROD_SHIFT)); + key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, indx, 0); + bnxt_qplib_write_db(info, key, info->db, &info->shadow_key); +} + +static inline void bnxt_qplib_armen_db(struct bnxt_qplib_db_info *info, + u32 type) +{ + u64 key = 0; + u8 toggle = 0; + + if (type == DBC_DBC_TYPE_CQ_ARMENA) + toggle = info->toggle; + /* Index always at 0 */ + key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, 0, toggle); + bnxt_qplib_write_db(info, key, info->priv_db, + &info->shadow_key_arm_ena); +} + +static inline void bnxt_qplib_cq_coffack_db(struct bnxt_qplib_db_info *info) +{ + u64 key = 0; + + /* Index always at 0 */ + key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_CQ_CUTOFF_ACK, 0, 0); + bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key); +} + +static inline void bnxt_qplib_srq_arm_db(struct bnxt_qplib_db_info *info) +{ + u64 key = 0; + + /* Index always at 0 */ + key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_SRQ_ARM, 0, 0); + bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key); +} + +static inline void bnxt_qplib_ring_nq_db(struct bnxt_qplib_db_info *info, + struct bnxt_qplib_chip_ctx *cctx, + bool arm) +{ + u32 type; + + type = arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ; + if (_is_chip_gen_p5_p7(cctx)) + bnxt_qplib_ring_db(info, type); + else + bnxt_qplib_ring_db32(info, arm); +} + +struct bnxt_qplib_max_res { + u32 max_qp; + u32 max_mr; + u32 max_cq; + u32 max_srq; + u32 max_ah; + u32 max_pd; +}; + +/* + * Defines for maximum resources supported for chip revisions + * Maximum PDs supported are restricted to Max QPs + * GENP4 - Wh+ + * DEFAULT - Thor + */ +#define BNXT_QPLIB_GENP4_PF_MAX_QP (16 * 1024) +#define BNXT_QPLIB_GENP4_PF_MAX_MRW (16 * 1024) +#define BNXT_QPLIB_GENP4_PF_MAX_CQ (16 * 1024) +#define BNXT_QPLIB_GENP4_PF_MAX_SRQ (1 * 1024) +#define BNXT_QPLIB_GENP4_PF_MAX_AH (16 * 1024) +#define BNXT_QPLIB_GENP4_PF_MAX_PD BNXT_QPLIB_GENP4_PF_MAX_QP + +#define BNXT_QPLIB_DEFAULT_PF_MAX_QP (64 * 1024) +#define BNXT_QPLIB_DEFAULT_PF_MAX_MRW (256 * 1024) +#define BNXT_QPLIB_DEFAULT_PF_MAX_CQ (64 * 1024) +#define BNXT_QPLIB_DEFAULT_PF_MAX_SRQ (4 * 1024) +#define BNXT_QPLIB_DEFAULT_PF_MAX_AH (64 * 1024) +#define BNXT_QPLIB_DEFAULT_PF_MAX_PD BNXT_QPLIB_DEFAULT_PF_MAX_QP + +#define BNXT_QPLIB_DEFAULT_VF_MAX_QP (6 * 1024) +#define BNXT_QPLIB_DEFAULT_VF_MAX_MRW (6 * 1024) +#define BNXT_QPLIB_DEFAULT_VF_MAX_CQ (6 * 1024) +#define BNXT_QPLIB_DEFAULT_VF_MAX_SRQ (4 * 1024) +#define BNXT_QPLIB_DEFAULT_VF_MAX_AH (6 * 1024) +#define BNXT_QPLIB_DEFAULT_VF_MAX_PD BNXT_QPLIB_DEFAULT_VF_MAX_QP + +static inline void bnxt_qplib_max_res_supported(struct bnxt_qplib_chip_ctx *cctx, + struct bnxt_qplib_res *qpl_res, + struct bnxt_qplib_max_res *max_res, + bool vf_res_limit) +{ + switch (cctx->chip_num) { + case CHIP_NUM_57608: + case CHIP_NUM_58818: + case CHIP_NUM_57504: + case CHIP_NUM_57502: + case CHIP_NUM_57508: + if (!qpl_res->is_vf) { + max_res->max_qp = BNXT_QPLIB_DEFAULT_PF_MAX_QP; + max_res->max_mr = BNXT_QPLIB_DEFAULT_PF_MAX_MRW; + max_res->max_cq = BNXT_QPLIB_DEFAULT_PF_MAX_CQ; + max_res->max_srq = BNXT_QPLIB_DEFAULT_PF_MAX_SRQ; + max_res->max_ah = BNXT_QPLIB_DEFAULT_PF_MAX_AH; + max_res->max_pd = BNXT_QPLIB_DEFAULT_PF_MAX_PD; + } else { + max_res->max_qp = BNXT_QPLIB_DEFAULT_VF_MAX_QP; + max_res->max_mr = BNXT_QPLIB_DEFAULT_VF_MAX_MRW; + max_res->max_cq = BNXT_QPLIB_DEFAULT_VF_MAX_CQ; + max_res->max_srq = BNXT_QPLIB_DEFAULT_VF_MAX_SRQ; + max_res->max_ah = BNXT_QPLIB_DEFAULT_VF_MAX_AH; + max_res->max_pd = BNXT_QPLIB_DEFAULT_VF_MAX_PD; + } + break; + default: + /* Wh+/Stratus max resources */ + max_res->max_qp = BNXT_QPLIB_GENP4_PF_MAX_QP; + max_res->max_mr = BNXT_QPLIB_GENP4_PF_MAX_MRW; + max_res->max_cq = BNXT_QPLIB_GENP4_PF_MAX_CQ; + max_res->max_srq = BNXT_QPLIB_GENP4_PF_MAX_SRQ; + max_res->max_ah = BNXT_QPLIB_GENP4_PF_MAX_AH; + max_res->max_pd = BNXT_QPLIB_GENP4_PF_MAX_PD; + break; + } +} +#endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.c b/sys/dev/bnxt/bnxt_re/qplib_sp.c new file mode 100644 index 00000000000..8faa3cd9390 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_sp.c @@ -0,0 +1,1234 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Slow Path Operators + */ + +#include +#include +#include +#include +#include +#include + +#include "hsi_struct_def.h" +#include "qplib_tlv.h" +#include "qplib_res.h" +#include "qplib_rcfw.h" +#include "qplib_sp.h" + +const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 }}; + +/* Device */ +static u8 bnxt_qplib_is_atomic_cap(struct bnxt_qplib_rcfw *rcfw) +{ + u16 pcie_ctl2 = 0; + + if (!_is_chip_gen_p5_p7(rcfw->res->cctx)) + return false; + pcie_capability_read_word(rcfw->pdev, PCI_EXP_DEVCTL2, &pcie_ctl2); + return (pcie_ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ); +} + +static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw, char *fw_ver) +{ + struct creq_query_version_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_query_version req = {}; + int rc = 0; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_VERSION, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) { + dev_err(&rcfw->pdev->dev, "QPLIB: Failed to query version\n"); + return; + } + fw_ver[0] = resp.fw_maj; + fw_ver[1] = resp.fw_minor; + fw_ver[2] = resp.fw_bld; + fw_ver[3] = resp.fw_rsvd; +} + +int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) +{ + struct creq_query_func_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct creq_query_func_resp_sb *sb; + struct bnxt_qplib_rcfw_sbuf sbuf; + struct bnxt_qplib_dev_attr *attr; + struct bnxt_qplib_chip_ctx *cctx; + struct cmdq_query_func req = {}; + u8 *tqm_alloc; + int i, rc = 0; + u32 temp; + u8 chip_gen = BNXT_RE_DEFAULT; + + cctx = rcfw->res->cctx; + attr = rcfw->res->dattr; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_FUNC, + sizeof(req)); + + sbuf.size = sizeof(*sb); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) + return -ENOMEM; + + sb = sbuf.sb; + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto bail; + /* Extract the context from the side buffer */ + chip_gen = _get_chip_gen_p5_type(cctx); + attr->max_qp = le32_to_cpu(sb->max_qp); + attr->max_qp = min_t(u32, attr->max_qp, BNXT_RE_MAX_QP_SUPPORTED(chip_gen)); + /* max_qp value reported by FW does not include the QP1 */ + attr->max_qp += 1; + attr->max_qp_rd_atom = + sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? + BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom; + attr->max_qp_init_rd_atom = + sb->max_qp_init_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? + BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_init_rd_atom; + /* Report 1 less than the max_qp_wqes reported by FW as driver adds + * one extra entry while creating the qp + */ + attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr) - 1; + /* Adjust for max_qp_wqes for variable wqe */ + if (cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) { + attr->max_qp_wqes = (BNXT_MAX_SQ_SIZE) / + (BNXT_MAX_VAR_WQE_SIZE / BNXT_SGE_SIZE) - 1; + } + if (!_is_chip_gen_p5_p7(cctx)) { + /* + * 128 WQEs needs to be reserved for the HW (8916). Prevent + * reporting the max number for gen-p4 only. + */ + attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS; + } + attr->max_qp_sges = sb->max_sge; + if (_is_chip_gen_p5_p7(cctx) && + cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) + attr->max_qp_sges = sb->max_sge_var_wqe; + attr->max_cq = le32_to_cpu(sb->max_cq); + attr->max_cq = min_t(u32, attr->max_cq, BNXT_RE_MAX_CQ_SUPPORTED(chip_gen)); + + attr->max_cq_wqes = le32_to_cpu(sb->max_cqe); + attr->max_cq_wqes = min_t(u32, BNXT_QPLIB_MAX_CQ_WQES, attr->max_cq_wqes); + + attr->max_cq_sges = attr->max_qp_sges; + attr->max_mr = le32_to_cpu(sb->max_mr); + attr->max_mr = min_t(u32, attr->max_mr, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen)); + attr->max_mw = le32_to_cpu(sb->max_mw); + attr->max_mw = min_t(u32, attr->max_mw, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen)); + + attr->max_mr_size = le64_to_cpu(sb->max_mr_size); + attr->max_pd = BNXT_QPLIB_MAX_PD; + attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp); + attr->max_ah = le32_to_cpu(sb->max_ah); + attr->max_ah = min_t(u32, attr->max_ah, BNXT_RE_MAX_AH_SUPPORTED(chip_gen)); + + attr->max_fmr = le32_to_cpu(sb->max_fmr); + attr->max_map_per_fmr = sb->max_map_per_fmr; + + attr->max_srq = le16_to_cpu(sb->max_srq); + attr->max_srq = min_t(u32, attr->max_srq, BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen)); + attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1; + attr->max_srq_sges = sb->max_srq_sge; + attr->max_pkey = 1; + + attr->max_inline_data = !cctx->modes.wqe_mode ? + le32_to_cpu(sb->max_inline_data) : + le16_to_cpu(sb->max_inline_data_var_wqe); + if (!_is_chip_p7(cctx)) { + attr->l2_db_size = (sb->l2_db_space_size + 1) * + (0x01 << RCFW_DBR_BASE_PAGE_SHIFT); + } + attr->max_sgid = le32_to_cpu(sb->max_gid); + + /* TODO: remove this hack for statically allocated gid_map */ + bnxt_re_set_max_gid(&attr->max_sgid); + + attr->dev_cap_flags = le16_to_cpu(sb->dev_cap_flags); + attr->page_size_cap = BIT_ULL(28) | BIT_ULL(21) | BIT_ULL(12); + + bnxt_qplib_query_version(rcfw, attr->fw_ver); + + for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) { + temp = le32_to_cpu(sb->tqm_alloc_reqs[i]); + tqm_alloc = (u8 *)&temp; + attr->tqm_alloc_reqs[i * 4] = *tqm_alloc; + attr->tqm_alloc_reqs[i * 4 + 1] = *(++tqm_alloc); + attr->tqm_alloc_reqs[i * 4 + 2] = *(++tqm_alloc); + attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc); + } + + if (rcfw->res->cctx->hwrm_intf_ver >= HWRM_VERSION_DEV_ATTR_MAX_DPI) + attr->max_dpi = le32_to_cpu(sb->max_dpi); + + attr->is_atomic = bnxt_qplib_is_atomic_cap(rcfw); +bail: + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + return rc; +} + +int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res) +{ + struct creq_set_func_resources_resp resp = {}; + struct cmdq_set_func_resources req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_rcfw *rcfw; + struct bnxt_qplib_ctx *hctx; + int rc = 0; + + rcfw = res->rcfw; + hctx = res->hctx; + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES, + sizeof(req)); + + req.number_of_qp = cpu_to_le32(hctx->qp_ctx.max); + req.number_of_mrw = cpu_to_le32(hctx->mrw_ctx.max); + req.number_of_srq = cpu_to_le32(hctx->srq_ctx.max); + req.number_of_cq = cpu_to_le32(hctx->cq_ctx.max); + + req.max_qp_per_vf = cpu_to_le32(hctx->vf_res.max_qp); + req.max_mrw_per_vf = cpu_to_le32(hctx->vf_res.max_mrw); + req.max_srq_per_vf = cpu_to_le32(hctx->vf_res.max_srq); + req.max_cq_per_vf = cpu_to_le32(hctx->vf_res.max_cq); + req.max_gid_per_vf = cpu_to_le32(hctx->vf_res.max_gid); + + /* Keep the old stats context id of PF */ + req.stat_ctx_id = cpu_to_le32(hctx->stats.fw_id); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + dev_err(&res->pdev->dev, + "QPLIB: Failed to set function resources\n"); + + return rc; +} + +int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac) +{ + struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, + struct bnxt_qplib_res, + sgid_tbl); + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_modify_gid_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_modify_gid req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_GID, + sizeof(req)); + + req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]); + req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]); + req.gid[2] = cpu_to_be32(((u32 *)gid->data)[1]); + req.gid[3] = cpu_to_be32(((u32 *)gid->data)[0]); + if (res->prio) { + req.vlan |= cpu_to_le16(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 | + CMDQ_ADD_GID_VLAN_VLAN_EN); + } + + /* MAC in network format */ + req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]); + req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]); + req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]); + req.gid_index = cpu_to_le16(gid_idx); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) { + dev_err(&res->pdev->dev, + "QPLIB: update SGID table failed\n"); + return rc; + } + return 0; +} + +/* SGID */ +int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, + struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, + struct bnxt_qplib_gid *gid) +{ + if (index > sgid_tbl->max) { + dev_err(&res->pdev->dev, + "QPLIB: Index %d exceeded SGID table max (%d)\n", + index, sgid_tbl->max); + return -EINVAL; + } + memcpy(gid, &sgid_tbl->tbl[index].gid, sizeof(*gid)); + return 0; +} + +int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, + u16 vlan_id, bool update) +{ + struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, + struct bnxt_qplib_res, + sgid_tbl); + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + int index; + + if (sgid_tbl == NULL) { + dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated\n"); + return -EINVAL; + } + /* Do we need a sgid_lock here? */ + if (!sgid_tbl->active) { + dev_err(&res->pdev->dev, + "QPLIB: SGID table has no active entries\n"); + return -ENOMEM; + } + for (index = 0; index < sgid_tbl->max; index++) { + if (!memcmp(&sgid_tbl->tbl[index].gid, gid, sizeof(*gid)) && + vlan_id == sgid_tbl->tbl[index].vlan_id) + break; + } + if (index == sgid_tbl->max) { + dev_warn(&res->pdev->dev, "GID not found in the SGID table\n"); + return 0; + } + + if (update) { + struct creq_delete_gid_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_delete_gid req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DELETE_GID, + sizeof(req)); + if (sgid_tbl->hw_id[index] == 0xFFFF) { + dev_err(&res->pdev->dev, + "QPLIB: GID entry contains an invalid HW id"); + return -EINVAL; + } + req.gid_index = cpu_to_le16(sgid_tbl->hw_id[index]); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + } + memcpy(&sgid_tbl->tbl[index].gid, &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero)); + sgid_tbl->tbl[index].vlan_id = 0xFFFF; + sgid_tbl->vlan[index] = false; + sgid_tbl->active--; + dev_dbg(&res->pdev->dev, + "QPLIB: SGID deleted hw_id[0x%x] = 0x%x active = 0x%x\n", + index, sgid_tbl->hw_id[index], sgid_tbl->active); + sgid_tbl->hw_id[index] = (u16)-1; + + return 0; +} + +int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + const union ib_gid *gid, const u8 *smac, u16 vlan_id, + bool update, u32 *index) +{ + struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, + struct bnxt_qplib_res, + sgid_tbl); + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + int i, free_idx; + + if (sgid_tbl == NULL) { + dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated\n"); + return -EINVAL; + } + /* Do we need a sgid_lock here? */ + if (sgid_tbl->active == sgid_tbl->max) { + dev_err(&res->pdev->dev, "QPLIB: SGID table is full\n"); + return -ENOMEM; + } + free_idx = sgid_tbl->max; + for (i = 0; i < sgid_tbl->max; i++) { + if (!memcmp(&sgid_tbl->tbl[i], gid, sizeof(*gid)) && + sgid_tbl->tbl[i].vlan_id == vlan_id) { + dev_dbg(&res->pdev->dev, + "QPLIB: SGID entry already exist in entry %d!\n", + i); + *index = i; + return -EALREADY; + } else if (!memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero)) && + free_idx == sgid_tbl->max) { + free_idx = i; + } + } + if (free_idx == sgid_tbl->max) { + dev_err(&res->pdev->dev, + "QPLIB: SGID table is FULL but count is not MAX??\n"); + return -ENOMEM; + } + if (update) { + struct creq_add_gid_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_add_gid req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_ADD_GID, + sizeof(req)); + + req.gid[0] = cpu_to_be32(((u32 *)gid->raw)[3]); + req.gid[1] = cpu_to_be32(((u32 *)gid->raw)[2]); + req.gid[2] = cpu_to_be32(((u32 *)gid->raw)[1]); + req.gid[3] = cpu_to_be32(((u32 *)gid->raw)[0]); + /* driver should ensure that all RoCE traffic is always VLAN tagged + * if RoCE traffic is running on non-zero VLAN ID or + * RoCE traffic is running on non-zero Priority. + */ + if ((vlan_id != 0xFFFF) || res->prio) { + if (vlan_id != 0xFFFF) + req.vlan = cpu_to_le16(vlan_id & + CMDQ_ADD_GID_VLAN_VLAN_ID_MASK); + req.vlan |= + cpu_to_le16(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 | + CMDQ_ADD_GID_VLAN_VLAN_EN); + } + + /* MAC in network format */ + req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]); + req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]); + req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp.xid); + } + + if (vlan_id != 0xFFFF) + sgid_tbl->vlan[free_idx] = true; + + memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid)); + sgid_tbl->tbl[free_idx].vlan_id = vlan_id; + sgid_tbl->active++; + dev_dbg(&res->pdev->dev, + "QPLIB: SGID added hw_id[0x%x] = 0x%x active = 0x%x\n", + free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active); + + *index = free_idx; + /* unlock */ + return 0; +} + +/* AH */ +int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, + bool block) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_create_ah_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_create_ah req = {}; + u32 temp32[4]; + u16 temp16[3]; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_AH, + sizeof(req)); + + memcpy(temp32, ah->dgid.data, sizeof(struct bnxt_qplib_gid)); + req.dgid[0] = cpu_to_le32(temp32[0]); + req.dgid[1] = cpu_to_le32(temp32[1]); + req.dgid[2] = cpu_to_le32(temp32[2]); + req.dgid[3] = cpu_to_le32(temp32[3]); + + req.type = ah->nw_type; + req.hop_limit = ah->hop_limit; + req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[ah->sgid_index]); + req.dest_vlan_id_flow_label = cpu_to_le32((ah->flow_label & + CMDQ_CREATE_AH_FLOW_LABEL_MASK) | + CMDQ_CREATE_AH_DEST_VLAN_ID_MASK); + req.pd_id = cpu_to_le32(ah->pd->id); + req.traffic_class = ah->traffic_class; + + /* MAC in network format */ + memcpy(temp16, ah->dmac, ETH_ALEN); + req.dest_mac[0] = cpu_to_le16(temp16[0]); + req.dest_mac[1] = cpu_to_le16(temp16[1]); + req.dest_mac[2] = cpu_to_le16(temp16[2]); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), block); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + + ah->id = le32_to_cpu(resp.xid); + /* for Cu/Wh AHID 0 is not valid */ + if (!_is_chip_gen_p5_p7(res->cctx) && !ah->id) + rc = -EINVAL; + + return rc; +} + +int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, + bool block) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_destroy_ah_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_destroy_ah req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_AH, + sizeof(req)); + + req.ah_cid = cpu_to_le32(ah->id); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), block); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + return rc; +} + +/* MRW */ +int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) +{ + struct creq_deallocate_key_resp resp = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_deallocate_key req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + int rc; + + if (mrw->lkey == 0xFFFFFFFF) { + dev_info(&res->pdev->dev, + "QPLIB: SP: Free a reserved lkey MRW\n"); + return 0; + } + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEALLOCATE_KEY, + sizeof(req)); + + req.mrw_flags = mrw->type; + + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) + req.key = cpu_to_le32(mrw->rkey); + else + req.key = cpu_to_le32(mrw->lkey); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + + if (mrw->hwq.max_elements) + bnxt_qplib_free_hwq(res, &mrw->hwq); + + return 0; +} + +int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_allocate_mrw_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_allocate_mrw req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_ALLOCATE_MRW, + sizeof(req)); + + req.pd_id = cpu_to_le32(mrw->pd->id); + req.mrw_flags = mrw->type; + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR && + mrw->flags & BNXT_QPLIB_FR_PMR) || + mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A || + mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B) + req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY; + req.mrw_handle = cpu_to_le64(mrw); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) + mrw->rkey = le32_to_cpu(resp.xid); + else + mrw->lkey = le32_to_cpu(resp.xid); + + return 0; +} + +int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, + bool block) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_deregister_mr_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_deregister_mr req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEREGISTER_MR, + sizeof(req)); + + req.lkey = cpu_to_le32(mrw->lkey); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), block); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + + if (mrw->hwq.max_elements) { + mrw->va = 0; + mrw->total_size = 0; + bnxt_qplib_free_hwq(res, &mrw->hwq); + } + + return 0; +} + +int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, + struct bnxt_qplib_mrinfo *mrinfo, + bool block) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_register_mr_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_register_mr req = {}; + struct bnxt_qplib_mrw *mr; + u32 buf_pg_size; + u32 pg_size; + u16 level; + u16 flags; + int rc; + + mr = mrinfo->mrw; + buf_pg_size = 0x01ULL << mrinfo->sg.pgshft; + if (mrinfo->sg.npages) { + /* Free the hwq if it already exist, must be a rereg */ + if (mr->hwq.max_elements) + bnxt_qplib_free_hwq(res, &mr->hwq); + /* Use system PAGE_SIZE */ + hwq_attr.res = res; + hwq_attr.depth = mrinfo->sg.npages; + hwq_attr.stride = PAGE_SIZE; + hwq_attr.type = HWQ_TYPE_MR; + hwq_attr.sginfo = &mrinfo->sg; + rc = bnxt_qplib_alloc_init_hwq(&mr->hwq, &hwq_attr); + if (rc) { + dev_err(&res->pdev->dev, + "SP: Reg MR memory allocation failed\n"); + return -ENOMEM; + } + } + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_REGISTER_MR, + sizeof(req)); + /* Configure the request */ + if (mrinfo->is_dma) { + /* No PBL provided, just use system PAGE_SIZE */ + level = 0; + req.pbl = 0; + pg_size = PAGE_SIZE; + } else { + level = mr->hwq.level; + req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]); + } + + pg_size = buf_pg_size ? buf_pg_size : PAGE_SIZE; + req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) | + ((ilog2(pg_size) << + CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) & + CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK); + req.log2_pbl_pg_size = cpu_to_le16(((ilog2(PAGE_SIZE) << + CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT) & + CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK)); + req.access = (mr->flags & 0xFFFF); + req.va = cpu_to_le64(mr->va); + req.key = cpu_to_le32(mr->lkey); + if (_is_alloc_mr_unified(res->dattr)) { + flags = 0; + req.key = cpu_to_le32(mr->pd->id); + flags |= CMDQ_REGISTER_MR_FLAGS_ALLOC_MR; + req.flags = cpu_to_le16(flags); + } + req.mr_size = cpu_to_le64(mr->total_size); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), block); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail; + + if (_is_alloc_mr_unified(res->dattr)) { + mr->lkey = le32_to_cpu(resp.xid); + mr->rkey = mr->lkey; + } + + return 0; +fail: + if (mr->hwq.max_elements) + bnxt_qplib_free_hwq(res, &mr->hwq); + return rc; +} + +int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl, + int max_pg_ptrs) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_sg_info sginfo = {}; + int pg_ptrs, rc; + + /* Re-calculate the max to fit the HWQ allocation model */ + pg_ptrs = roundup_pow_of_two(max_pg_ptrs); + + sginfo.pgsize = PAGE_SIZE; + sginfo.nopte = true; + + hwq_attr.res = res; + hwq_attr.depth = pg_ptrs; + hwq_attr.stride = PAGE_SIZE; + hwq_attr.sginfo = &sginfo; + hwq_attr.type = HWQ_TYPE_CTX; + rc = bnxt_qplib_alloc_init_hwq(&frpl->hwq, &hwq_attr); + if (!rc) + frpl->max_pg_ptrs = pg_ptrs; + + return rc; +} + +void bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl) +{ + bnxt_qplib_free_hwq(res, &frpl->hwq); +} + +int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_map_tc_to_cos_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_map_tc_to_cos req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MAP_TC_TO_COS, + sizeof(req)); + req.cos0 = cpu_to_le16(cids[0]); + req.cos1 = cpu_to_le16(cids[1]); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + return rc; +} + +static void bnxt_qplib_fill_cc_gen1(struct cmdq_modify_roce_cc_gen1_tlv *ext_req, + struct bnxt_qplib_cc_param_ext *cc_ext) +{ + ext_req->modify_mask = cpu_to_le64(cc_ext->ext_mask); + cc_ext->ext_mask = 0; + ext_req->inactivity_th_hi = cpu_to_le16(cc_ext->inact_th_hi); + ext_req->min_time_between_cnps = cpu_to_le16(cc_ext->min_delta_cnp); + ext_req->init_cp = cpu_to_le16(cc_ext->init_cp); + ext_req->tr_update_mode = cc_ext->tr_update_mode; + ext_req->tr_update_cycles = cc_ext->tr_update_cyls; + ext_req->fr_num_rtts = cc_ext->fr_rtt; + ext_req->ai_rate_increase = cc_ext->ai_rate_incr; + ext_req->reduction_relax_rtts_th = cpu_to_le16(cc_ext->rr_rtt_th); + ext_req->additional_relax_cr_th = cpu_to_le16(cc_ext->ar_cr_th); + ext_req->cr_min_th = cpu_to_le16(cc_ext->cr_min_th); + ext_req->bw_avg_weight = cc_ext->bw_avg_weight; + ext_req->actual_cr_factor = cc_ext->cr_factor; + ext_req->max_cp_cr_th = cpu_to_le16(cc_ext->cr_th_max_cp); + ext_req->cp_bias_en = cc_ext->cp_bias_en; + ext_req->cp_bias = cc_ext->cp_bias; + ext_req->cnp_ecn = cc_ext->cnp_ecn; + ext_req->rtt_jitter_en = cc_ext->rtt_jitter_en; + ext_req->link_bytes_per_usec = cpu_to_le16(cc_ext->bytes_per_usec); + ext_req->reset_cc_cr_th = cpu_to_le16(cc_ext->cc_cr_reset_th); + ext_req->cr_width = cc_ext->cr_width; + ext_req->quota_period_min = cc_ext->min_quota; + ext_req->quota_period_max = cc_ext->max_quota; + ext_req->quota_period_abs_max = cc_ext->abs_max_quota; + ext_req->tr_lower_bound = cpu_to_le16(cc_ext->tr_lb); + ext_req->cr_prob_factor = cc_ext->cr_prob_fac; + ext_req->tr_prob_factor = cc_ext->tr_prob_fac; + ext_req->fairness_cr_th = cpu_to_le16(cc_ext->fair_cr_th); + ext_req->red_div = cc_ext->red_div; + ext_req->cnp_ratio_th = cc_ext->cnp_ratio_th; + ext_req->exp_ai_rtts = cpu_to_le16(cc_ext->ai_ext_rtt); + ext_req->exp_ai_cr_cp_ratio = cc_ext->exp_crcp_ratio; + ext_req->use_rate_table = cc_ext->low_rate_en; + ext_req->cp_exp_update_th = cpu_to_le16(cc_ext->cpcr_update_th); + ext_req->high_exp_ai_rtts_th1 = cpu_to_le16(cc_ext->ai_rtt_th1); + ext_req->high_exp_ai_rtts_th2 = cpu_to_le16(cc_ext->ai_rtt_th2); + ext_req->actual_cr_cong_free_rtts_th = cpu_to_le16(cc_ext->cf_rtt_th); + ext_req->severe_cong_cr_th1 = cpu_to_le16(cc_ext->sc_cr_th1); + ext_req->severe_cong_cr_th2 = cpu_to_le16(cc_ext->sc_cr_th2); + ext_req->link64B_per_rtt = cpu_to_le32(cc_ext->l64B_per_rtt); + ext_req->cc_ack_bytes = cc_ext->cc_ack_bytes; + ext_req->reduce_init_cong_free_rtts_th = cpu_to_le16(cc_ext->reduce_cf_rtt_th); +} + +int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res, + struct bnxt_qplib_cc_param *cc_param) +{ + struct bnxt_qplib_tlv_modify_cc_req tlv_req = {}; + struct creq_modify_roce_cc_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_modify_roce_cc *req; + int req_size; + void *cmd; + int rc; + + /* Prepare the older base command */ + req = &tlv_req.base_req; + cmd = req; + req_size = sizeof(*req); + bnxt_qplib_rcfw_cmd_prep(req, CMDQ_BASE_OPCODE_MODIFY_ROCE_CC, + sizeof(*req)); + req->modify_mask = cpu_to_le32(cc_param->mask); + req->enable_cc = cc_param->enable; + req->g = cc_param->g; + req->num_phases_per_state = cc_param->nph_per_state; + req->time_per_phase = cc_param->time_pph; + req->pkts_per_phase = cc_param->pkts_pph; + req->init_cr = cpu_to_le16(cc_param->init_cr); + req->init_tr = cpu_to_le16(cc_param->init_tr); + req->tos_dscp_tos_ecn = (cc_param->tos_dscp << + CMDQ_MODIFY_ROCE_CC_TOS_DSCP_SFT) | + (cc_param->tos_ecn & + CMDQ_MODIFY_ROCE_CC_TOS_ECN_MASK); + req->alt_vlan_pcp = cc_param->alt_vlan_pcp; + req->alt_tos_dscp = cpu_to_le16(cc_param->alt_tos_dscp); + req->rtt = cpu_to_le16(cc_param->rtt); + req->tcp_cp = cpu_to_le16(cc_param->tcp_cp); + req->cc_mode = cc_param->cc_mode; + req->inactivity_th = cpu_to_le16(cc_param->inact_th); + + /* For chip gen P5 onwards fill extended cmd and header */ + if (_is_chip_gen_p5_p7(res->cctx)) { + struct roce_tlv *hdr; + u32 payload; + u32 chunks; + + cmd = &tlv_req; + req_size = sizeof(tlv_req); + /* Prepare primary tlv header */ + hdr = &tlv_req.tlv_hdr; + chunks = CHUNKS(sizeof(struct bnxt_qplib_tlv_modify_cc_req)); + payload = sizeof(struct cmdq_modify_roce_cc); + ROCE_1ST_TLV_PREP(hdr, chunks, payload, true); + /* Prepare secondary tlv header */ + hdr = (struct roce_tlv *)&tlv_req.ext_req; + payload = sizeof(struct cmdq_modify_roce_cc_gen1_tlv) - + sizeof(struct roce_tlv); + ROCE_EXT_TLV_PREP(hdr, TLV_TYPE_MODIFY_ROCE_CC_GEN1, payload, + false, true); + bnxt_qplib_fill_cc_gen1(&tlv_req.ext_req, &cc_param->cc_ext); + } + + bnxt_qplib_fill_cmdqmsg(&msg, cmd, &resp, NULL, req_size, + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(res->rcfw, &msg); + return rc; +} + +static void bnxt_qplib_read_cc_gen1(struct bnxt_qplib_cc_param_ext *cc_ext, + struct creq_query_roce_cc_gen1_resp_sb_tlv *sb) +{ + cc_ext->inact_th_hi = le16_to_cpu(sb->inactivity_th_hi); + cc_ext->min_delta_cnp = le16_to_cpu(sb->min_time_between_cnps); + cc_ext->init_cp = le16_to_cpu(sb->init_cp); + cc_ext->tr_update_mode = sb->tr_update_mode; + cc_ext->tr_update_cyls = sb->tr_update_cycles; + cc_ext->fr_rtt = sb->fr_num_rtts; + cc_ext->ai_rate_incr = sb->ai_rate_increase; + cc_ext->rr_rtt_th = le16_to_cpu(sb->reduction_relax_rtts_th); + cc_ext->ar_cr_th = le16_to_cpu(sb->additional_relax_cr_th); + cc_ext->cr_min_th = le16_to_cpu(sb->cr_min_th); + cc_ext->bw_avg_weight = sb->bw_avg_weight; + cc_ext->cr_factor = sb->actual_cr_factor; + cc_ext->cr_th_max_cp = le16_to_cpu(sb->max_cp_cr_th); + cc_ext->cp_bias_en = sb->cp_bias_en; + cc_ext->cp_bias = sb->cp_bias; + cc_ext->cnp_ecn = sb->cnp_ecn; + cc_ext->rtt_jitter_en = sb->rtt_jitter_en; + cc_ext->bytes_per_usec = le16_to_cpu(sb->link_bytes_per_usec); + cc_ext->cc_cr_reset_th = le16_to_cpu(sb->reset_cc_cr_th); + cc_ext->cr_width = sb->cr_width; + cc_ext->min_quota = sb->quota_period_min; + cc_ext->max_quota = sb->quota_period_max; + cc_ext->abs_max_quota = sb->quota_period_abs_max; + cc_ext->tr_lb = le16_to_cpu(sb->tr_lower_bound); + cc_ext->cr_prob_fac = sb->cr_prob_factor; + cc_ext->tr_prob_fac = sb->tr_prob_factor; + cc_ext->fair_cr_th = le16_to_cpu(sb->fairness_cr_th); + cc_ext->red_div = sb->red_div; + cc_ext->cnp_ratio_th = sb->cnp_ratio_th; + cc_ext->ai_ext_rtt = le16_to_cpu(sb->exp_ai_rtts); + cc_ext->exp_crcp_ratio = sb->exp_ai_cr_cp_ratio; + cc_ext->low_rate_en = sb->use_rate_table; + cc_ext->cpcr_update_th = le16_to_cpu(sb->cp_exp_update_th); + cc_ext->ai_rtt_th1 = le16_to_cpu(sb->high_exp_ai_rtts_th1); + cc_ext->ai_rtt_th2 = le16_to_cpu(sb->high_exp_ai_rtts_th2); + cc_ext->cf_rtt_th = le16_to_cpu(sb->actual_cr_cong_free_rtts_th); + cc_ext->sc_cr_th1 = le16_to_cpu(sb->severe_cong_cr_th1); + cc_ext->sc_cr_th2 = le16_to_cpu(sb->severe_cong_cr_th2); + cc_ext->l64B_per_rtt = le32_to_cpu(sb->link64B_per_rtt); + cc_ext->cc_ack_bytes = sb->cc_ack_bytes; + cc_ext->reduce_cf_rtt_th = le16_to_cpu(sb->reduce_init_cong_free_rtts_th); +} + +int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res, + struct bnxt_qplib_cc_param *cc_param) +{ + struct creq_query_roce_cc_gen1_resp_sb_tlv *gen1_sb; + struct bnxt_qplib_tlv_query_rcc_sb *ext_sb; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_query_roce_cc_resp resp = {}; + struct creq_query_roce_cc_resp_sb *sb; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_query_roce_cc req = {}; + struct bnxt_qplib_rcfw_sbuf sbuf; + size_t resp_size; + int rc; + + /* Query the parameters from chip */ + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_ROCE_CC, + sizeof(req)); + if (_is_chip_gen_p5_p7(res->cctx)) + resp_size = sizeof(*ext_sb); + else + resp_size = sizeof(*sb); + sbuf.size = ALIGN(resp_size, BNXT_QPLIB_CMDQE_UNITS); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) + return -ENOMEM; + + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(res->rcfw, &msg); + if (rc) { + dev_dbg(&res->pdev->dev, "%s:Query CC param failed:0x%x\n", + __func__, rc); + goto out; + } + + ext_sb = sbuf.sb; + gen1_sb = &ext_sb->gen1_sb; + sb = _is_chip_gen_p5_p7(res->cctx) ? &ext_sb->base_sb : + (struct creq_query_roce_cc_resp_sb *)ext_sb; + + cc_param->enable = sb->enable_cc & CREQ_QUERY_ROCE_CC_RESP_SB_ENABLE_CC; + cc_param->tos_ecn = (sb->tos_dscp_tos_ecn & + CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_MASK) >> + CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_SFT; + cc_param->tos_dscp = (sb->tos_dscp_tos_ecn & + CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_MASK) >> + CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_SFT; + cc_param->alt_tos_dscp = sb->alt_tos_dscp; + cc_param->alt_vlan_pcp = sb->alt_vlan_pcp; + + cc_param->g = sb->g; + cc_param->nph_per_state = sb->num_phases_per_state; + cc_param->init_cr = le16_to_cpu(sb->init_cr); + cc_param->init_tr = le16_to_cpu(sb->init_tr); + cc_param->cc_mode = sb->cc_mode; + cc_param->inact_th = le16_to_cpu(sb->inactivity_th); + cc_param->rtt = le16_to_cpu(sb->rtt); + cc_param->tcp_cp = le16_to_cpu(sb->tcp_cp); + cc_param->time_pph = sb->time_per_phase; + cc_param->pkts_pph = sb->pkts_per_phase; + if (_is_chip_gen_p5_p7(res->cctx)) + bnxt_qplib_read_cc_gen1(&cc_param->cc_ext, gen1_sb); +out: + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + return rc; +} + + +int bnxt_qplib_get_roce_error_stats(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_roce_stats *stats, + struct bnxt_qplib_query_stats_info *sinfo) +{ + struct creq_query_roce_stats_resp resp = {}; + struct creq_query_roce_stats_resp_sb *sb; + struct cmdq_query_roce_stats req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_rcfw_sbuf sbuf; + u16 cmd_flags = 0; + u32 fn_id = 0; + int rc = 0; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_ROCE_STATS, + sizeof(req)); + + sbuf.size = sizeof(*sb); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) + return -ENOMEM; + sb = sbuf.sb; + + if (rcfw->res->cctx->hwrm_intf_ver >= HWRM_VERSION_ROCE_STATS_FN_ID) { + if (sinfo->function_id != 0xFFFFFFFF) { + cmd_flags = CMDQ_QUERY_ROCE_STATS_FLAGS_FUNCTION_ID; + if (sinfo->vf_valid) { + fn_id = CMDQ_QUERY_ROCE_STATS_VF_VALID; + fn_id |= (sinfo->function_id << + CMDQ_QUERY_ROCE_STATS_VF_NUM_SFT) & + CMDQ_QUERY_ROCE_STATS_VF_NUM_MASK; + } else { + fn_id = sinfo->function_id & + CMDQ_QUERY_ROCE_STATS_PF_NUM_MASK; + } + } + + req.flags = cpu_to_le16(cmd_flags); + req.function_id = cpu_to_le32(fn_id); + + if (sinfo->collection_id != 0xFF) { + cmd_flags |= CMDQ_QUERY_ROCE_STATS_FLAGS_COLLECTION_ID; + req.collection_id = sinfo->collection_id; + } + } else { + /* For older HWRM version, the command length has to be + * adjusted. 8 bytes are more in the newer command. + * So subtract these 8 bytes for older HWRM version. + * command units are adjusted inside + * bnxt_qplib_rcfw_send_message. + */ + req.cmd_size -= 8; + } + + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto bail; + /* Extract the context from the side buffer */ + stats->to_retransmits = le64_to_cpu(sb->to_retransmits); + stats->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd); + stats->max_retry_exceeded = le64_to_cpu(sb->max_retry_exceeded); + stats->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd); + stats->missing_resp = le64_to_cpu(sb->missing_resp); + stats->unrecoverable_err = le64_to_cpu(sb->unrecoverable_err); + stats->bad_resp_err = le64_to_cpu(sb->bad_resp_err); + stats->local_qp_op_err = le64_to_cpu(sb->local_qp_op_err); + stats->local_protection_err = le64_to_cpu(sb->local_protection_err); + stats->mem_mgmt_op_err = le64_to_cpu(sb->mem_mgmt_op_err); + stats->remote_invalid_req_err = le64_to_cpu(sb->remote_invalid_req_err); + stats->remote_access_err = le64_to_cpu(sb->remote_access_err); + stats->remote_op_err = le64_to_cpu(sb->remote_op_err); + stats->dup_req = le64_to_cpu(sb->dup_req); + stats->res_exceed_max = le64_to_cpu(sb->res_exceed_max); + stats->res_length_mismatch = le64_to_cpu(sb->res_length_mismatch); + stats->res_exceeds_wqe = le64_to_cpu(sb->res_exceeds_wqe); + stats->res_opcode_err = le64_to_cpu(sb->res_opcode_err); + stats->res_rx_invalid_rkey = le64_to_cpu(sb->res_rx_invalid_rkey); + stats->res_rx_domain_err = le64_to_cpu(sb->res_rx_domain_err); + stats->res_rx_no_perm = le64_to_cpu(sb->res_rx_no_perm); + stats->res_rx_range_err = le64_to_cpu(sb->res_rx_range_err); + stats->res_tx_invalid_rkey = le64_to_cpu(sb->res_tx_invalid_rkey); + stats->res_tx_domain_err = le64_to_cpu(sb->res_tx_domain_err); + stats->res_tx_no_perm = le64_to_cpu(sb->res_tx_no_perm); + stats->res_tx_range_err = le64_to_cpu(sb->res_tx_range_err); + stats->res_irrq_oflow = le64_to_cpu(sb->res_irrq_oflow); + stats->res_unsup_opcode = le64_to_cpu(sb->res_unsup_opcode); + stats->res_unaligned_atomic = le64_to_cpu(sb->res_unaligned_atomic); + stats->res_rem_inv_err = le64_to_cpu(sb->res_rem_inv_err); + stats->res_mem_error = le64_to_cpu(sb->res_mem_error); + stats->res_srq_err = le64_to_cpu(sb->res_srq_err); + stats->res_cmp_err = le64_to_cpu(sb->res_cmp_err); + stats->res_invalid_dup_rkey = le64_to_cpu(sb->res_invalid_dup_rkey); + stats->res_wqe_format_err = le64_to_cpu(sb->res_wqe_format_err); + stats->res_cq_load_err = le64_to_cpu(sb->res_cq_load_err); + stats->res_srq_load_err = le64_to_cpu(sb->res_srq_load_err); + stats->res_tx_pci_err = le64_to_cpu(sb->res_tx_pci_err); + stats->res_rx_pci_err = le64_to_cpu(sb->res_rx_pci_err); + + if (!rcfw->init_oos_stats) { + rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count); + rcfw->init_oos_stats = true; + } else { + stats->res_oos_drop_count += (le64_to_cpu(sb->res_oos_drop_count) - + rcfw->oos_prev) & + BNXT_QPLIB_OOS_COUNT_MASK; + rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count); + } + + stats->active_qp_count_p0 = le64_to_cpu(sb->active_qp_count_p0); + stats->active_qp_count_p1 = le64_to_cpu(sb->active_qp_count_p1); + stats->active_qp_count_p2 = le64_to_cpu(sb->active_qp_count_p2); + stats->active_qp_count_p3 = le64_to_cpu(sb->active_qp_count_p3); +bail: + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + return rc; +} + +int bnxt_qplib_set_link_aggr_mode(struct bnxt_qplib_res *res, + u8 aggr_mode, u8 member_port_map, + u8 active_port_map, bool aggr_en, + u32 stats_fw_id) +{ + struct creq_set_link_aggr_mode_resources_resp resp = {}; + struct cmdq_set_link_aggr_mode_cc req = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct bnxt_qplib_cmdqmsg msg = {}; + int rc = 0; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE, + sizeof(req)); + + req.aggr_enable = aggr_en; + req.active_port_map = active_port_map; + req.member_port_map = member_port_map; + req.link_aggr_mode = aggr_mode; + + /* need to specify only second port stats ctx id for now */ + req.stat_ctx_id[1] = cpu_to_le16((u16)(stats_fw_id)); + + req.modify_mask = + cpu_to_le32(CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_EN | + CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_ACTIVE_PORT_MAP | + CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_MEMBER_PORT_MAP | + CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_MODE | + CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_STAT_CTX_ID); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + dev_err(&res->pdev->dev, + "QPLIB: Failed to set link aggr mode, %#x\n", rc); + + return rc; +} + +int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid, + struct bnxt_qplib_ext_stat *estat, + struct bnxt_qplib_query_stats_info *sinfo) +{ + struct creq_query_roce_stats_ext_resp resp = {}; + struct creq_query_roce_stats_ext_resp_sb *sb; + struct cmdq_query_roce_stats_ext req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_rcfw_sbuf sbuf; + int rc; + + sbuf.size = sizeof(*sb); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) { + dev_err(&rcfw->pdev->dev, + "QPLIB: SP: QUERY_ROCE_STATS_EXT alloc sb failed\n"); + return -ENOMEM; + } + sb = sbuf.sb; + + bnxt_qplib_rcfw_cmd_prep(&req, + CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS, + sizeof(req)); + req.resp_size = sbuf.size; + req.resp_addr = cpu_to_le64(sbuf.dma_addr); + req.flags = cpu_to_le16(CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID); + if (_is_chip_p7(rcfw->res->cctx) && rcfw->res->is_vf) { + if (sinfo->vf_valid) + req.function_id = + cpu_to_le32(CMDQ_QUERY_ROCE_STATS_EXT_VF_VALID | + (fid << CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_SFT)); + else + req.flags = cpu_to_le16(0); + } else { + req.function_id = cpu_to_le32(fid); + } + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto bail; + + /* dump when dyndbg is enabled */ + print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, sb, sizeof(*sb)); + estat->tx_atomic_req = le64_to_cpu(sb->tx_atomic_req_pkts); + estat->tx_read_req = le64_to_cpu(sb->tx_read_req_pkts); + estat->tx_read_res = le64_to_cpu(sb->tx_read_res_pkts); + estat->tx_write_req = le64_to_cpu(sb->tx_write_req_pkts); + estat->tx_send_req = le64_to_cpu(sb->tx_send_req_pkts); + estat->tx_roce_pkts = le64_to_cpu(sb->tx_roce_pkts); + estat->tx_roce_bytes = le64_to_cpu(sb->tx_roce_bytes); + estat->rx_atomic_req = le64_to_cpu(sb->rx_atomic_req_pkts); + estat->rx_read_req = le64_to_cpu(sb->rx_read_req_pkts); + estat->rx_read_res = le64_to_cpu(sb->rx_read_res_pkts); + estat->rx_write_req = le64_to_cpu(sb->rx_write_req_pkts); + estat->rx_send_req = le64_to_cpu(sb->rx_send_req_pkts); + estat->rx_roce_pkts = le64_to_cpu(sb->rx_roce_pkts); + estat->rx_roce_bytes = le64_to_cpu(sb->rx_roce_bytes); + estat->rx_roce_good_pkts = le64_to_cpu(sb->rx_roce_good_pkts); + estat->rx_roce_good_bytes = le64_to_cpu(sb->rx_roce_good_bytes); + estat->rx_out_of_buffer = le64_to_cpu(sb->rx_out_of_buffer_pkts); + estat->rx_out_of_sequence = le64_to_cpu(sb->rx_out_of_sequence_pkts); + estat->tx_cnp = le64_to_cpu(sb->tx_cnp_pkts); + estat->rx_cnp = le64_to_cpu(sb->rx_cnp_pkts); + estat->rx_ecn_marked = le64_to_cpu(sb->rx_ecn_marked_pkts); + estat->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd); + estat->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd); + estat->missing_resp = le64_to_cpu(sb->missing_resp); + estat->to_retransmits = le64_to_cpu(sb->to_retransmit); + estat->dup_req = le64_to_cpu(sb->dup_req); + estat->rx_dcn_payload_cut = le64_to_cpu(sb->rx_dcn_payload_cut); + estat->te_bypassed = le64_to_cpu(sb->te_bypassed); +bail: + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + return rc; +} diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.h b/sys/dev/bnxt/bnxt_re/qplib_sp.h new file mode 100644 index 00000000000..e306db3b9d8 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_sp.h @@ -0,0 +1,432 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Slow Path Operators (header) + */ + +#ifndef __BNXT_QPLIB_SP_H__ +#define __BNXT_QPLIB_SP_H__ + +#include + +#define BNXT_QPLIB_RESERVED_QP_WRS 128 + +/* Resource maximums reported by the firmware */ +struct bnxt_qplib_dev_attr { +#define FW_VER_ARR_LEN 4 + u8 fw_ver[FW_VER_ARR_LEN]; + u16 max_sgid; + u16 max_mrw; + u32 max_qp; +#define BNXT_QPLIB_MAX_OUT_RD_ATOM 126 + u32 max_qp_rd_atom; + u32 max_qp_init_rd_atom; + u32 max_qp_wqes; + u32 max_qp_sges; + u32 max_cq; + /* HW supports only 8K entries in PBL. + * So max CQEs that can be supported per CQ is 1M. + */ +#define BNXT_QPLIB_MAX_CQ_WQES 0xfffff + u32 max_cq_wqes; + u32 max_cq_sges; + u32 max_mr; + u64 max_mr_size; +#define BNXT_QPLIB_MAX_PD (64 * 1024) + u32 max_pd; + u32 max_mw; + u32 max_raw_ethy_qp; + u32 max_ah; + u32 max_fmr; + u32 max_map_per_fmr; + u32 max_srq; + u32 max_srq_wqes; + u32 max_srq_sges; + u32 max_pkey; + u32 max_inline_data; + u32 l2_db_size; + u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; + u8 is_atomic; + u16 dev_cap_flags; + u64 page_size_cap; + u32 max_dpi; +}; + +struct bnxt_qplib_pd { + u32 id; +}; + +struct bnxt_qplib_gid { + u8 data[16]; +}; + +struct bnxt_qplib_gid_info { + struct bnxt_qplib_gid gid; + u16 vlan_id; +}; + +struct bnxt_qplib_ah { + struct bnxt_qplib_gid dgid; + struct bnxt_qplib_pd *pd; + u32 id; + u8 sgid_index; + u8 host_sgid_index; /* For Query AH if the hw table and SW table are differnt */ + u8 traffic_class; + u32 flow_label; + u8 hop_limit; + u8 sl; + u8 dmac[6]; + u16 vlan_id; + u8 nw_type; + u8 enable_cc; +}; + +struct bnxt_qplib_mrw { + struct bnxt_qplib_pd *pd; + int type; + u32 flags; +#define BNXT_QPLIB_FR_PMR 0x80000000 + u32 lkey; + u32 rkey; +#define BNXT_QPLIB_RSVD_LKEY 0xFFFFFFFF + u64 va; + u64 total_size; + u32 npages; + u64 mr_handle; + struct bnxt_qplib_hwq hwq; +}; + +struct bnxt_qplib_mrinfo { + struct bnxt_qplib_mrw *mrw; + struct bnxt_qplib_sg_info sg; + u64 *ptes; + bool is_dma; +}; + +struct bnxt_qplib_frpl { + int max_pg_ptrs; + struct bnxt_qplib_hwq hwq; +}; + +struct bnxt_qplib_cc_param_ext { + u64 ext_mask; + u16 inact_th_hi; + u16 min_delta_cnp; + u16 init_cp; + u8 tr_update_mode; + u8 tr_update_cyls; + u8 fr_rtt; + u8 ai_rate_incr; + u16 rr_rtt_th; + u16 ar_cr_th; + u16 cr_min_th; + u8 bw_avg_weight; + u8 cr_factor; + u16 cr_th_max_cp; + u8 cp_bias_en; + u8 cp_bias; + u8 cnp_ecn; + u8 rtt_jitter_en; + u16 bytes_per_usec; + u16 cc_cr_reset_th; + u8 cr_width; + u8 min_quota; + u8 max_quota; + u8 abs_max_quota; + u16 tr_lb; + u8 cr_prob_fac; + u8 tr_prob_fac; + u16 fair_cr_th; + u8 red_div; + u8 cnp_ratio_th; + u16 ai_ext_rtt; + u8 exp_crcp_ratio; + u8 low_rate_en; + u16 cpcr_update_th; + u16 ai_rtt_th1; + u16 ai_rtt_th2; + u16 cf_rtt_th; + u16 sc_cr_th1; /* severe congestion cr threshold 1 */ + u16 sc_cr_th2; /* severe congestion cr threshold 2 */ + u32 l64B_per_rtt; + u8 cc_ack_bytes; + u16 reduce_cf_rtt_th; +}; + +struct bnxt_qplib_cc_param { + u8 alt_vlan_pcp; + u16 alt_tos_dscp; +#define BNXT_QPLIB_USER_DSCP_VALID 0x80 + u8 cnp_dscp_user; + u8 roce_dscp_user; + u8 cc_mode; + u8 enable; + u16 inact_th; + u16 init_cr; + u16 init_tr; + u16 rtt; + u8 g; + u8 nph_per_state; + u8 time_pph; + u8 pkts_pph; + u8 tos_ecn; + u8 tos_dscp; + u8 qp1_tos_dscp; + u16 tcp_cp; + struct bnxt_qplib_cc_param_ext cc_ext; + u8 disable_prio_vlan_tx; + /* Mask used while programming the configfs values */ + u32 mask; + /* Mask used while displaying the configfs values */ + u32 cur_mask; + u8 roce_pri; +#define BNXT_QPLIB_CC_PARAM_MASK_VLAN_TX_DISABLE 0x40000 +#define BNXT_QPLIB_CC_PARAM_MASK_ROCE_PRI 0x80000 + /* prev value to clear dscp table */ + u8 prev_roce_pri; + u8 prev_alt_vlan_pcp; + u8 prev_tos_dscp; + u16 prev_alt_tos_dscp; + /* To track if admin has enabled ECN explicitly */ + u8 admin_enable; +}; + +struct bnxt_qplib_roce_stats { + u64 to_retransmits; + u64 seq_err_naks_rcvd; + /* seq_err_naks_rcvd is 64 b */ + u64 max_retry_exceeded; + /* max_retry_exceeded is 64 b */ + u64 rnr_naks_rcvd; + /* rnr_naks_rcvd is 64 b */ + u64 missing_resp; + u64 unrecoverable_err; + /* unrecoverable_err is 64 b */ + u64 bad_resp_err; + /* bad_resp_err is 64 b */ + u64 local_qp_op_err; + /* local_qp_op_err is 64 b */ + u64 local_protection_err; + /* local_protection_err is 64 b */ + u64 mem_mgmt_op_err; + /* mem_mgmt_op_err is 64 b */ + u64 remote_invalid_req_err; + /* remote_invalid_req_err is 64 b */ + u64 remote_access_err; + /* remote_access_err is 64 b */ + u64 remote_op_err; + /* remote_op_err is 64 b */ + u64 dup_req; + /* dup_req is 64 b */ + u64 res_exceed_max; + /* res_exceed_max is 64 b */ + u64 res_length_mismatch; + /* res_length_mismatch is 64 b */ + u64 res_exceeds_wqe; + /* res_exceeds_wqe is 64 b */ + u64 res_opcode_err; + /* res_opcode_err is 64 b */ + u64 res_rx_invalid_rkey; + /* res_rx_invalid_rkey is 64 b */ + u64 res_rx_domain_err; + /* res_rx_domain_err is 64 b */ + u64 res_rx_no_perm; + /* res_rx_no_perm is 64 b */ + u64 res_rx_range_err; + /* res_rx_range_err is 64 b */ + u64 res_tx_invalid_rkey; + /* res_tx_invalid_rkey is 64 b */ + u64 res_tx_domain_err; + /* res_tx_domain_err is 64 b */ + u64 res_tx_no_perm; + /* res_tx_no_perm is 64 b */ + u64 res_tx_range_err; + /* res_tx_range_err is 64 b */ + u64 res_irrq_oflow; + /* res_irrq_oflow is 64 b */ + u64 res_unsup_opcode; + /* res_unsup_opcode is 64 b */ + u64 res_unaligned_atomic; + /* res_unaligned_atomic is 64 b */ + u64 res_rem_inv_err; + /* res_rem_inv_err is 64 b */ + u64 res_mem_error; + /* res_mem_error is 64 b */ + u64 res_srq_err; + /* res_srq_err is 64 b */ + u64 res_cmp_err; + /* res_cmp_err is 64 b */ + u64 res_invalid_dup_rkey; + /* res_invalid_dup_rkey is 64 b */ + u64 res_wqe_format_err; + /* res_wqe_format_err is 64 b */ + u64 res_cq_load_err; + /* res_cq_load_err is 64 b */ + u64 res_srq_load_err; + /* res_srq_load_err is 64 b */ + u64 res_tx_pci_err; + /* res_tx_pci_err is 64 b */ + u64 res_rx_pci_err; + /* res_rx_pci_err is 64 b */ + u64 res_oos_drop_count; + /* res_oos_drop_count */ + u64 active_qp_count_p0; + /* port 0 active qps */ + u64 active_qp_count_p1; + /* port 1 active qps */ + u64 active_qp_count_p2; + /* port 2 active qps */ + u64 active_qp_count_p3; + /* port 3 active qps */ +}; + +struct bnxt_qplib_ext_stat { + u64 tx_atomic_req; + u64 tx_read_req; + u64 tx_read_res; + u64 tx_write_req; + u64 tx_send_req; + u64 tx_roce_pkts; + u64 tx_roce_bytes; + u64 rx_atomic_req; + u64 rx_read_req; + u64 rx_read_res; + u64 rx_write_req; + u64 rx_send_req; + u64 rx_roce_pkts; + u64 rx_roce_bytes; + u64 rx_roce_good_pkts; + u64 rx_roce_good_bytes; + u64 rx_out_of_buffer; + u64 rx_out_of_sequence; + u64 tx_cnp; + u64 rx_cnp; + u64 rx_ecn_marked; + u64 seq_err_naks_rcvd; + u64 rnr_naks_rcvd; + u64 missing_resp; + u64 to_retransmits; + u64 dup_req; + u64 rx_dcn_payload_cut; + u64 te_bypassed; +}; + +#define BNXT_QPLIB_ACCESS_LOCAL_WRITE (1 << 0) +#define BNXT_QPLIB_ACCESS_REMOTE_READ (1 << 1) +#define BNXT_QPLIB_ACCESS_REMOTE_WRITE (1 << 2) +#define BNXT_QPLIB_ACCESS_REMOTE_ATOMIC (1 << 3) +#define BNXT_QPLIB_ACCESS_MW_BIND (1 << 4) +#define BNXT_QPLIB_ACCESS_ZERO_BASED (1 << 5) +#define BNXT_QPLIB_ACCESS_ON_DEMAND (1 << 6) + +int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, + struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, + struct bnxt_qplib_gid *gid); +int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, u16 vlan_id, bool update); +int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + const union ib_gid *gid, const u8 *mac, u16 vlan_id, + bool update, u32 *index); +int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac); +int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw); +int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res); +int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, + bool block); +int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, + bool block); +int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw); +int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, + bool block); +int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, + struct bnxt_qplib_mrinfo *mrinfo, bool block); +int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr); +int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res, + struct bnxt_qplib_mrw *mr, int max); +int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl, int max); +void bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl); +int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids); +int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res, + struct bnxt_qplib_cc_param *cc_param); +int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res, + struct bnxt_qplib_cc_param *cc_param); +int bnxt_qplib_set_link_aggr_mode(struct bnxt_qplib_res *res, + u8 aggr_mode, u8 member_port_map, + u8 active_port_map, bool aggr_en, + u32 stats_fw_id); +int bnxt_qplib_get_roce_error_stats(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_roce_stats *stats, + struct bnxt_qplib_query_stats_info *sinfo); +int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid, + struct bnxt_qplib_ext_stat *estat, + struct bnxt_qplib_query_stats_info *sinfo); +static inline void bnxt_re_set_max_gid(u16 *max_sgid); +bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, enum ib_qp_state next_state, + enum ib_qp_type type, enum ib_qp_attr_mask mask); + +#define BNXT_MAX_SQ_SIZE 0xFFFF +#define BNXT_MAX_VAR_WQE_SIZE 512 +#define BNXT_SGE_SIZE 16 + +/* PF defines */ +#define BNXT_RE_MAX_QP_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0 + +#define BNXT_RE_MAX_MRW_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (256 * 1024) : 0 + +#define BNXT_RE_MAX_CQ_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0 + +#define BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (4 * 1024) : 0 + +#define BNXT_RE_MAX_AH_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0 + +/* VF defines */ +#define BNXT_RE_VF_MAX_QP_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0 + +#define BNXT_RE_VF_MAX_MRW_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0 + +#define BNXT_RE_VF_MAX_CQ_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0 + +#define BNXT_RE_VF_MAX_SRQ_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (4 * 1024) : 0 + +static inline void bnxt_re_set_max_gid(u16 *max_sgid) +{ + *max_sgid = max_t(u32, 256, *max_sgid); + *max_sgid = min_t(u32, 256, *max_sgid); +} + +#endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_tlv.h b/sys/dev/bnxt/bnxt_re/qplib_tlv.h new file mode 100644 index 00000000000..eeaea41a37e --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_tlv.h @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2017 - 2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef __QPLIB_TLV_H__ +#define __QPLIB_TLV_H__ + +struct roce_tlv { + struct tlv tlv; + u8 total_size; + u8 unused[7]; +}; + +#define CHUNK_SIZE 16 +#define CHUNKS(x) (((x) + CHUNK_SIZE - 1) / CHUNK_SIZE) + +#define ROCE_1ST_TLV_PREP(rtlv, tot_chunks, content_bytes, more) \ + do { \ + (rtlv)->tlv.cmd_discr = CMD_DISCR_TLV_ENCAP; \ + (rtlv)->tlv.tlv_type = TLV_TYPE_ROCE_SP_COMMAND; \ + (rtlv)->tlv.length = (content_bytes); \ + (rtlv)->tlv.flags = TLV_FLAGS_REQUIRED; \ + (rtlv)->tlv.flags |= (more) ? TLV_FLAGS_MORE : 0; \ + (rtlv)->total_size = (tot_chunks); \ + } while (0) + +#define ROCE_EXT_TLV_PREP(rtlv, ext_type, content_bytes, more, reqd) \ + do { \ + (rtlv)->tlv.cmd_discr = CMD_DISCR_TLV_ENCAP; \ + (rtlv)->tlv.tlv_type = (ext_type); \ + (rtlv)->tlv.length = (content_bytes); \ + (rtlv)->tlv.flags |= (more) ? TLV_FLAGS_MORE : 0; \ + (rtlv)->tlv.flags |= (reqd) ? TLV_FLAGS_REQUIRED : 0; \ + } while (0) + +/* + * TLV size in units of 16 byte chunks + */ +#define TLV_SIZE ((sizeof(struct roce_tlv) + 15) / 16) +/* + * TLV length in bytes + */ +#define TLV_BYTES (TLV_SIZE * 16) + +#define HAS_TLV_HEADER(msg) (((struct tlv *)(msg))->cmd_discr == CMD_DISCR_TLV_ENCAP) +#define GET_TLV_DATA(tlv) ((void *)&((uint8_t *)(tlv))[TLV_BYTES]) + +static inline u8 __get_cmdq_base_opcode(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct cmdq_base *)GET_TLV_DATA(req))->opcode; + else + return req->opcode; +} + +static inline void __set_cmdq_base_opcode(struct cmdq_base *req, + u32 size, u8 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->opcode = val; + else + req->opcode = val; +} + +static inline __le16 __get_cmdq_base_cookie(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct cmdq_base *)GET_TLV_DATA(req))->cookie; + else + return req->cookie; +} + +static inline void __set_cmdq_base_cookie(struct cmdq_base *req, + u32 size, __le16 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->cookie = val; + else + req->cookie = val; +} + +static inline __le64 __get_cmdq_base_resp_addr(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct cmdq_base *)GET_TLV_DATA(req))->resp_addr; + else + return req->resp_addr; +} + +static inline void __set_cmdq_base_resp_addr(struct cmdq_base *req, + u32 size, __le64 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->resp_addr = val; + else + req->resp_addr = val; +} + +static inline u8 __get_cmdq_base_resp_size(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct cmdq_base *)GET_TLV_DATA(req))->resp_size; + else + return req->resp_size; +} + +static inline void __set_cmdq_base_resp_size(struct cmdq_base *req, + u32 size, u8 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->resp_size = val; + else + req->resp_size = val; +} + +static inline u8 __get_cmdq_base_cmd_size(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct roce_tlv *)(req))->total_size; + else + return req->cmd_size; +} + +static inline void __set_cmdq_base_cmd_size(struct cmdq_base *req, + u32 size, u8 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->cmd_size = val; + else + req->cmd_size = val; +} + +static inline __le16 __get_cmdq_base_flags(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct cmdq_base *)GET_TLV_DATA(req))->flags; + else + return req->flags; +} + +static inline void __set_cmdq_base_flags(struct cmdq_base *req, + u32 size, __le16 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->flags = val; + else + req->flags = val; +} + +struct bnxt_qplib_tlv_modify_cc_req { + struct roce_tlv tlv_hdr; + struct cmdq_modify_roce_cc base_req; + __le64 tlvpad; + struct cmdq_modify_roce_cc_gen1_tlv ext_req; +}; + +struct bnxt_qplib_tlv_query_rcc_sb { + struct roce_tlv tlv_hdr; + struct creq_query_roce_cc_resp_sb base_sb; + struct creq_query_roce_cc_gen1_resp_sb_tlv gen1_sb; +}; +#endif diff --git a/sys/dev/bnxt/bnxt_re/stats.c b/sys/dev/bnxt/bnxt_re/stats.c new file mode 100644 index 00000000000..7b0e6097aae --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/stats.c @@ -0,0 +1,773 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: statistics related functions + */ + +#include "bnxt_re.h" +#include "bnxt.h" + +int bnxt_re_get_flow_stats_from_service_pf(struct bnxt_re_dev *rdev, + struct bnxt_re_flow_counters *stats, + struct bnxt_qplib_query_stats_info *sinfo) +{ + struct hwrm_cfa_flow_stats_output resp = {}; + struct hwrm_cfa_flow_stats_input req = {}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg = {}; + u16 target_id; + int rc = 0; + + if (sinfo->function_id == 0xFFFFFFFF) + target_id = -1; + else + target_id = sinfo->function_id + 1; + + /* Issue HWRM cmd to read flow counters for CNP tx and rx */ + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_CFA_FLOW_STATS, -1, target_id); + req.num_flows = cpu_to_le16(6); + req.flow_handle_0 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT); + req.flow_handle_1 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + req.flow_handle_2 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT); + req.flow_handle_3 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + req.flow_handle_4 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT); + req.flow_handle_5 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + bnxt_re_fill_fw_msg(&fw_msg, &req, sizeof(req), &resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to get CFA Flow stats : rc = 0x%x\n", rc); + return rc; + } + + stats->cnp_stats.cnp_tx_pkts = le64_to_cpu(resp.packet_0); + stats->cnp_stats.cnp_tx_bytes = le64_to_cpu(resp.byte_0); + stats->cnp_stats.cnp_rx_pkts = le64_to_cpu(resp.packet_1); + stats->cnp_stats.cnp_rx_bytes = le64_to_cpu(resp.byte_1); + + stats->ro_stats.tx_pkts = le64_to_cpu(resp.packet_2) + + le64_to_cpu(resp.packet_4); + stats->ro_stats.tx_bytes = le64_to_cpu(resp.byte_2) + + le64_to_cpu(resp.byte_4); + stats->ro_stats.rx_pkts = le64_to_cpu(resp.packet_3) + + le64_to_cpu(resp.packet_5); + stats->ro_stats.rx_bytes = le64_to_cpu(resp.byte_3) + + le64_to_cpu(resp.byte_5); + + return 0; +} + +int bnxt_re_get_qos_stats(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_ro_counters roce_only_tmp[2] = {{}, {}}; + struct bnxt_re_cnp_counters tmp_counters[2] = {{}, {}}; + struct hwrm_cfa_flow_stats_output resp = {}; + struct hwrm_cfa_flow_stats_input req = {}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg = {}; + struct bnxt_re_cc_stat *cnps; + struct bnxt_re_rstat *dstat; + int rc = 0; + u64 bytes; + u64 pkts; + + /* Issue HWRM cmd to read flow counters for CNP tx and rx */ + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_CFA_FLOW_STATS, -1, -1); + req.num_flows = cpu_to_le16(6); + req.flow_handle_0 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT); + req.flow_handle_1 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + req.flow_handle_2 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT); + req.flow_handle_3 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + req.flow_handle_4 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT); + req.flow_handle_5 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to get CFA Flow stats : rc = 0x%x\n", rc); + goto done; + } + + tmp_counters[0].cnp_tx_pkts = le64_to_cpu(resp.packet_0); + tmp_counters[0].cnp_tx_bytes = le64_to_cpu(resp.byte_0); + tmp_counters[0].cnp_rx_pkts = le64_to_cpu(resp.packet_1); + tmp_counters[0].cnp_rx_bytes = le64_to_cpu(resp.byte_1); + + roce_only_tmp[0].tx_pkts = le64_to_cpu(resp.packet_2) + + le64_to_cpu(resp.packet_4); + roce_only_tmp[0].tx_bytes = le64_to_cpu(resp.byte_2) + + le64_to_cpu(resp.byte_4); + roce_only_tmp[0].rx_pkts = le64_to_cpu(resp.packet_3) + + le64_to_cpu(resp.packet_5); + roce_only_tmp[0].rx_bytes = le64_to_cpu(resp.byte_3) + + le64_to_cpu(resp.byte_5); + + cnps = &rdev->stats.cnps; + dstat = &rdev->stats.dstat; + if (!cnps->is_first) { + /* First query done.. */ + cnps->is_first = true; + cnps->prev[0].cnp_tx_pkts = tmp_counters[0].cnp_tx_pkts; + cnps->prev[0].cnp_tx_bytes = tmp_counters[0].cnp_tx_bytes; + cnps->prev[0].cnp_rx_pkts = tmp_counters[0].cnp_rx_pkts; + cnps->prev[0].cnp_rx_bytes = tmp_counters[0].cnp_rx_bytes; + + cnps->prev[1].cnp_tx_pkts = tmp_counters[1].cnp_tx_pkts; + cnps->prev[1].cnp_tx_bytes = tmp_counters[1].cnp_tx_bytes; + cnps->prev[1].cnp_rx_pkts = tmp_counters[1].cnp_rx_pkts; + cnps->prev[1].cnp_rx_bytes = tmp_counters[1].cnp_rx_bytes; + + dstat->prev[0].tx_pkts = roce_only_tmp[0].tx_pkts; + dstat->prev[0].tx_bytes = roce_only_tmp[0].tx_bytes; + dstat->prev[0].rx_pkts = roce_only_tmp[0].rx_pkts; + dstat->prev[0].rx_bytes = roce_only_tmp[0].rx_bytes; + + dstat->prev[1].tx_pkts = roce_only_tmp[1].tx_pkts; + dstat->prev[1].tx_bytes = roce_only_tmp[1].tx_bytes; + dstat->prev[1].rx_pkts = roce_only_tmp[1].rx_pkts; + dstat->prev[1].rx_bytes = roce_only_tmp[1].rx_bytes; + } else { + u64 byte_mask, pkts_mask; + u64 diff; + + byte_mask = bnxt_re_get_cfa_stat_mask(rdev->chip_ctx, + BYTE_MASK); + pkts_mask = bnxt_re_get_cfa_stat_mask(rdev->chip_ctx, + PKTS_MASK); + /* + * Calculate the number of cnp packets and use + * the value to calculate the CRC bytes. + * Multply pkts with 4 and add it to total bytes + */ + pkts = bnxt_re_stat_diff(tmp_counters[0].cnp_tx_pkts, + &cnps->prev[0].cnp_tx_pkts, + pkts_mask); + cnps->cur[0].cnp_tx_pkts += pkts; + diff = bnxt_re_stat_diff(tmp_counters[0].cnp_tx_bytes, + &cnps->prev[0].cnp_tx_bytes, + byte_mask); + bytes = diff + pkts * 4; + cnps->cur[0].cnp_tx_bytes += bytes; + pkts = bnxt_re_stat_diff(tmp_counters[0].cnp_rx_pkts, + &cnps->prev[0].cnp_rx_pkts, + pkts_mask); + cnps->cur[0].cnp_rx_pkts += pkts; + bytes = bnxt_re_stat_diff(tmp_counters[0].cnp_rx_bytes, + &cnps->prev[0].cnp_rx_bytes, + byte_mask); + cnps->cur[0].cnp_rx_bytes += bytes; + + /* + * Calculate the number of cnp packets and use + * the value to calculate the CRC bytes. + * Multply pkts with 4 and add it to total bytes + */ + pkts = bnxt_re_stat_diff(tmp_counters[1].cnp_tx_pkts, + &cnps->prev[1].cnp_tx_pkts, + pkts_mask); + cnps->cur[1].cnp_tx_pkts += pkts; + diff = bnxt_re_stat_diff(tmp_counters[1].cnp_tx_bytes, + &cnps->prev[1].cnp_tx_bytes, + byte_mask); + cnps->cur[1].cnp_tx_bytes += diff + pkts * 4; + pkts = bnxt_re_stat_diff(tmp_counters[1].cnp_rx_pkts, + &cnps->prev[1].cnp_rx_pkts, + pkts_mask); + cnps->cur[1].cnp_rx_pkts += pkts; + bytes = bnxt_re_stat_diff(tmp_counters[1].cnp_rx_bytes, + &cnps->prev[1].cnp_rx_bytes, + byte_mask); + cnps->cur[1].cnp_rx_bytes += bytes; + + pkts = bnxt_re_stat_diff(roce_only_tmp[0].tx_pkts, + &dstat->prev[0].tx_pkts, + pkts_mask); + dstat->cur[0].tx_pkts += pkts; + diff = bnxt_re_stat_diff(roce_only_tmp[0].tx_bytes, + &dstat->prev[0].tx_bytes, + byte_mask); + dstat->cur[0].tx_bytes += diff + pkts * 4; + pkts = bnxt_re_stat_diff(roce_only_tmp[0].rx_pkts, + &dstat->prev[0].rx_pkts, + pkts_mask); + dstat->cur[0].rx_pkts += pkts; + + bytes = bnxt_re_stat_diff(roce_only_tmp[0].rx_bytes, + &dstat->prev[0].rx_bytes, + byte_mask); + dstat->cur[0].rx_bytes += bytes; + pkts = bnxt_re_stat_diff(roce_only_tmp[1].tx_pkts, + &dstat->prev[1].tx_pkts, + pkts_mask); + dstat->cur[1].tx_pkts += pkts; + diff = bnxt_re_stat_diff(roce_only_tmp[1].tx_bytes, + &dstat->prev[1].tx_bytes, + byte_mask); + dstat->cur[1].tx_bytes += diff + pkts * 4; + pkts = bnxt_re_stat_diff(roce_only_tmp[1].rx_pkts, + &dstat->prev[1].rx_pkts, + pkts_mask); + dstat->cur[1].rx_pkts += pkts; + bytes = bnxt_re_stat_diff(roce_only_tmp[1].rx_bytes, + &dstat->prev[1].rx_bytes, + byte_mask); + dstat->cur[1].rx_bytes += bytes; + } +done: + return rc; +} + +static void bnxt_re_copy_ext_stats(struct bnxt_re_dev *rdev, + u8 indx, struct bnxt_qplib_ext_stat *s) +{ + struct bnxt_re_ext_roce_stats *e_errs; + struct bnxt_re_cnp_counters *cnp; + struct bnxt_re_ext_rstat *ext_d; + struct bnxt_re_ro_counters *ro; + + cnp = &rdev->stats.cnps.cur[indx]; + ro = &rdev->stats.dstat.cur[indx]; + ext_d = &rdev->stats.dstat.ext_rstat[indx]; + e_errs = &rdev->stats.dstat.e_errs; + + cnp->cnp_tx_pkts = s->tx_cnp; + cnp->cnp_rx_pkts = s->rx_cnp; + /* In bonding mode do not duplicate other stats */ + if (indx) + return; + cnp->ecn_marked = s->rx_ecn_marked; + + ro->tx_pkts = s->tx_roce_pkts; + ro->tx_bytes = s->tx_roce_bytes; + ro->rx_pkts = s->rx_roce_pkts; + ro->rx_bytes = s->rx_roce_bytes; + + ext_d->tx.atomic_req = s->tx_atomic_req; + ext_d->tx.read_req = s->tx_read_req; + ext_d->tx.read_resp = s->tx_read_res; + ext_d->tx.write_req = s->tx_write_req; + ext_d->tx.send_req = s->tx_send_req; + ext_d->rx.atomic_req = s->rx_atomic_req; + ext_d->rx.read_req = s->rx_read_req; + ext_d->rx.read_resp = s->rx_read_res; + ext_d->rx.write_req = s->rx_write_req; + ext_d->rx.send_req = s->rx_send_req; + ext_d->grx.rx_pkts = s->rx_roce_good_pkts; + ext_d->grx.rx_bytes = s->rx_roce_good_bytes; + ext_d->rx_dcn_payload_cut = s->rx_dcn_payload_cut; + ext_d->te_bypassed = s->te_bypassed; + e_errs->oob = s->rx_out_of_buffer; + e_errs->oos = s->rx_out_of_sequence; + e_errs->seq_err_naks_rcvd = s->seq_err_naks_rcvd; + e_errs->rnr_naks_rcvd = s->rnr_naks_rcvd; + e_errs->missing_resp = s->missing_resp; + e_errs->to_retransmits = s->to_retransmits; + e_errs->dup_req = s->dup_req; +} + +static int bnxt_re_get_ext_stat(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_ext_stat estat[2] = {{}, {}}; + struct bnxt_qplib_query_stats_info sinfo; + u32 fid; + int rc; + + fid = PCI_FUNC(rdev->en_dev->pdev->devfn); + /* Set default values for sinfo */ + sinfo.function_id = 0xFFFFFFFF; + sinfo.collection_id = 0xFF; + sinfo.vf_valid = false; + rc = bnxt_qplib_qext_stat(&rdev->rcfw, fid, &estat[0], &sinfo); + if (rc) + goto done; + bnxt_re_copy_ext_stats(rdev, 0, &estat[0]); + +done: + return rc; +} + +static void bnxt_re_copy_rstat(struct bnxt_re_rdata_counters *d, + struct ctx_hw_stats_ext *s, + bool is_thor) +{ + d->tx_ucast_pkts = le64_to_cpu(s->tx_ucast_pkts); + d->tx_mcast_pkts = le64_to_cpu(s->tx_mcast_pkts); + d->tx_bcast_pkts = le64_to_cpu(s->tx_bcast_pkts); + d->tx_discard_pkts = le64_to_cpu(s->tx_discard_pkts); + d->tx_error_pkts = le64_to_cpu(s->tx_error_pkts); + d->tx_ucast_bytes = le64_to_cpu(s->tx_ucast_bytes); + /* Add four bytes of CRC bytes per packet */ + d->tx_ucast_bytes += d->tx_ucast_pkts * 4; + d->tx_mcast_bytes = le64_to_cpu(s->tx_mcast_bytes); + d->tx_bcast_bytes = le64_to_cpu(s->tx_bcast_bytes); + d->rx_ucast_pkts = le64_to_cpu(s->rx_ucast_pkts); + d->rx_mcast_pkts = le64_to_cpu(s->rx_mcast_pkts); + d->rx_bcast_pkts = le64_to_cpu(s->rx_bcast_pkts); + d->rx_discard_pkts = le64_to_cpu(s->rx_discard_pkts); + d->rx_error_pkts = le64_to_cpu(s->rx_error_pkts); + d->rx_ucast_bytes = le64_to_cpu(s->rx_ucast_bytes); + d->rx_mcast_bytes = le64_to_cpu(s->rx_mcast_bytes); + d->rx_bcast_bytes = le64_to_cpu(s->rx_bcast_bytes); + if (is_thor) { + d->rx_agg_pkts = le64_to_cpu(s->rx_tpa_pkt); + d->rx_agg_bytes = le64_to_cpu(s->rx_tpa_bytes); + d->rx_agg_events = le64_to_cpu(s->rx_tpa_events); + d->rx_agg_aborts = le64_to_cpu(s->rx_tpa_errors); + } +} + +static void bnxt_re_get_roce_data_stats(struct bnxt_re_dev *rdev) +{ + bool is_thor = _is_chip_gen_p5_p7(rdev->chip_ctx); + struct bnxt_re_rdata_counters *rstat; + + rstat = &rdev->stats.dstat.rstat[0]; + bnxt_re_copy_rstat(rstat, rdev->qplib_res.hctx->stats.dma, is_thor); + +} + +int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_query_stats_info sinfo; + int rc = 0; + + /* Stats are in 1s cadence */ + if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags)) { + if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn)) + rc = bnxt_re_get_ext_stat(rdev); + else + rc = bnxt_re_get_qos_stats(rdev); + + if (rc && rc != -ENOMEM) + clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, + &rdev->flags); + } + + if (test_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags)) { + bnxt_re_get_roce_data_stats(rdev); + + /* Set default values for sinfo */ + sinfo.function_id = 0xFFFFFFFF; + sinfo.collection_id = 0xFF; + sinfo.vf_valid = false; + rc = bnxt_qplib_get_roce_error_stats(&rdev->rcfw, + &rdev->stats.dstat.errs, + &sinfo); + if (rc && rc != -ENOMEM) + clear_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, + &rdev->flags); + } + + return rc; +} + +static const char * const bnxt_re_stat_descs[] = { + "link_state", + "max_qp", + "max_srq", + "max_cq", + "max_mr", + "max_mw", + "max_ah", + "max_pd", + "active_qp", + "active_rc_qp", + "active_ud_qp", + "active_srq", + "active_cq", + "active_mr", + "active_mw", + "active_ah", + "active_pd", + "qp_watermark", + "rc_qp_watermark", + "ud_qp_watermark", + "srq_watermark", + "cq_watermark", + "mr_watermark", + "mw_watermark", + "ah_watermark", + "pd_watermark", + "resize_cq_count", + "hw_retransmission", + "recoverable_errors", + "rx_pkts", + "rx_bytes", + "tx_pkts", + "tx_bytes", + "cnp_tx_pkts", + "cnp_tx_bytes", + "cnp_rx_pkts", + "cnp_rx_bytes", + "roce_only_rx_pkts", + "roce_only_rx_bytes", + "roce_only_tx_pkts", + "roce_only_tx_bytes", + "rx_roce_error_pkts", + "rx_roce_discard_pkts", + "tx_roce_error_pkts", + "tx_roce_discards_pkts", + "res_oob_drop_count", + "tx_atomic_req", + "rx_atomic_req", + "tx_read_req", + "tx_read_resp", + "rx_read_req", + "rx_read_resp", + "tx_write_req", + "rx_write_req", + "tx_send_req", + "rx_send_req", + "rx_good_pkts", + "rx_good_bytes", + "rx_dcn_payload_cut", + "te_bypassed", + "rx_ecn_marked_pkts", + "max_retry_exceeded", + "to_retransmits", + "seq_err_naks_rcvd", + "rnr_naks_rcvd", + "missing_resp", + "dup_reqs", + "unrecoverable_err", + "bad_resp_err", + "local_qp_op_err", + "local_protection_err", + "mem_mgmt_op_err", + "remote_invalid_req_err", + "remote_access_err", + "remote_op_err", + "res_exceed_max", + "res_length_mismatch", + "res_exceeds_wqe", + "res_opcode_err", + "res_rx_invalid_rkey", + "res_rx_domain_err", + "res_rx_no_perm", + "res_rx_range_err", + "res_tx_invalid_rkey", + "res_tx_domain_err", + "res_tx_no_perm", + "res_tx_range_err", + "res_irrq_oflow", + "res_unsup_opcode", + "res_unaligned_atomic", + "res_rem_inv_err", + "res_mem_error64", + "res_srq_err", + "res_cmp_err", + "res_invalid_dup_rkey", + "res_wqe_format_err", + "res_cq_load_err", + "res_srq_load_err", + "res_tx_pci_err", + "res_rx_pci_err", + "res_oos_drop_count", + "num_irq_started", + "num_irq_stopped", + "poll_in_intr_en", + "poll_in_intr_dis", + "cmdq_full_dbg_cnt", + "fw_service_prof_type_sup", + "dbq_int_recv", + "dbq_int_en", + "dbq_pacing_resched", + "dbq_pacing_complete", + "dbq_pacing_alerts", + "dbq_dbr_fifo_reg" + +}; + +static void bnxt_re_print_ext_stat(struct bnxt_re_dev *rdev, + struct rdma_hw_stats *stats) +{ + struct bnxt_re_cnp_counters *cnp; + struct bnxt_re_ext_rstat *ext_s; + + ext_s = &rdev->stats.dstat.ext_rstat[0]; + cnp = &rdev->stats.cnps.cur[0]; + + stats->value[BNXT_RE_TX_ATOMIC_REQ] = ext_s->tx.atomic_req; + stats->value[BNXT_RE_RX_ATOMIC_REQ] = ext_s->rx.atomic_req; + stats->value[BNXT_RE_TX_READ_REQ] = ext_s->tx.read_req; + stats->value[BNXT_RE_TX_READ_RESP] = ext_s->tx.read_resp; + stats->value[BNXT_RE_RX_READ_REQ] = ext_s->rx.read_req; + stats->value[BNXT_RE_RX_READ_RESP] = ext_s->rx.read_resp; + stats->value[BNXT_RE_TX_WRITE_REQ] = ext_s->tx.write_req; + stats->value[BNXT_RE_RX_WRITE_REQ] = ext_s->rx.write_req; + stats->value[BNXT_RE_TX_SEND_REQ] = ext_s->tx.send_req; + stats->value[BNXT_RE_RX_SEND_REQ] = ext_s->rx.send_req; + stats->value[BNXT_RE_RX_GOOD_PKTS] = ext_s->grx.rx_pkts; + stats->value[BNXT_RE_RX_GOOD_BYTES] = ext_s->grx.rx_bytes; + if (_is_chip_p7(rdev->chip_ctx)) { + stats->value[BNXT_RE_RX_DCN_PAYLOAD_CUT] = ext_s->rx_dcn_payload_cut; + stats->value[BNXT_RE_TE_BYPASSED] = ext_s->te_bypassed; + } + stats->value[BNXT_RE_RX_ECN_MARKED_PKTS] = cnp->ecn_marked; +} + +static void bnxt_re_print_roce_only_counters(struct bnxt_re_dev *rdev, + struct rdma_hw_stats *stats) +{ + struct bnxt_re_ro_counters *roce_only = &rdev->stats.dstat.cur[0]; + + stats->value[BNXT_RE_ROCE_ONLY_RX_PKTS] = roce_only->rx_pkts; + stats->value[BNXT_RE_ROCE_ONLY_RX_BYTES] = roce_only->rx_bytes; + stats->value[BNXT_RE_ROCE_ONLY_TX_PKTS] = roce_only->tx_pkts; + stats->value[BNXT_RE_ROCE_ONLY_TX_BYTES] = roce_only->tx_bytes; +} + +static void bnxt_re_print_normal_total_counters(struct bnxt_re_dev *rdev, + struct rdma_hw_stats *stats) +{ + struct bnxt_re_ro_counters *roce_only; + struct bnxt_re_cc_stat *cnps; + + cnps = &rdev->stats.cnps; + roce_only = &rdev->stats.dstat.cur[0]; + + stats->value[BNXT_RE_RX_PKTS] = cnps->cur[0].cnp_rx_pkts + roce_only->rx_pkts; + stats->value[BNXT_RE_RX_BYTES] = cnps->cur[0].cnp_rx_bytes + roce_only->rx_bytes; + stats->value[BNXT_RE_TX_PKTS] = cnps->cur[0].cnp_tx_pkts + roce_only->tx_pkts; + stats->value[BNXT_RE_TX_BYTES] = cnps->cur[0].cnp_tx_bytes + roce_only->tx_bytes; +} + +static void bnxt_re_print_normal_counters(struct bnxt_re_dev *rdev, + struct rdma_hw_stats *rstats) +{ + struct bnxt_re_rdata_counters *stats; + struct bnxt_re_cc_stat *cnps; + bool en_disp; + + stats = &rdev->stats.dstat.rstat[0]; + cnps = &rdev->stats.cnps; + en_disp = !_is_chip_gen_p5_p7(rdev->chip_ctx); + + bnxt_re_print_normal_total_counters(rdev, rstats); + if (!rdev->is_virtfn) { + rstats->value[BNXT_RE_CNP_TX_PKTS] = cnps->cur[0].cnp_tx_pkts; + if (en_disp) + rstats->value[BNXT_RE_CNP_TX_BYTES] = cnps->cur[0].cnp_tx_bytes; + rstats->value[BNXT_RE_CNP_RX_PKTS] = cnps->cur[0].cnp_rx_pkts; + if (en_disp) + rstats->value[BNXT_RE_CNP_RX_BYTES] = cnps->cur[0].cnp_rx_bytes; + } + /* Print RoCE only bytes.. CNP counters include RoCE packets also */ + bnxt_re_print_roce_only_counters(rdev, rstats); + + rstats->value[BNXT_RE_RX_ROCE_ERROR_PKTS] = stats ? stats->rx_error_pkts : 0; + rstats->value[BNXT_RE_RX_ROCE_DISCARD_PKTS] = stats ? stats->rx_discard_pkts : 0; + if (!en_disp) { + rstats->value[BNXT_RE_TX_ROCE_ERROR_PKTS] = stats ? stats->tx_error_pkts : 0; + rstats->value[BNXT_RE_TX_ROCE_DISCARDS_PKTS] = stats ? stats->tx_discard_pkts : 0; + } + + if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn)) { + rstats->value[BNXT_RE_RES_OOB_DROP_COUNT] = rdev->stats.dstat.e_errs.oob; + bnxt_re_print_ext_stat(rdev, rstats); + } +} + +static void bnxt_re_copy_db_pacing_stats(struct bnxt_re_dev *rdev, + struct rdma_hw_stats *stats) +{ + struct bnxt_re_dbr_sw_stats *dbr_sw_stats = rdev->dbr_sw_stats; + + stats->value[BNXT_RE_DBQ_PACING_RESCHED] = dbr_sw_stats->dbq_pacing_resched; + stats->value[BNXT_RE_DBQ_PACING_CMPL] = dbr_sw_stats->dbq_pacing_complete; + stats->value[BNXT_RE_DBQ_PACING_ALERT] = dbr_sw_stats->dbq_pacing_alerts; + stats->value[BNXT_RE_DBQ_DBR_FIFO_REG] = readl_fbsd(rdev->en_dev->softc, + rdev->dbr_db_fifo_reg_off, 0); +} + +int bnxt_re_get_hw_stats(struct ib_device *ibdev, + struct rdma_hw_stats *stats, + u8 port, int index) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_re_ext_roce_stats *e_errs; + struct bnxt_re_rdata_counters *rstat; + struct bnxt_qplib_roce_stats *errs; + unsigned long tstamp_diff; + struct pci_dev *pdev; + int sched_msec; + int rc = 0; + + if (!port || !stats) + return -EINVAL; + + if (!rdev) + return -ENODEV; + + if (!__bnxt_re_is_rdev_valid(rdev)) { + return -ENODEV; + } + + pdev = rdev->en_dev->pdev; + errs = &rdev->stats.dstat.errs; + rstat = &rdev->stats.dstat.rstat[0]; + e_errs = &rdev->stats.dstat.e_errs; +#define BNXT_RE_STATS_CTX_UPDATE_TIMER 250 + sched_msec = BNXT_RE_STATS_CTX_UPDATE_TIMER; + tstamp_diff = jiffies - rdev->stats.read_tstamp; + if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { + if (/* restrict_stats && */ tstamp_diff < msecs_to_jiffies(sched_msec)) + goto skip_query; + rc = bnxt_re_get_device_stats(rdev); + if (rc) + dev_err(rdev_to_dev(rdev), + "Failed to query device stats\n"); + rdev->stats.read_tstamp = jiffies; + } + + if (rdev->dbr_pacing) + bnxt_re_copy_db_pacing_stats(rdev, stats); + +skip_query: + + if (rdev->netdev) + stats->value[BNXT_RE_LINK_STATE] = bnxt_re_link_state(rdev); + stats->value[BNXT_RE_MAX_QP] = rdev->dev_attr->max_qp; + stats->value[BNXT_RE_MAX_SRQ] = rdev->dev_attr->max_srq; + stats->value[BNXT_RE_MAX_CQ] = rdev->dev_attr->max_cq; + stats->value[BNXT_RE_MAX_MR] = rdev->dev_attr->max_mr; + stats->value[BNXT_RE_MAX_MW] = rdev->dev_attr->max_mw; + stats->value[BNXT_RE_MAX_AH] = rdev->dev_attr->max_ah; + stats->value[BNXT_RE_MAX_PD] = rdev->dev_attr->max_pd; + stats->value[BNXT_RE_ACTIVE_QP] = atomic_read(&rdev->stats.rsors.qp_count); + stats->value[BNXT_RE_ACTIVE_RC_QP] = atomic_read(&rdev->stats.rsors.rc_qp_count); + stats->value[BNXT_RE_ACTIVE_UD_QP] = atomic_read(&rdev->stats.rsors.ud_qp_count); + stats->value[BNXT_RE_ACTIVE_SRQ] = atomic_read(&rdev->stats.rsors.srq_count); + stats->value[BNXT_RE_ACTIVE_CQ] = atomic_read(&rdev->stats.rsors.cq_count); + stats->value[BNXT_RE_ACTIVE_MR] = atomic_read(&rdev->stats.rsors.mr_count); + stats->value[BNXT_RE_ACTIVE_MW] = atomic_read(&rdev->stats.rsors.mw_count); + stats->value[BNXT_RE_ACTIVE_AH] = atomic_read(&rdev->stats.rsors.ah_count); + stats->value[BNXT_RE_ACTIVE_PD] = atomic_read(&rdev->stats.rsors.pd_count); + stats->value[BNXT_RE_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_qp_count); + stats->value[BNXT_RE_RC_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_rc_qp_count); + stats->value[BNXT_RE_UD_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_ud_qp_count); + stats->value[BNXT_RE_SRQ_WATERMARK] = atomic_read(&rdev->stats.rsors.max_srq_count); + stats->value[BNXT_RE_CQ_WATERMARK] = atomic_read(&rdev->stats.rsors.max_cq_count); + stats->value[BNXT_RE_MR_WATERMARK] = atomic_read(&rdev->stats.rsors.max_mr_count); + stats->value[BNXT_RE_MW_WATERMARK] = atomic_read(&rdev->stats.rsors.max_mw_count); + stats->value[BNXT_RE_AH_WATERMARK] = atomic_read(&rdev->stats.rsors.max_ah_count); + stats->value[BNXT_RE_PD_WATERMARK] = atomic_read(&rdev->stats.rsors.max_pd_count); + stats->value[BNXT_RE_RESIZE_CQ_COUNT] = atomic_read(&rdev->stats.rsors.resize_count); + stats->value[BNXT_RE_HW_RETRANSMISSION] = BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags) ? 1 : 0; + stats->value[BNXT_RE_RECOVERABLE_ERRORS] = rstat ? rstat->tx_bcast_pkts : 0; + + bnxt_re_print_normal_counters(rdev, stats); + + + stats->value[BNXT_RE_MAX_RETRY_EXCEEDED] = errs->max_retry_exceeded; + if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn) && + _is_hw_retx_supported(rdev->dev_attr->dev_cap_flags)) { + stats->value[BNXT_RE_TO_RETRANSMITS] = e_errs->to_retransmits; + stats->value[BNXT_RE_SEQ_ERR_NAKS_RCVD] = e_errs->seq_err_naks_rcvd; + stats->value[BNXT_RE_RNR_NAKS_RCVD] = e_errs->rnr_naks_rcvd; + stats->value[BNXT_RE_MISSING_RESP] = e_errs->missing_resp; + stats->value[BNXT_RE_DUP_REQS] = e_errs->dup_req; + } else { + stats->value[BNXT_RE_TO_RETRANSMITS] = errs->to_retransmits; + stats->value[BNXT_RE_SEQ_ERR_NAKS_RCVD] = errs->seq_err_naks_rcvd; + stats->value[BNXT_RE_RNR_NAKS_RCVD] = errs->rnr_naks_rcvd; + stats->value[BNXT_RE_MISSING_RESP] = errs->missing_resp; + stats->value[BNXT_RE_DUP_REQS] = errs->dup_req; + } + + stats->value[BNXT_RE_UNRECOVERABLE_ERR] = errs->unrecoverable_err; + stats->value[BNXT_RE_BAD_RESP_ERR] = errs->bad_resp_err; + stats->value[BNXT_RE_LOCAL_QP_OP_ERR] = errs->local_qp_op_err; + stats->value[BNXT_RE_LOCAL_PROTECTION_ERR] = errs->local_protection_err; + stats->value[BNXT_RE_MEM_MGMT_OP_ERR] = errs->mem_mgmt_op_err; + stats->value[BNXT_RE_REMOTE_INVALID_REQ_ERR] = errs->remote_invalid_req_err; + stats->value[BNXT_RE_REMOTE_ACCESS_ERR] = errs->remote_access_err; + stats->value[BNXT_RE_REMOTE_OP_ERR] = errs->remote_op_err; + stats->value[BNXT_RE_RES_EXCEED_MAX] = errs->res_exceed_max; + stats->value[BNXT_RE_RES_LENGTH_MISMATCH] = errs->res_length_mismatch; + stats->value[BNXT_RE_RES_EXCEEDS_WQE] = errs->res_exceeds_wqe; + stats->value[BNXT_RE_RES_OPCODE_ERR] = errs->res_opcode_err; + stats->value[BNXT_RE_RES_RX_INVALID_RKEY] = errs->res_rx_invalid_rkey; + stats->value[BNXT_RE_RES_RX_DOMAIN_ERR] = errs->res_rx_domain_err; + stats->value[BNXT_RE_RES_RX_NO_PERM] = errs->res_rx_no_perm; + stats->value[BNXT_RE_RES_RX_RANGE_ERR] = errs->res_rx_range_err; + stats->value[BNXT_RE_RES_TX_INVALID_RKEY] = errs->res_tx_invalid_rkey; + stats->value[BNXT_RE_RES_TX_DOMAIN_ERR] = errs->res_tx_domain_err; + stats->value[BNXT_RE_RES_TX_NO_PERM] = errs->res_tx_no_perm; + stats->value[BNXT_RE_RES_TX_RANGE_ERR] = errs->res_tx_range_err; + stats->value[BNXT_RE_RES_IRRQ_OFLOW] = errs->res_irrq_oflow; + stats->value[BNXT_RE_RES_UNSUP_OPCODE] = errs->res_unsup_opcode; + stats->value[BNXT_RE_RES_UNALIGNED_ATOMIC] = errs->res_unaligned_atomic; + stats->value[BNXT_RE_RES_REM_INV_ERR] = errs->res_rem_inv_err; + stats->value[BNXT_RE_RES_MEM_ERROR64] = errs->res_mem_error; + stats->value[BNXT_RE_RES_SRQ_ERR] = errs->res_srq_err; + stats->value[BNXT_RE_RES_CMP_ERR] = errs->res_cmp_err; + stats->value[BNXT_RE_RES_INVALID_DUP_RKEY] = errs->res_invalid_dup_rkey; + stats->value[BNXT_RE_RES_WQE_FORMAT_ERR] = errs->res_wqe_format_err; + stats->value[BNXT_RE_RES_CQ_LOAD_ERR] = errs->res_cq_load_err; + stats->value[BNXT_RE_RES_SRQ_LOAD_ERR] = errs->res_srq_load_err; + stats->value[BNXT_RE_RES_TX_PCI_ERR] = errs->res_tx_pci_err; + stats->value[BNXT_RE_RES_RX_PCI_ERR] = errs->res_rx_pci_err; + + + if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn)) { + stats->value[BNXT_RE_RES_OOS_DROP_COUNT] = e_errs->oos; + } else { + /* Display on function 0 as OOS counters are chip-wide */ + if (PCI_FUNC(pdev->devfn) == 0) + stats->value[BNXT_RE_RES_OOS_DROP_COUNT] = errs->res_oos_drop_count; + } + stats->value[BNXT_RE_NUM_IRQ_STARTED] = rdev->rcfw.num_irq_started; + stats->value[BNXT_RE_NUM_IRQ_STOPPED] = rdev->rcfw.num_irq_stopped; + stats->value[BNXT_RE_POLL_IN_INTR_EN] = rdev->rcfw.poll_in_intr_en; + stats->value[BNXT_RE_POLL_IN_INTR_DIS] = rdev->rcfw.poll_in_intr_dis; + stats->value[BNXT_RE_CMDQ_FULL_DBG_CNT] = rdev->rcfw.cmdq_full_dbg; + if (!rdev->is_virtfn) + stats->value[BNXT_RE_FW_SERVICE_PROF_TYPE_SUP] = is_qport_service_type_supported(rdev); + + return ARRAY_SIZE(bnxt_re_stat_descs); +} + +struct rdma_hw_stats *bnxt_re_alloc_hw_port_stats(struct ib_device *ibdev, + u8 port_num) +{ + return rdma_alloc_hw_stats_struct(bnxt_re_stat_descs, + ARRAY_SIZE(bnxt_re_stat_descs), + RDMA_HW_STATS_DEFAULT_LIFESPAN); +} diff --git a/sys/dev/bnxt/bnxt_re/stats.h b/sys/dev/bnxt/bnxt_re/stats.h new file mode 100644 index 00000000000..748d8165947 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/stats.h @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: statistics related data structures + */ + +#ifndef __STATS_H__ +#define __STATS_H__ + +#define BNXT_RE_CFA_STAT_BYTES_MASK 0xFFFFFFFFF +#define BNXT_RE_CFA_STAT_PKTS_MASK 0xFFFFFFF +enum { + BYTE_MASK = 0, + PKTS_MASK = 1 +}; + +struct bnxt_re_cnp_counters { + u64 cnp_tx_pkts; + u64 cnp_tx_bytes; + u64 cnp_rx_pkts; + u64 cnp_rx_bytes; + u64 ecn_marked; +}; + +struct bnxt_re_ro_counters { + u64 tx_pkts; + u64 tx_bytes; + u64 rx_pkts; + u64 rx_bytes; +}; + +struct bnxt_re_flow_counters { + struct bnxt_re_ro_counters ro_stats; + struct bnxt_re_cnp_counters cnp_stats; +}; + +struct bnxt_re_ext_cntr { + u64 atomic_req; + u64 read_req; + u64 read_resp; + u64 write_req; + u64 send_req; +}; + +struct bnxt_re_ext_good { + u64 rx_pkts; + u64 rx_bytes; +}; + +struct bnxt_re_ext_rstat { + struct bnxt_re_ext_cntr tx; + struct bnxt_re_ext_cntr rx; + struct bnxt_re_ext_good grx; + u64 rx_dcn_payload_cut; + u64 te_bypassed; +}; + +struct bnxt_re_rdata_counters { + u64 tx_ucast_pkts; + u64 tx_mcast_pkts; + u64 tx_bcast_pkts; + u64 tx_discard_pkts; + u64 tx_error_pkts; + u64 tx_ucast_bytes; + u64 tx_mcast_bytes; + u64 tx_bcast_bytes; + u64 rx_ucast_pkts; + u64 rx_mcast_pkts; + u64 rx_bcast_pkts; + u64 rx_discard_pkts; + u64 rx_error_pkts; + u64 rx_ucast_bytes; + u64 rx_mcast_bytes; + u64 rx_bcast_bytes; + u64 rx_agg_pkts; + u64 rx_agg_bytes; + u64 rx_agg_events; + u64 rx_agg_aborts; +}; + +struct bnxt_re_cc_stat { + struct bnxt_re_cnp_counters prev[2]; + struct bnxt_re_cnp_counters cur[2]; + bool is_first; +}; + +struct bnxt_re_ext_roce_stats { + u64 oob; + u64 oos; + u64 seq_err_naks_rcvd; + u64 rnr_naks_rcvd; + u64 missing_resp; + u64 to_retransmits; + u64 dup_req; +}; + +struct bnxt_re_rstat { + struct bnxt_re_ro_counters prev[2]; + struct bnxt_re_ro_counters cur[2]; + struct bnxt_re_rdata_counters rstat[2]; + struct bnxt_re_ext_rstat ext_rstat[2]; + struct bnxt_re_ext_roce_stats e_errs; + struct bnxt_qplib_roce_stats errs; + unsigned long long prev_oob; +}; + +struct bnxt_re_res_cntrs { + atomic_t qp_count; + atomic_t rc_qp_count; + atomic_t ud_qp_count; + atomic_t cq_count; + atomic_t srq_count; + atomic_t mr_count; + atomic_t mw_count; + atomic_t ah_count; + atomic_t pd_count; + atomic_t resize_count; + atomic_t max_qp_count; + atomic_t max_rc_qp_count; + atomic_t max_ud_qp_count; + atomic_t max_cq_count; + atomic_t max_srq_count; + atomic_t max_mr_count; + atomic_t max_mw_count; + atomic_t max_ah_count; + atomic_t max_pd_count; +}; + +struct bnxt_re_device_stats { + struct bnxt_re_rstat dstat; + struct bnxt_re_res_cntrs rsors; + struct bnxt_re_cc_stat cnps; + unsigned long read_tstamp; + /* To be used in case to disable stats query from worker or change + * query interval. 0 means stats_query disabled. + */ + u32 stats_query_sec; + /* A free running counter to be used along with stats_query_sec to + * decide whether to issue the command to FW. + */ + u32 stats_query_counter; +}; + +static inline u64 bnxt_re_get_cfa_stat_mask(struct bnxt_qplib_chip_ctx *cctx, + bool type) +{ + u64 mask; + + if (type == BYTE_MASK) { + mask = BNXT_RE_CFA_STAT_BYTES_MASK; /* 36 bits */ + if (_is_chip_gen_p5_p7(cctx)) + mask >>= 0x01; /* 35 bits */ + } else { + mask = BNXT_RE_CFA_STAT_PKTS_MASK; /* 28 bits */ + if (_is_chip_gen_p5_p7(cctx)) + mask |= (0x10000000ULL); /* 29 bits */ + } + + return mask; +} + +static inline u64 bnxt_re_stat_diff(u64 cur, u64 *prev, u64 mask) +{ + u64 diff; + + if (!cur) + return 0; + diff = (cur - *prev) & mask; + if (diff) + *prev = cur; + return diff; +} + +static inline void bnxt_re_clear_rsors_stat(struct bnxt_re_res_cntrs *rsors) +{ + atomic_set(&rsors->qp_count, 0); + atomic_set(&rsors->cq_count, 0); + atomic_set(&rsors->srq_count, 0); + atomic_set(&rsors->mr_count, 0); + atomic_set(&rsors->mw_count, 0); + atomic_set(&rsors->ah_count, 0); + atomic_set(&rsors->pd_count, 0); + atomic_set(&rsors->resize_count, 0); + atomic_set(&rsors->max_qp_count, 0); + atomic_set(&rsors->max_cq_count, 0); + atomic_set(&rsors->max_srq_count, 0); + atomic_set(&rsors->max_mr_count, 0); + atomic_set(&rsors->max_mw_count, 0); + atomic_set(&rsors->max_ah_count, 0); + atomic_set(&rsors->max_pd_count, 0); + atomic_set(&rsors->max_rc_qp_count, 0); + atomic_set(&rsors->max_ud_qp_count, 0); +} + +enum bnxt_re_hw_stats { + BNXT_RE_LINK_STATE, + BNXT_RE_MAX_QP, + BNXT_RE_MAX_SRQ, + BNXT_RE_MAX_CQ, + BNXT_RE_MAX_MR, + BNXT_RE_MAX_MW, + BNXT_RE_MAX_AH, + BNXT_RE_MAX_PD, + BNXT_RE_ACTIVE_QP, + BNXT_RE_ACTIVE_RC_QP, + BNXT_RE_ACTIVE_UD_QP, + BNXT_RE_ACTIVE_SRQ, + BNXT_RE_ACTIVE_CQ, + BNXT_RE_ACTIVE_MR, + BNXT_RE_ACTIVE_MW, + BNXT_RE_ACTIVE_AH, + BNXT_RE_ACTIVE_PD, + BNXT_RE_QP_WATERMARK, + BNXT_RE_RC_QP_WATERMARK, + BNXT_RE_UD_QP_WATERMARK, + BNXT_RE_SRQ_WATERMARK, + BNXT_RE_CQ_WATERMARK, + BNXT_RE_MR_WATERMARK, + BNXT_RE_MW_WATERMARK, + BNXT_RE_AH_WATERMARK, + BNXT_RE_PD_WATERMARK, + BNXT_RE_RESIZE_CQ_COUNT, + BNXT_RE_HW_RETRANSMISSION, + BNXT_RE_RECOVERABLE_ERRORS, + BNXT_RE_RX_PKTS, + BNXT_RE_RX_BYTES, + BNXT_RE_TX_PKTS, + BNXT_RE_TX_BYTES, + BNXT_RE_CNP_TX_PKTS, + BNXT_RE_CNP_TX_BYTES, + BNXT_RE_CNP_RX_PKTS, + BNXT_RE_CNP_RX_BYTES, + BNXT_RE_ROCE_ONLY_RX_PKTS, + BNXT_RE_ROCE_ONLY_RX_BYTES, + BNXT_RE_ROCE_ONLY_TX_PKTS, + BNXT_RE_ROCE_ONLY_TX_BYTES, + BNXT_RE_RX_ROCE_ERROR_PKTS, + BNXT_RE_RX_ROCE_DISCARD_PKTS, + BNXT_RE_TX_ROCE_ERROR_PKTS, + BNXT_RE_TX_ROCE_DISCARDS_PKTS, + BNXT_RE_RES_OOB_DROP_COUNT, + BNXT_RE_TX_ATOMIC_REQ, + BNXT_RE_RX_ATOMIC_REQ, + BNXT_RE_TX_READ_REQ, + BNXT_RE_TX_READ_RESP, + BNXT_RE_RX_READ_REQ, + BNXT_RE_RX_READ_RESP, + BNXT_RE_TX_WRITE_REQ, + BNXT_RE_RX_WRITE_REQ, + BNXT_RE_TX_SEND_REQ, + BNXT_RE_RX_SEND_REQ, + BNXT_RE_RX_GOOD_PKTS, + BNXT_RE_RX_GOOD_BYTES, + BNXT_RE_RX_DCN_PAYLOAD_CUT, + BNXT_RE_TE_BYPASSED, + BNXT_RE_RX_ECN_MARKED_PKTS, + BNXT_RE_MAX_RETRY_EXCEEDED, + BNXT_RE_TO_RETRANSMITS, + BNXT_RE_SEQ_ERR_NAKS_RCVD, + BNXT_RE_RNR_NAKS_RCVD, + BNXT_RE_MISSING_RESP, + BNXT_RE_DUP_REQS, + BNXT_RE_UNRECOVERABLE_ERR, + BNXT_RE_BAD_RESP_ERR, + BNXT_RE_LOCAL_QP_OP_ERR, + BNXT_RE_LOCAL_PROTECTION_ERR, + BNXT_RE_MEM_MGMT_OP_ERR, + BNXT_RE_REMOTE_INVALID_REQ_ERR, + BNXT_RE_REMOTE_ACCESS_ERR, + BNXT_RE_REMOTE_OP_ERR, + BNXT_RE_RES_EXCEED_MAX, + BNXT_RE_RES_LENGTH_MISMATCH, + BNXT_RE_RES_EXCEEDS_WQE, + BNXT_RE_RES_OPCODE_ERR, + BNXT_RE_RES_RX_INVALID_RKEY, + BNXT_RE_RES_RX_DOMAIN_ERR, + BNXT_RE_RES_RX_NO_PERM, + BNXT_RE_RES_RX_RANGE_ERR, + BNXT_RE_RES_TX_INVALID_RKEY, + BNXT_RE_RES_TX_DOMAIN_ERR, + BNXT_RE_RES_TX_NO_PERM, + BNXT_RE_RES_TX_RANGE_ERR, + BNXT_RE_RES_IRRQ_OFLOW, + BNXT_RE_RES_UNSUP_OPCODE, + BNXT_RE_RES_UNALIGNED_ATOMIC, + BNXT_RE_RES_REM_INV_ERR, + BNXT_RE_RES_MEM_ERROR64, + BNXT_RE_RES_SRQ_ERR, + BNXT_RE_RES_CMP_ERR, + BNXT_RE_RES_INVALID_DUP_RKEY, + BNXT_RE_RES_WQE_FORMAT_ERR, + BNXT_RE_RES_CQ_LOAD_ERR, + BNXT_RE_RES_SRQ_LOAD_ERR, + BNXT_RE_RES_TX_PCI_ERR, + BNXT_RE_RES_RX_PCI_ERR, + BNXT_RE_RES_OOS_DROP_COUNT, + BNXT_RE_NUM_IRQ_STARTED, + BNXT_RE_NUM_IRQ_STOPPED, + BNXT_RE_POLL_IN_INTR_EN, + BNXT_RE_POLL_IN_INTR_DIS, + BNXT_RE_CMDQ_FULL_DBG_CNT, + BNXT_RE_FW_SERVICE_PROF_TYPE_SUP, + BNXT_RE_DBQ_INT_RECV, + BNXT_RE_DBQ_INT_EN, + BNXT_RE_DBQ_PACING_RESCHED, + BNXT_RE_DBQ_PACING_CMPL, + BNXT_RE_DBQ_PACING_ALERT, + BNXT_RE_DBQ_DBR_FIFO_REG, + BNXT_RE_DBQ_NUM_EXT_COUNTERS +}; + +#define BNXT_RE_NUM_STD_COUNTERS (BNXT_RE_OUT_OF_SEQ_ERR + 1) + +struct bnxt_re_stats { + struct bnxt_qplib_roce_stats errs; + struct bnxt_qplib_ext_stat ext_stat; +}; + +struct rdma_hw_stats *bnxt_re_alloc_hw_port_stats(struct ib_device *ibdev, + u8 port_num); +int bnxt_re_get_hw_stats(struct ib_device *ibdev, + struct rdma_hw_stats *stats, + u8 port, int index); +int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev); +int bnxt_re_get_flow_stats_from_service_pf(struct bnxt_re_dev *rdev, + struct bnxt_re_flow_counters *stats, + struct bnxt_qplib_query_stats_info *sinfo); +int bnxt_re_get_qos_stats(struct bnxt_re_dev *rdev); +#endif /* __STATS_H__ */ diff --git a/sys/modules/bnxt/bnxt_re/Makefile b/sys/modules/bnxt/bnxt_re/Makefile new file mode 100644 index 00000000000..d6aa701dbae --- /dev/null +++ b/sys/modules/bnxt/bnxt_re/Makefile @@ -0,0 +1,22 @@ +.PATH: ${SRCTOP}/sys/dev/bnxt/bnxt_re + +KMOD=bnxt_re +SRCS += ib_verbs.c ib_verbs.h +SRCS += qplib_fp.c qplib_fp.h +SRCS += qplib_sp.c qplib_sp.h +SRCS += qplib_res.c qplib_res.h +SRCS += qplib_rcfw.c qplib_rcfw.h +SRCS += stats.c stats.h +SRCS += main.c bnxt_re.h +SRCS += opt_inet.h opt_inet6.h opt_ratelimit.h +SRCS += ${LINUXKPI_GENSRCS} + +CFLAGS+= -I${SRCTOP}/sys/dev/bnxt/bnxt_en +CFLAGS+= -I${SRCTOP}/sys/ofed/include +CFLAGS+= -I${SRCTOP}/sys/ofed/include/uapi +CFLAGS+= ${LINUXKPI_INCLUDES} +CFLAGS+= -DCONFIG_INFINIBAND_USER_MEM + +.include + +CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS}