upstream/opnsense-src
1
0
Fork 0
mirror of https://github.com/opnsense/src.git synced 2026-05-28 04:12:45 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

qat: driver updates to improve code and fix bugs

Browse source 
Bug fixes and improvements are done for the qat code base
to improve code quality.

Reviewed by:	markj, ziaee
MFC after:	2 weeks
Sponsored by:   Intel Corporation
Differential Revision:  https://reviews.freebsd.org/D50379

(cherry picked from commit ded037e65e5239671b1292ec987a2e0894b217b5)
This commit is contained in:
Hareshx Sankar Raj 2025-05-07 05:38:21 -04:00 committed by Franco Fichtner
parent b77125c7c7
commit cfb6574431
36 changed files with 263 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
sys/dev/qat/include/adf_cfg_device.h
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#ifndef ADF_CFG_DEVICE_H_
#define ADF_CFG_DEVICE_H_
@ -79,4 +79,6 @@ int adf_cfg_device_init(struct adf_cfg_device *device,
void adf_cfg_device_clear(struct adf_cfg_device *device,
struct adf_accel_dev *accel_dev);
void adf_cfg_device_clear_all(struct adf_accel_dev *accel_dev);
#endif

4
sys/dev/qat/include/adf_gen4vf_hw_csr_data.h
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#ifndef ADF_GEN4VF_HW_CSR_DATA_H_
#define ADF_GEN4VF_HW_CSR_DATA_H_
@ -51,7 +51,7 @@
struct resource *_csr_base_addr = csr_base_addr; \
u32 _bank = bank; \
u32 _ring = ring; \
dma_addr_t _value = value; \
bus_addr_t _value = value; \
u32 l_base = 0, u_base = 0; \
l_base = (u32)((_value)&0xFFFFFFFF); \
u_base = (u32)(((_value)&0xFFFFFFFF00000000ULL) >> 32); \

3
sys/dev/qat/include/adf_pfvf_vf_msg.h
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#ifndef ADF_PFVF_VF_MSG_H
#define ADF_PFVF_VF_MSG_H
@ -8,5 +8,6 @@ void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev);
int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev);
int adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev);
int adf_vf2pf_get_ring_to_svc(struct adf_accel_dev *accel_dev);
void adf_vf2pf_restarting_complete(struct adf_accel_dev *accel_dev);
#endif /* ADF_PFVF_VF_MSG_H */

2
sys/dev/qat/include/common/adf_accel_devices.h
View file

@ -448,6 +448,7 @@ struct adf_hw_device_data {
uint8_t num_accel;
uint8_t num_logical_accel;
uint8_t num_engines;
bool get_ring_to_svc_done;
int (*get_storage_enabled)(struct adf_accel_dev *accel_dev,
uint32_t *storage_enabled);
u8 query_storage_cap;
@ -721,5 +722,6 @@ struct adf_accel_dev {
bool is_vf;
u32 accel_id;
void *lac_dev;
struct mutex lock; /* protect accel_dev during start/stop e.t.c */
};
#endif

4
sys/dev/qat/include/common/adf_cfg_common.h
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#ifndef ADF_CFG_COMMON_H_
#define ADF_CFG_COMMON_H_
@ -62,7 +62,7 @@ struct adf_pci_address {
unsigned char bus;
unsigned char dev;
unsigned char func;
} __packed;
};
#define ADF_CFG_SERV_RING_PAIR_0_SHIFT 0
#define ADF_CFG_SERV_RING_PAIR_1_SHIFT 3

3
sys/dev/qat/include/common/adf_common_drv.h
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#ifndef ADF_DRV_H
#define ADF_DRV_H
@ -304,6 +304,7 @@ void adf_flush_vf_wq(struct adf_accel_dev *accel_dev);
int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev);
int adf_pf2vf_handle_pf_rp_reset(struct adf_accel_dev *accel_dev,
struct pfvf_message msg);
int adf_pf2vf_handle_pf_error(struct adf_accel_dev *accel_dev);
bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev);
static inline int
adf_sriov_configure(device_t *pdev, int numvfs)

6
sys/dev/qat/include/common/adf_gen4_hw_data.h
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2021 Intel Corporation */
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2025 Intel Corporation */
#ifndef ADF_GEN4_HW_CSR_DATA_H_
#define ADF_GEN4_HW_CSR_DATA_H_
@ -62,7 +62,7 @@
struct resource *_csr_base_addr = csr_base_addr; \
u32 _bank = bank; \
u32 _ring = ring; \
dma_addr_t _value = value; \
bus_addr_t _value = value; \
u32 l_base = 0, u_base = 0; \
l_base = lower_32_bits(_value); \
u_base = upper_32_bits(_value); \

8
sys/dev/qat/include/common/adf_pfvf_msg.h
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#ifndef ADF_PFVF_MSG_H
#define ADF_PFVF_MSG_H
@ -97,6 +97,7 @@ enum pf2vf_msgtype {
ADF_PF2VF_MSGTYPE_RESTARTING = 0x01,
ADF_PF2VF_MSGTYPE_VERSION_RESP = 0x02,
ADF_PF2VF_MSGTYPE_BLKMSG_RESP = 0x03,
ADF_PF2VF_MSGTYPE_FATAL_ERROR = 0x04,
/* Values from 0x10 are Gen4 specific, message type is only 4 bits in
Gen2 devices. */
ADF_PF2VF_MSGTYPE_RP_RESET_RESP = 0x10,
@ -111,6 +112,7 @@ enum vf2pf_msgtype {
ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ = 0x07,
ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ = 0x08,
ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ = 0x09,
ADF_VF2PF_MSGTYPE_RESTARTING_COMPLETE = 0x0a,
/* Values from 0x10 are Gen4 specific, message type is only 4 bits in
Gen2 devices. */
ADF_VF2PF_MSGTYPE_RP_RESET = 0x10,
@ -124,8 +126,10 @@ enum pfvf_compatibility_version {
ADF_PFVF_COMPAT_FAST_ACK = 0x03,
/* Ring to service mapping support for non-standard mappings */
ADF_PFVF_COMPAT_RING_TO_SVC_MAP = 0x04,
/* Fallback compat */
ADF_PFVF_COMPAT_FALLBACK = 0x05,
/* Reference to the latest version */
ADF_PFVF_COMPAT_THIS_VERSION = 0x04,
ADF_PFVF_COMPAT_THIS_VERSION = 0x05,
};
/* PF->VF Version Response */

2
sys/dev/qat/include/icp_qat_fw_init_admin.h
View file

@ -198,8 +198,8 @@ struct icp_qat_fw_init_admin_resp {
enum icp_qat_fw_init_admin_init_flag { ICP_QAT_FW_INIT_FLAG_PKE_DISABLED = 0 };
struct icp_qat_fw_init_admin_hb_cnt {
u16 resp_heartbeat_cnt;
u16 req_heartbeat_cnt;
u16 resp_heartbeat_cnt;
};
#define ICP_QAT_FW_COMN_HEARTBEAT_OK 0

11
sys/dev/qat/qat_api/common/compression/dc_datapath.c
View file

@ -118,14 +118,19 @@ dcCompression_ProcessCallback(void *pRespMsg)
/* Cast response message to compression response message type */
pCompRespMsg = (icp_qat_fw_comp_resp_t *)pRespMsg;
if (!(pCompRespMsg)) {
QAT_UTILS_LOG("pCompRespMsg is NULL\n");
return;
}
/* Extract request data pointer from the opaque data */
LAC_MEM_SHARED_READ_TO_PTR(pCompRespMsg->opaque_data, pReqData);
if (!(pReqData)) {
QAT_UTILS_LOG("pReqData is NULL\n");
return;
}
/* Extract fields from the request data structure */
pCookie = (dc_compression_cookie_t *)pReqData;
if (!pCookie)
return;
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pCookie->pSessionHandle);
pService = (sal_compression_service_t *)(pCookie->dcInstance);

8
sys/dev/qat/qat_api/common/crypto/sym/key/lac_sym_key.c
View file

@ -2160,6 +2160,14 @@ LacSymKey_CheckParamSslTls(const void *pKeyGenOpData,
}
}
/* check 0 secret length as it is not valid for SSL3 Key Gen
* request */
if (0 == uSecretLen) {
LAC_INVALID_PARAM_LOG1("%u secret.dataLenInBytes",
uSecretLen);
return CPA_STATUS_INVALID_PARAM;
}
/* Only seed length for SSL3 Key Gen request */
if (maxSeedLen != uSeedLen) {
LAC_INVALID_PARAM_LOG("seed.dataLenInBytes");

11
sys/dev/qat/qat_api/common/crypto/sym/lac_sym_hash.c
View file

@ -751,10 +751,13 @@ LacHash_PerformParamCheck(CpaInstanceHandle instanceHandle,
&pHashAlgInfo);
/* check if the message is a multiple of the block size. */
if ((pOpData->messageLenToHashInBytes %
pHashAlgInfo->blockLength) != 0) {
LAC_INVALID_PARAM_LOG(
"messageLenToHashInBytes not block size");
if (pOpData->messageLenToHashInBytes %
pHashAlgInfo->blockLength !=
0) {
LAC_INVALID_PARAM_LOG2(
"message(%d) not block-size(%d) multiple",
pOpData->messageLenToHashInBytes,
pHashAlgInfo->blockLength);
return CPA_STATUS_INVALID_PARAM;
}
}

37
sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_cipher.c
View file

@ -214,6 +214,11 @@ static const uint8_t key_size_f8[] = {
ICP_QAT_HW_CIPHER_ALGO_AES256 /* ICP_QAT_HW_AES_256_F8_KEY_SZ */
};
/* This array must be kept aligned with CpaCySymCipherAlgorithm enum but
* offset by -1 as that enum starts at 1. LacSymQat_CipherGetCfgData()
* below relies on that alignment and uses that enum -1 to index into this
* array.
*/
typedef struct _icp_qat_hw_cipher_info {
icp_qat_hw_cipher_algo_t algorithm;
icp_qat_hw_cipher_mode_t mode;
@ -542,7 +547,7 @@ LacSymQat_CipherGetCfgData(lac_session_desc_t *pSession,
sal_crypto_service_t *pService =
(sal_crypto_service_t *)pSession->pInstance;
CpaCySymCipherAlgorithm cipherAlgorithm = 0;
int cipherIdx = 0;
icp_qat_hw_cipher_dir_t cipherDirection = 0;
/* Set defaults */
@ -551,21 +556,33 @@ LacSymQat_CipherGetCfgData(lac_session_desc_t *pSession,
*pMode = ICP_QAT_HW_CIPHER_ECB_MODE;
*pDir = ICP_QAT_HW_CIPHER_ENCRYPT;
/* decrease since it's numbered from 1 instead of 0 */
cipherAlgorithm = pSession->cipherAlgorithm - 1;
/* offset index as CpaCySymCipherAlgorithm enum starts from 1, not from
* 0 */
cipherIdx = pSession->cipherAlgorithm - 1;
cipherDirection =
pSession->cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ?
ICP_QAT_HW_CIPHER_ENCRYPT :
ICP_QAT_HW_CIPHER_DECRYPT;
*pAlgorithm = icp_qat_alg_info[cipherAlgorithm].algorithm;
*pMode = icp_qat_alg_info[cipherAlgorithm].mode;
*pDir = icp_qat_alg_info[cipherAlgorithm].dir[cipherDirection];
*pKey_convert =
icp_qat_alg_info[cipherAlgorithm].key_convert[cipherDirection];
/* Boundary check against the last value in the algorithm enum */
if (!(pSession->cipherAlgorithm <= CPA_CY_SYM_CIPHER_SM4_CTR)) {
QAT_UTILS_LOG("Invalid cipherAlgorithm value\n");
return;
}
if (IS_KEY_DEP_NO != icp_qat_alg_info[cipherAlgorithm].isKeyLenDepend) {
*pAlgorithm = icp_qat_alg_info[cipherAlgorithm]
if (!(cipherDirection <= ICP_QAT_HW_CIPHER_DECRYPT)) {
QAT_UTILS_LOG("Invalid cipherDirection value\n");
return;
}
*pAlgorithm = icp_qat_alg_info[cipherIdx].algorithm;
*pMode = icp_qat_alg_info[cipherIdx].mode;
*pDir = icp_qat_alg_info[cipherIdx].dir[cipherDirection];
*pKey_convert =
icp_qat_alg_info[cipherIdx].key_convert[cipherDirection];
if (IS_KEY_DEP_NO != icp_qat_alg_info[cipherIdx].isKeyLenDepend) {
*pAlgorithm = icp_qat_alg_info[cipherIdx]
.pAlgByKeySize[pSession->cipherKeyLenInBytes];
}

8
sys/dev/qat/qat_api/common/ctrl/sal_compression.c
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
/**
*****************************************************************************
* @file sal_compression.c
@ -371,9 +371,6 @@ SalCtrl_CompressionInit(icp_accel_dev_t *device, sal_service_t *service)
SAL_SERVICE_GOOD_FOR_INIT(pCompressionService);
pCompressionService->generic_service_info.state =
SAL_SERVICE_STATE_INITIALIZING;
if (CPA_FALSE == pCompressionService->generic_service_info.is_dyn) {
section = icpGetProcessName();
}
@ -1438,7 +1435,8 @@ cpaDcInstanceGetInfo2(const CpaInstanceHandle instanceHandle,
pInstanceInfo2->isOffloaded = CPA_TRUE;
/* Get the instance name and part name from the config file */
dev = icp_adf_getAccelDevByAccelId(pCompressionService->pkgID);
if (NULL == dev) {
if (NULL == dev ||
0 == strnlen(dev->deviceName, ADF_DEVICE_TYPE_LENGTH + 1)) {
QAT_UTILS_LOG("Can not find device for the instance.\n");
LAC_OS_BZERO(pInstanceInfo2, sizeof(CpaInstanceInfo2));
return CPA_STATUS_FAIL;

4
sys/dev/qat/qat_api/common/ctrl/sal_crypto.c
View file

@ -1235,7 +1235,8 @@ cpaCyInstanceGetInfo2(const CpaInstanceHandle instanceHandle_in,
/* Get the instance name and part name */
dev = icp_adf_getAccelDevByAccelId(pCryptoService->pkgID);
if (NULL == dev) {
if (NULL == dev ||
0 == strnlen(dev->deviceName, ADF_DEVICE_TYPE_LENGTH + 1)) {
LAC_LOG_ERROR("Can not find device for the instance\n");
LAC_OS_BZERO(pInstanceInfo2, sizeof(CpaInstanceInfo2));
return CPA_STATUS_FAIL;
@ -1717,7 +1718,6 @@ Lac_GetFirstHandle(sal_service_type_t svc_type)
default:
LAC_LOG_ERROR("Invalid service type\n");
return NULL;
break;
}
/* Only need 1 dev with crypto enabled - so check all devices*/
status = icp_amgr_getAllAccelDevByEachCapability(capabilities,

4
sys/dev/qat/qat_api/common/include/sal_types_compression.h
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
/**
***************************************************************************
* @file sal_types_compression.h
@ -24,6 +24,8 @@
#define DC_NUM_RX_RINGS (1)
#define DC_NUM_COMPRESSION_LEVELS (CPA_DC_L12)
#define MAX_SGL_NUM 0x10000
/**
*****************************************************************************
* @ingroup SalCtrl

4
sys/dev/qat/qat_api/include/icp_sal_versions.h
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
/**
***************************************************************************
* @file icp_sal_versions.h
@ -26,7 +26,7 @@
/* Part name and number of the accelerator device */
#define SAL_INFO2_DRIVER_SW_VERSION_MAJ_NUMBER 3
#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 14
#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 15
#define SAL_INFO2_DRIVER_SW_VERSION_PATCH_NUMBER 0
/**

11
sys/dev/qat/qat_common/adf_aer.c
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#include "qat_freebsd.h"
#include "adf_cfg.h"
#include "adf_common_drv.h"
@ -280,6 +280,15 @@ adf_notify_fatal_error_work(struct work_struct *work)
struct adf_fatal_error_data *wq_data =
container_of(work, struct adf_fatal_error_data, work);
struct adf_accel_dev *accel_dev = wq_data->accel_dev;
struct adf_hw_device_data *hw_device = accel_dev->hw_device;
if (adf_dev_in_use(accel_dev)) {
if (hw_device->pre_reset) {
device_printf(GET_DEV(accel_dev),
"Performing pre reset save\n");
hw_device->pre_reset(accel_dev);
}
}
adf_error_notifier((uintptr_t)accel_dev);
if (!accel_dev->is_vf) {

14
sys/dev/qat/qat_common/adf_cfg_device.c
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#include "adf_cfg_instance.h"
#include "adf_cfg_section.h"
#include "adf_cfg_device.h"
@ -677,6 +677,18 @@ adf_cfg_device_clear(struct adf_cfg_device *device,
device->instances = NULL;
}
void
adf_cfg_device_clear_all(struct adf_accel_dev *accel_dev)
{
sx_xlock(&accel_dev->cfg->lock);
if (accel_dev->cfg->dev) {
adf_cfg_device_clear(accel_dev->cfg->dev, accel_dev);
free(accel_dev->cfg->dev, M_QAT);
accel_dev->cfg->dev = NULL;
}
sx_xunlock(&accel_dev->cfg->lock);
}
/*
* Static configuration for userspace
*/

1
sys/dev/qat/qat_common/adf_freebsd_cnvnr_ctrs_dbg.c
View file

@ -93,6 +93,7 @@ static int qat_cnvnr_ctrs_dbg_read(SYSCTL_HANDLER_ARGS)
/* Extracting number of Acceleration Engines */
num_aes = hw_device->get_num_aes(hw_device);
explicit_bzero(&request, sizeof(struct icp_qat_fw_init_admin_req));
for (ae = 0; ae < num_aes; ae++) {
if (accel_dev->au_info && !test_bit(ae, &dc_ae_msk))
continue;

20
sys/dev/qat/qat_common/adf_freebsd_dev_processes.c
View file

@ -1,6 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#include "qat_freebsd.h"
#include "adf_cfg.h"
#include "adf_common_drv.h"
@ -85,7 +84,7 @@ static struct cdevsw adf_state_cdevsw = {
.d_name = ADF_DEV_STATE_NAME,
};
static const struct filterops adf_state_read_filterops = {
static struct filterops adf_state_read_filterops = {
.f_isfd = 1,
.f_attach = NULL,
.f_detach = adf_state_kqread_detach,
@ -410,17 +409,6 @@ adf_state_set(int dev, enum adf_event event)
state->state.dev_state = event;
state->state.dev_id = dev;
STAILQ_INSERT_TAIL(head, state, entries_state);
if (event == ADF_EVENT_STOP) {
state = NULL;
state = malloc(sizeof(struct entry_state),
M_QAT,
M_NOWAIT | M_ZERO);
if (!state)
continue;
state->state.dev_state = ADF_EVENT_SHUTDOWN;
state->state.dev_id = dev;
STAILQ_INSERT_TAIL(head, state, entries_state);
}
}
mtx_unlock(&mtx);
callout_schedule(&callout, ADF_STATE_CALLOUT_TIME);
@ -451,7 +439,7 @@ adf_state_event_handler(struct adf_accel_dev *accel_dev, enum adf_event event)
case ADF_EVENT_START:
return ret;
case ADF_EVENT_STOP:
break;
return ret;
case ADF_EVENT_ERROR:
break;
#if defined(QAT_UIO) && defined(QAT_DBG)
@ -548,6 +536,7 @@ adf_state_destroy(void)
struct entry_proc_events *proc_events = NULL;
adf_service_unregister(&adf_state_hndl);
destroy_dev(adf_state_dev);
mtx_lock(&callout_mtx);
callout_stop(&callout);
mtx_unlock(&callout_mtx);
@ -560,7 +549,6 @@ adf_state_destroy(void)
}
mtx_unlock(&mtx);
mtx_destroy(&mtx);
destroy_dev(adf_state_dev);
}
static int

6
sys/dev/qat/qat_common/adf_gen2_hw_data.c
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2021 Intel Corporation */
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2025 Intel Corporation */
#include "adf_gen2_hw_data.h"
#include "icp_qat_hw.h"
@ -54,7 +54,7 @@ write_csr_ring_config(struct resource *csr_base_addr,
WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value);
}
static dma_addr_t
static bus_addr_t
read_csr_ring_base(struct resource *csr_base_addr, u32 bank, u32 ring)
{
return READ_CSR_RING_BASE(csr_base_addr, bank, ring);

3
sys/dev/qat/qat_common/adf_gen4_pfvf.c
View file

@ -1,7 +1,6 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#include <linux/iopoll.h>
#include <linux/mutex.h>
#include <linux/types.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"

8
sys/dev/qat/qat_common/adf_gen4vf_hw_csr_data.c
View file

@ -1,10 +1,10 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#include "adf_accel_devices.h"
#include "adf_gen4vf_hw_csr_data.h"
static u64
build_csr_ring_base_addr(dma_addr_t addr, u32 size)
build_csr_ring_base_addr(bus_addr_t addr, u32 size)
{
return BUILD_RING_BASE_ADDR_GEN4(addr, size);
}
@ -54,7 +54,7 @@ write_csr_ring_config(struct resource *csr_base_addr,
WRITE_CSR_RING_CONFIG_GEN4VF(csr_base_addr, bank, ring, value);
}
static dma_addr_t
static bus_addr_t
read_csr_ring_base(struct resource *csr_base_addr, u32 bank, u32 ring)
{
return READ_CSR_RING_BASE_GEN4VF(csr_base_addr, bank, ring);
@ -64,7 +64,7 @@ static void
write_csr_ring_base(struct resource *csr_base_addr,
u32 bank,
u32 ring,
dma_addr_t addr)
bus_addr_t addr)
{
WRITE_CSR_RING_BASE_GEN4VF(csr_base_addr, bank, ring, addr);
}

66
sys/dev/qat/qat_common/adf_init.c
View file

@ -20,6 +20,10 @@
#include "adf_common_drv.h"
#include "icp_qat_fw.h"
#if defined(QAT_UIO)
#include "adf_cfg_device.h"
#endif /* QAT_UIO*/
/* Mask used to check the CompressAndVerify capability bit */
#define DC_CNV_EXTENDED_CAPABILITY (0x01)
@ -29,6 +33,11 @@
static LIST_HEAD(service_table);
static DEFINE_MUTEX(service_lock);
static int adf_dev_init_locked(struct adf_accel_dev *accel_dev);
static int adf_dev_start_locked(struct adf_accel_dev *accel_dev);
static int adf_dev_stop_locked(struct adf_accel_dev *accel_dev);
static void adf_dev_shutdown_locked(struct adf_accel_dev *accel_dev);
static void
adf_service_add(struct service_hndl *service)
{
@ -261,6 +270,18 @@ adf_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
*/
int
adf_dev_init(struct adf_accel_dev *accel_dev)
{
int ret = 0;
mutex_lock(&accel_dev->lock);
ret = adf_dev_init_locked(accel_dev);
mutex_unlock(&accel_dev->lock);
return ret;
}
static int
adf_dev_init_locked(struct adf_accel_dev *accel_dev)
{
struct service_hndl *service;
struct list_head *list_itr;
@ -410,6 +431,18 @@ adf_dev_init(struct adf_accel_dev *accel_dev)
*/
int
adf_dev_start(struct adf_accel_dev *accel_dev)
{
int ret = 0;
mutex_lock(&accel_dev->lock);
ret = adf_dev_start_locked(accel_dev);
mutex_unlock(&accel_dev->lock);
return ret;
}
static int
adf_dev_start_locked(struct adf_accel_dev *accel_dev)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct service_hndl *service;
@ -507,6 +540,18 @@ adf_dev_start(struct adf_accel_dev *accel_dev)
*/
int
adf_dev_stop(struct adf_accel_dev *accel_dev)
{
int ret = 0;
mutex_lock(&accel_dev->lock);
ret = adf_dev_stop_locked(accel_dev);
mutex_unlock(&accel_dev->lock);
return ret;
}
static int
adf_dev_stop_locked(struct adf_accel_dev *accel_dev)
{
struct service_hndl *service;
struct list_head *list_itr;
@ -517,6 +562,10 @@ adf_dev_stop(struct adf_accel_dev *accel_dev)
accel_dev->accel_id);
return ENODEV;
}
if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status))
return 0;
if (!adf_dev_started(accel_dev) &&
!test_bit(ADF_STATUS_STARTING, &accel_dev->status)) {
return 0;
@ -570,11 +619,22 @@ adf_dev_stop(struct adf_accel_dev *accel_dev)
*/
void
adf_dev_shutdown(struct adf_accel_dev *accel_dev)
{
mutex_lock(&accel_dev->lock);
adf_dev_shutdown_locked(accel_dev);
mutex_unlock(&accel_dev->lock);
}
static void
adf_dev_shutdown_locked(struct adf_accel_dev *accel_dev)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct service_hndl *service;
struct list_head *list_itr;
if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status))
return;
if (test_bit(ADF_STATUS_SYSCTL_CTX_INITIALISED, &accel_dev->status)) {
sysctl_ctx_free(&accel_dev->sysctl_ctx);
clear_bit(ADF_STATUS_SYSCTL_CTX_INITIALISED,
@ -623,8 +683,12 @@ adf_dev_shutdown(struct adf_accel_dev *accel_dev)
}
/* Delete configuration only if not restarting */
if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) {
adf_cfg_del_all(accel_dev);
#ifdef QAT_UIO
adf_cfg_device_clear_all(accel_dev);
#endif
}
if (hw_data->remove_pke_stats)
hw_data->remove_pke_stats(accel_dev);

20
sys/dev/qat/qat_common/adf_pfvf_vf_msg.c
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#include <linux/bitfield.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"
@ -98,6 +98,22 @@ adf_vf2pf_request_version(struct adf_accel_dev *accel_dev)
return 0;
}
void
adf_vf2pf_restarting_complete(struct adf_accel_dev *accel_dev)
{
struct pfvf_message msg = { .type =
ADF_VF2PF_MSGTYPE_RESTARTING_COMPLETE };
if (accel_dev->u1.vf.pf_compat_ver < ADF_PFVF_COMPAT_FALLBACK)
return;
if (adf_send_vf2pf_msg(accel_dev, msg)) {
device_printf(
GET_DEV(accel_dev),
"Failed to send Restarting complete event to PF\n");
}
}
int
adf_vf2pf_get_capabilities(struct adf_accel_dev *accel_dev)
{
@ -180,5 +196,7 @@ adf_vf2pf_get_ring_to_svc(struct adf_accel_dev *accel_dev)
/* Only v1 at present */
accel_dev->hw_device->ring_to_svc_map = rts_map_msg.map;
accel_dev->hw_device->get_ring_to_svc_done = true;
return 0;
}

17
sys/dev/qat/qat_common/adf_pfvf_vf_proto.c
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#include <linux/kernel.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"
@ -89,9 +89,15 @@ adf_send_vf2pf_req(struct adf_accel_dev *accel_dev,
}
/* Wait for response, if it times out retry */
ret =
wait_for_completion_timeout(&accel_dev->u1.vf.msg_received,
timeout);
if (!cold) {
ret = wait_for_completion_timeout(
&accel_dev->u1.vf.msg_received, timeout);
} else {
/* In cold start timers may not be initialized yet */
DELAY(ADF_PFVF_MSG_RESP_TIMEOUT * 1000);
ret = try_wait_for_completion(
&accel_dev->u1.vf.msg_received);
}
if (ret) {
if (likely(resp))
*resp = accel_dev->u1.vf.response;
@ -346,6 +352,9 @@ adf_handle_pf2vf_msg(struct adf_accel_dev *accel_dev, struct pfvf_message msg)
case ADF_PF2VF_MSGTYPE_RP_RESET_RESP:
adf_pf2vf_handle_pf_rp_reset(accel_dev, msg);
return true;
case ADF_PF2VF_MSGTYPE_FATAL_ERROR:
adf_pf2vf_handle_pf_error(accel_dev);
return true;
case ADF_PF2VF_MSGTYPE_VERSION_RESP:
case ADF_PF2VF_MSGTYPE_BLKMSG_RESP:
accel_dev->u1.vf.response = msg;

15
sys/dev/qat/qat_common/adf_vf_isr.c
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#include "qat_freebsd.h"
#include <sys/kernel.h>
#include <sys/systm.h>
@ -17,6 +17,7 @@
#include "adf_transport_access_macros.h"
#include "adf_transport_internal.h"
#include "adf_pfvf_utils.h"
#include "adf_pfvf_vf_msg.h"
static TASKQUEUE_DEFINE_THREAD(qat_vf);
static TASKQUEUE_DEFINE_THREAD(qat_bank_handler);
@ -65,6 +66,7 @@ adf_dev_stop_async(struct work_struct *work)
/* Re-enable PF2VF interrupts */
hw_data->enable_pf2vf_interrupt(accel_dev);
adf_vf2pf_restarting_complete(accel_dev);
kfree(stop_data);
}
@ -123,6 +125,17 @@ adf_pf2vf_handle_pf_rp_reset(struct adf_accel_dev *accel_dev,
return 0;
}
int
adf_pf2vf_handle_pf_error(struct adf_accel_dev *accel_dev)
{
device_printf(GET_DEV(accel_dev), "Fatal error received from PF\n");
if (adf_notify_fatal_error(accel_dev))
device_printf(GET_DEV(accel_dev), "Couldn't notify fatal error\n");
return 0;
}
static void
adf_pf2vf_bh_handler(void *data, int pending)
{

17
sys/dev/qat/qat_common/qat_hal.c
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#include "qat_freebsd.h"
#include "adf_cfg.h"
#include "adf_common_drv.h"
@ -1052,8 +1052,7 @@ qat_hal_init(struct adf_accel_dev *accel_dev)
handle->hal_cap_ae_xfer_csr_addr_v = ae_offset;
handle->hal_ep_csr_addr_v = ep_offset;
handle->hal_cap_ae_local_csr_addr_v =
((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v +
LOCAL_TO_XFER_REG_OFFSET);
((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET);
handle->fw_auth = (pci_get_device(GET_DEV(handle->accel_dev)) ==
ADF_DH895XCC_PCI_DEVICE_ID) ?
false :
@ -1283,7 +1282,7 @@ qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
unsigned int max_cycle,
unsigned int *endpc)
{
uint64_t savuwords[MAX_EXEC_INST];
u64 *savuwords = NULL;
unsigned int ind_lm_addr0, ind_lm_addr1;
unsigned int ind_lm_addr2, ind_lm_addr3;
unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1;
@ -1300,6 +1299,11 @@ qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
pr_err("QAT: invalid instruction num %d\n", inst_num);
return EINVAL;
}
savuwords = kzalloc(sizeof(u64) * MAX_EXEC_INST, GFP_KERNEL);
if (!savuwords)
return ENOMEM;
/* save current context */
qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0);
qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1);
@ -1360,8 +1364,10 @@ qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
qat_hal_enable_ctx(handle, ae, (1 << ctx));
/* wait for micro codes to finish */
if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0) {
kfree(savuwords);
return EFAULT;
}
if (endpc) {
unsigned int ctx_status;
@ -1429,6 +1435,7 @@ qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, ind_sig);
qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
kfree(savuwords);
return 0;
}

12
sys/dev/qat/qat_common/qat_uclo.c
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#include "qat_freebsd.h"
#include "adf_cfg.h"
#include "adf_common_drv.h"
@ -1618,6 +1618,11 @@ qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle,
unsigned int length, simg_offset = sizeof(*auth_chunk);
unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev));
if (size <= ICP_QAT_AE_IMG_OFFSET(device_id)) {
pr_err("QAT: error, input image size too small %d\n", size);
return EINVAL;
}
if (size >
(ICP_QAT_AE_IMG_OFFSET(device_id) + ICP_QAT_CSS_MAX_IMAGE_LEN)) {
pr_err("QAT: error, input image size overflow %d\n", size);
@ -1825,11 +1830,6 @@ qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle,
GET_DEV(handle->accel_dev)));
return status;
}
if (pci_get_device(GET_DEV(handle->accel_dev)) ==
ADF_C3XXX_PCI_DEVICE_ID) {
pr_err("QAT: C3XXX doesn't support unsigned MMP\n");
return EINVAL;
}
status = qat_uclo_wr_sram_by_words(handle,
handle->hal_sram_offset,
addr_ptr,

6
sys/dev/qat/qat_hw/qat_200xx/adf_200xx_hw_data.h
View file

@ -1,11 +1,11 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#ifndef ADF_200XX_HW_DATA_H_
#define ADF_200XX_HW_DATA_H_
/* PCIe configuration space */
#define ADF_200XX_PMISC_BAR 0
#define ADF_200XX_ETR_BAR 1
#define ADF_200XX_PMISC_BAR 1
#define ADF_200XX_ETR_BAR 2
#define ADF_200XX_RX_RINGS_OFFSET 8
#define ADF_200XX_TX_RINGS_MASK 0xFF
#define ADF_200XX_MAX_ACCELERATORS 3

3
sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c
View file

@ -1013,6 +1013,9 @@ adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 id)
hw_data->get_hb_clock = get_hb_clock;
hw_data->int_timer_init = adf_int_timer_init;
hw_data->int_timer_exit = adf_int_timer_exit;
hw_data->pre_reset = adf_dev_pre_reset;
hw_data->post_reset = adf_dev_post_reset;
hw_data->disable_arb = adf_disable_arb;
hw_data->get_heartbeat_status = adf_get_heartbeat_status;
hw_data->get_ae_clock = get_ae_clock;
hw_data->measure_clock = measure_clock;

5
sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c
View file

@ -192,11 +192,6 @@ adf_attach(device_t dev)
adf_init_hw_data_4xxx(accel_dev->hw_device, pci_get_device(dev));
accel_pci_dev->revid = pci_get_revid(dev);
hw_data->fuses = pci_read_config(dev, ADF_4XXX_FUSECTL4_OFFSET, 4);
if (accel_pci_dev->revid == 0x00) {
device_printf(dev, "A0 stepping is not supported.\n");
ret = ENODEV;
goto out_err;
}
/* Get PPAERUCM values and store */
ret = adf_aer_store_ppaerucm_reg(dev, hw_data);

3
sys/dev/qat/qat_hw/qat_4xxxvf/adf_4xxxvf_hw_data.c
View file

@ -294,6 +294,9 @@ get_ring_to_svc_map(struct adf_accel_dev *accel_dev, u16 *ring_to_svc_map)
char val[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
u32 i = 0;
if (accel_dev->hw_device->get_ring_to_svc_done)
return 0;
/* Get the services enabled by user if provided.
* The function itself will also be called during the driver probe
* procedure where no ServicesEnable is provided. Then the device

6
sys/dev/qat/qat_hw/qat_c3xxx/adf_c3xxx_hw_data.h
View file

@ -1,11 +1,11 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#ifndef ADF_C3XXX_HW_DATA_H_
#define ADF_C3XXX_HW_DATA_H_
/* PCIe configuration space */
#define ADF_C3XXX_PMISC_BAR 0
#define ADF_C3XXX_ETR_BAR 1
#define ADF_C3XXX_PMISC_BAR 1
#define ADF_C3XXX_ETR_BAR 2
#define ADF_C3XXX_RX_RINGS_OFFSET 8
#define ADF_C3XXX_TX_RINGS_MASK 0xFF
#define ADF_C3XXX_MAX_ACCELERATORS 3

6
sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_hw_data.c
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/* Copyright(c) 2007-2025 Intel Corporation */
#include <linux/atomic.h>
#include <linux/compiler.h>
#include <adf_accel_devices.h>
@ -858,10 +858,10 @@ update_hw_capability(struct adf_accel_dev *accel_dev)
if (!au_info->asym_ae_msk)
disabled_caps = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
ICP_ACCEL_CAPABILITIES_ECEDMONT;
if (!au_info->sym_ae_msk)
disabled_caps |= ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
ICP_ACCEL_CAPABILITIES_CIPHER | ICP_ACCEL_CAPABILITIES_ZUC |
ICP_ACCEL_CAPABILITIES_SHA3_EXT |
ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 |