1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 /** 5 ***************************************************************************** 6 * @file dc_dp.c 7 * 8 * @defgroup cpaDcDp Data Compression Data Plane API 9 * 10 * @ingroup cpaDcDp 11 * 12 * @description 13 * Implementation of the Data Compression DP operations. 14 * 15 *****************************************************************************/ 16 17 /* 18 ******************************************************************************* 19 * Include public/global header files 20 ******************************************************************************* 21 */ 22 #include "cpa.h" 23 #include "cpa_dc.h" 24 #include "cpa_dc_dp.h" 25 26 #include "icp_qat_fw_comp.h" 27 28 /* 29 ******************************************************************************* 30 * Include private header files 31 ******************************************************************************* 32 */ 33 #include "dc_session.h" 34 #include "dc_datapath.h" 35 #include "lac_common.h" 36 #include "lac_mem.h" 37 #include "lac_mem_pools.h" 38 #include "sal_types_compression.h" 39 #include "lac_sal.h" 40 #include "lac_sync.h" 41 #include "sal_service_state.h" 42 #include "sal_qat_cmn_msg.h" 43 #include "icp_sal_poll.h" 44 #include "sal_hw_gen.h" 45 46 /** 47 ***************************************************************************** 48 * @ingroup cpaDcDp 49 * Check that pOpData is valid 50 * 51 * @description 52 * Check that all the parameters defined in the pOpData are valid 53 * 54 * @param[in] pOpData Pointer to a structure containing the 55 * request parameters 56 * 57 * @retval CPA_STATUS_SUCCESS Function executed successfully 58 * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in 59 * 60 *****************************************************************************/ 61 static CpaStatus 62 dcDataPlaneParamCheck(const CpaDcDpOpData *pOpData) 63 { 64 sal_compression_service_t *pService = NULL; 65 dc_session_desc_t *pSessionDesc = NULL; 66 67 LAC_CHECK_NULL_PARAM(pOpData); 68 LAC_CHECK_NULL_PARAM(pOpData->dcInstance); 69 LAC_CHECK_NULL_PARAM(pOpData->pSessionHandle); 70 71 /* Ensure this is a compression instance */ 72 SAL_CHECK_INSTANCE_TYPE(pOpData->dcInstance, 73 SAL_SERVICE_TYPE_COMPRESSION); 74 75 pService = (sal_compression_service_t *)(pOpData->dcInstance); 76 77 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 78 if (NULL == pSessionDesc) { 79 QAT_UTILS_LOG("Session handle not as expected.\n"); 80 return CPA_STATUS_INVALID_PARAM; 81 } 82 83 if (CPA_FALSE == pSessionDesc->isDcDp) { 84 QAT_UTILS_LOG("The session type should be data plane.\n"); 85 return CPA_STATUS_INVALID_PARAM; 86 } 87 88 /* Compressing zero byte is not supported */ 89 if ((CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) && 90 (0 == pOpData->bufferLenToCompress)) { 91 QAT_UTILS_LOG("The source buffer length to compress needs to " 92 "be greater than zero byte.\n"); 93 return CPA_STATUS_INVALID_PARAM; 94 } 95 96 if (pOpData->sessDirection > CPA_DC_DIR_DECOMPRESS) { 97 QAT_UTILS_LOG("Invalid direction of operation.\n"); 98 return CPA_STATUS_INVALID_PARAM; 99 } 100 101 if (0 == pOpData->srcBuffer) { 102 QAT_UTILS_LOG("Invalid srcBuffer\n"); 103 return CPA_STATUS_INVALID_PARAM; 104 } 105 if (0 == pOpData->destBuffer) { 106 QAT_UTILS_LOG("Invalid destBuffer\n"); 107 return CPA_STATUS_INVALID_PARAM; 108 } 109 if (pOpData->srcBuffer == pOpData->destBuffer) { 110 QAT_UTILS_LOG("In place operation is not supported.\n"); 111 return CPA_STATUS_INVALID_PARAM; 112 } 113 if (0 == pOpData->thisPhys) { 114 QAT_UTILS_LOG("Invalid thisPhys\n"); 115 return CPA_STATUS_INVALID_PARAM; 116 } 117 118 if ((CPA_TRUE != pOpData->compressAndVerify) && 119 (CPA_FALSE != pOpData->compressAndVerify)) { 120 QAT_UTILS_LOG("Invalid compressAndVerify\n"); 121 return CPA_STATUS_INVALID_PARAM; 122 } 123 if ((CPA_TRUE == pOpData->compressAndVerify) && 124 !(pService->generic_service_info.dcExtendedFeatures & 125 DC_CNV_EXTENDED_CAPABILITY)) { 126 QAT_UTILS_LOG("Invalid compressAndVerify, no CNV capability\n"); 127 return CPA_STATUS_UNSUPPORTED; 128 } 129 if ((CPA_TRUE != pOpData->compressAndVerifyAndRecover) && 130 (CPA_FALSE != pOpData->compressAndVerifyAndRecover)) { 131 QAT_UTILS_LOG("Invalid compressAndVerifyAndRecover\n"); 132 return CPA_STATUS_INVALID_PARAM; 133 } 134 if ((CPA_TRUE == pOpData->compressAndVerifyAndRecover) && 135 (CPA_FALSE == pOpData->compressAndVerify)) { 136 QAT_UTILS_LOG("CnVnR option set without setting CnV\n"); 137 return CPA_STATUS_INVALID_PARAM; 138 } 139 if ((CPA_TRUE == pOpData->compressAndVerifyAndRecover) && 140 !(pService->generic_service_info.dcExtendedFeatures & 141 DC_CNVNR_EXTENDED_CAPABILITY)) { 142 QAT_UTILS_LOG( 143 "Invalid CnVnR option set and no CnVnR capability.\n"); 144 return CPA_STATUS_UNSUPPORTED; 145 } 146 147 if ((CPA_DP_BUFLIST == pOpData->srcBufferLen) && 148 (CPA_DP_BUFLIST != pOpData->destBufferLen)) { 149 QAT_UTILS_LOG( 150 "The source and destination buffers need to be of the same type (both flat buffers or buffer lists).\n"); 151 return CPA_STATUS_INVALID_PARAM; 152 } 153 if ((CPA_DP_BUFLIST != pOpData->srcBufferLen) && 154 (CPA_DP_BUFLIST == pOpData->destBufferLen)) { 155 QAT_UTILS_LOG( 156 "The source and destination buffers need to be of the same type (both flat buffers or buffer lists).\n"); 157 return CPA_STATUS_INVALID_PARAM; 158 } 159 160 if (CPA_DP_BUFLIST != pOpData->srcBufferLen) { 161 if (pOpData->srcBufferLen < pOpData->bufferLenToCompress) { 162 QAT_UTILS_LOG( 163 "srcBufferLen is smaller than bufferLenToCompress.\n"); 164 return CPA_STATUS_INVALID_PARAM; 165 } 166 167 if (pOpData->destBufferLen < pOpData->bufferLenForData) { 168 QAT_UTILS_LOG( 169 "destBufferLen is smaller than bufferLenForData.\n"); 170 return CPA_STATUS_INVALID_PARAM; 171 } 172 } else { 173 /* We are assuming that there is enough memory in the source and 174 * destination buffer lists. We only receive physical addresses 175 * of the buffers so we are unable to test it here */ 176 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->srcBuffer); 177 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->destBuffer); 178 } 179 180 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->thisPhys); 181 182 if ((CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) || 183 (CPA_DC_DIR_COMBINED == pSessionDesc->sessDirection)) { 184 if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) { 185 /* Check if Intermediate Buffer Array pointer is NULL */ 186 if (isDcGen2x(pService) && 187 ((0 == pService->pInterBuffPtrsArrayPhyAddr) || 188 (NULL == pService->pInterBuffPtrsArray))) { 189 QAT_UTILS_LOG( 190 "No intermediate buffer defined for this instance - see cpaDcStartInstance.\n"); 191 return CPA_STATUS_INVALID_PARAM; 192 } 193 194 /* Ensure that the destination buffer length for data is 195 * greater 196 * or equal to 128B */ 197 if (pOpData->bufferLenForData < 198 DC_DEST_BUFFER_DYN_MIN_SIZE) { 199 QAT_UTILS_LOG( 200 "Destination buffer length for data should be greater or equal to 128B.\n"); 201 return CPA_STATUS_INVALID_PARAM; 202 } 203 } else { 204 /* Ensure that the destination buffer length for data is 205 * greater 206 * or equal to min output buffsize */ 207 if (pOpData->bufferLenForData < 208 pService->comp_device_data.minOutputBuffSize) { 209 QAT_UTILS_LOG( 210 "Destination buffer size should be greater or equal to %d bytes.\n", 211 pService->comp_device_data 212 .minOutputBuffSize); 213 return CPA_STATUS_INVALID_PARAM; 214 } 215 } 216 } 217 218 return CPA_STATUS_SUCCESS; 219 } 220 221 CpaStatus 222 cpaDcDpGetSessionSize(CpaInstanceHandle dcInstance, 223 CpaDcSessionSetupData *pSessionData, 224 Cpa32U *pSessionSize) 225 { 226 return dcGetSessionSize(dcInstance, pSessionData, pSessionSize, NULL); 227 } 228 229 CpaStatus 230 cpaDcDpInitSession(CpaInstanceHandle dcInstance, 231 CpaDcSessionHandle pSessionHandle, 232 CpaDcSessionSetupData *pSessionData) 233 { 234 CpaStatus status = CPA_STATUS_SUCCESS; 235 dc_session_desc_t *pSessionDesc = NULL; 236 sal_compression_service_t *pService = NULL; 237 238 LAC_CHECK_INSTANCE_HANDLE(dcInstance); 239 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 240 241 pService = (sal_compression_service_t *)dcInstance; 242 243 /* Check if SAL is initialised otherwise return an error */ 244 SAL_RUNNING_CHECK(pService); 245 246 /* Stateful is not supported */ 247 if (CPA_DC_STATELESS != pSessionData->sessState) { 248 QAT_UTILS_LOG("Invalid sessState value\n"); 249 return CPA_STATUS_INVALID_PARAM; 250 } 251 252 status = 253 dcInitSession(dcInstance, pSessionHandle, pSessionData, NULL, NULL); 254 if (CPA_STATUS_SUCCESS == status) { 255 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); 256 pSessionDesc->isDcDp = CPA_TRUE; 257 258 ICP_QAT_FW_COMN_PTR_TYPE_SET( 259 pSessionDesc->reqCacheDecomp.comn_hdr.comn_req_flags, 260 DC_DP_QAT_PTR_TYPE); 261 ICP_QAT_FW_COMN_PTR_TYPE_SET( 262 pSessionDesc->reqCacheComp.comn_hdr.comn_req_flags, 263 DC_DP_QAT_PTR_TYPE); 264 } 265 266 return status; 267 } 268 269 CpaStatus 270 cpaDcDpRemoveSession(const CpaInstanceHandle dcInstance, 271 CpaDcSessionHandle pSessionHandle) 272 { 273 return cpaDcRemoveSession(dcInstance, pSessionHandle); 274 } 275 276 CpaStatus 277 cpaDcDpRegCbFunc(const CpaInstanceHandle dcInstance, 278 const CpaDcDpCallbackFn pNewCb) 279 { 280 sal_compression_service_t *pService = NULL; 281 282 LAC_CHECK_NULL_PARAM(dcInstance); 283 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 284 LAC_CHECK_NULL_PARAM(pNewCb); 285 286 /* Check if SAL is initialised otherwise return an error */ 287 SAL_RUNNING_CHECK(dcInstance); 288 289 pService = (sal_compression_service_t *)dcInstance; 290 pService->pDcDpCb = pNewCb; 291 292 return CPA_STATUS_SUCCESS; 293 } 294 295 /** 296 ***************************************************************************** 297 * @ingroup cpaDcDp 298 * 299 * @description 300 * Writes the message to the ring 301 * 302 * @param[in] pOpData Pointer to a structure containing the 303 * request parameters 304 * @param[in] pCurrentQatMsg Pointer to current QAT message on the ring 305 * 306 *****************************************************************************/ 307 static void 308 dcDpWriteRingMsg(CpaDcDpOpData *pOpData, icp_qat_fw_comp_req_t *pCurrentQatMsg) 309 { 310 icp_qat_fw_comp_req_t *pReqCache = NULL; 311 dc_session_desc_t *pSessionDesc = NULL; 312 Cpa8U bufferFormat; 313 314 Cpa8U cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV; 315 Cpa8U cnvnrCompReq = ICP_QAT_FW_COMP_NO_CNV_RECOVERY; 316 CpaBoolean cnvErrorInjection = ICP_QAT_FW_COMP_NO_CNV_DFX; 317 sal_compression_service_t *pService = NULL; 318 319 pService = (sal_compression_service_t *)(pOpData->dcInstance); 320 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 321 322 if (CPA_DC_DIR_COMPRESS == pOpData->sessDirection) { 323 pReqCache = &(pSessionDesc->reqCacheComp); 324 /* CNV check */ 325 if (CPA_TRUE == pOpData->compressAndVerify) { 326 cnvDecompReq = ICP_QAT_FW_COMP_CNV; 327 if (isDcGen4x(pService)) { 328 cnvErrorInjection = 329 pSessionDesc->cnvErrorInjection; 330 } 331 332 /* CNVNR check */ 333 if (CPA_TRUE == pOpData->compressAndVerifyAndRecover) { 334 cnvnrCompReq = ICP_QAT_FW_COMP_CNV_RECOVERY; 335 } 336 } 337 } else { 338 pReqCache = &(pSessionDesc->reqCacheDecomp); 339 } 340 341 /* Fills in the template DC ET ring message - cached from the 342 * session descriptor */ 343 memcpy((void *)pCurrentQatMsg, 344 (void *)(pReqCache), 345 (LAC_QAT_DC_REQ_SZ_LW * LAC_LONG_WORD_IN_BYTES)); 346 347 if (CPA_DP_BUFLIST == pOpData->srcBufferLen) { 348 bufferFormat = QAT_COMN_PTR_TYPE_SGL; 349 } else { 350 bufferFormat = QAT_COMN_PTR_TYPE_FLAT; 351 } 352 353 pCurrentQatMsg->comp_pars.req_par_flags |= 354 ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD( 355 ICP_QAT_FW_COMP_NOT_SOP, 356 ICP_QAT_FW_COMP_NOT_EOP, 357 ICP_QAT_FW_COMP_NOT_BFINAL, 358 cnvDecompReq, 359 cnvnrCompReq, 360 cnvErrorInjection, 361 ICP_QAT_FW_COMP_CRC_MODE_LEGACY); 362 363 SalQatMsg_CmnMidWrite((icp_qat_fw_la_bulk_req_t *)pCurrentQatMsg, 364 pOpData, 365 bufferFormat, 366 pOpData->srcBuffer, 367 pOpData->destBuffer, 368 pOpData->srcBufferLen, 369 pOpData->destBufferLen); 370 371 pCurrentQatMsg->comp_pars.comp_len = pOpData->bufferLenToCompress; 372 pCurrentQatMsg->comp_pars.out_buffer_sz = pOpData->bufferLenForData; 373 } 374 375 CpaStatus 376 cpaDcDpEnqueueOp(CpaDcDpOpData *pOpData, const CpaBoolean performOpNow) 377 { 378 icp_qat_fw_comp_req_t *pCurrentQatMsg = NULL; 379 icp_comms_trans_handle trans_handle = NULL; 380 dc_session_desc_t *pSessionDesc = NULL; 381 CpaStatus status = CPA_STATUS_SUCCESS; 382 383 status = dcDataPlaneParamCheck(pOpData); 384 if (CPA_STATUS_SUCCESS != status) { 385 return status; 386 } 387 388 if ((CPA_FALSE == pOpData->compressAndVerify) && 389 (CPA_DC_DIR_COMPRESS == pOpData->sessDirection)) { 390 return CPA_STATUS_UNSUPPORTED; 391 } 392 393 /* Check if SAL is initialised otherwise return an error */ 394 SAL_RUNNING_CHECK(pOpData->dcInstance); 395 396 trans_handle = ((sal_compression_service_t *)pOpData->dcInstance) 397 ->trans_handle_compression_tx; 398 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 399 400 if ((CPA_DC_DIR_COMPRESS == pOpData->sessDirection) && 401 (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection)) { 402 QAT_UTILS_LOG( 403 "The session does not support this direction of operation.\n"); 404 return CPA_STATUS_INVALID_PARAM; 405 } else if ((CPA_DC_DIR_DECOMPRESS == pOpData->sessDirection) && 406 (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection)) { 407 QAT_UTILS_LOG( 408 "The session does not support this direction of operation.\n"); 409 return CPA_STATUS_INVALID_PARAM; 410 } 411 412 icp_adf_getSingleQueueAddr(trans_handle, (void **)&pCurrentQatMsg); 413 if (NULL == pCurrentQatMsg) { 414 return CPA_STATUS_RETRY; 415 } 416 417 dcDpWriteRingMsg(pOpData, pCurrentQatMsg); 418 pSessionDesc->pendingDpStatelessCbCount++; 419 420 if (CPA_TRUE == performOpNow) { 421 SalQatMsg_updateQueueTail(trans_handle); 422 } 423 424 return CPA_STATUS_SUCCESS; 425 } 426 427 CpaStatus 428 cpaDcDpEnqueueOpBatch(const Cpa32U numberRequests, 429 CpaDcDpOpData *pOpData[], 430 const CpaBoolean performOpNow) 431 { 432 icp_qat_fw_comp_req_t *pCurrentQatMsg = NULL; 433 icp_comms_trans_handle trans_handle = NULL; 434 dc_session_desc_t *pSessionDesc = NULL; 435 Cpa32U i = 0; 436 CpaStatus status = CPA_STATUS_SUCCESS; 437 sal_compression_service_t *pService = NULL; 438 439 LAC_CHECK_NULL_PARAM(pOpData); 440 LAC_CHECK_NULL_PARAM(pOpData[0]); 441 LAC_CHECK_NULL_PARAM(pOpData[0]->dcInstance); 442 443 pService = (sal_compression_service_t *)(pOpData[0]->dcInstance); 444 if ((numberRequests == 0) || 445 (numberRequests > pService->maxNumCompConcurrentReq)) { 446 QAT_UTILS_LOG( 447 "The number of requests needs to be between 1 and %d.\n", 448 pService->maxNumCompConcurrentReq); 449 return CPA_STATUS_INVALID_PARAM; 450 } 451 452 for (i = 0; i < numberRequests; i++) { 453 status = dcDataPlaneParamCheck(pOpData[i]); 454 if (CPA_STATUS_SUCCESS != status) { 455 return status; 456 } 457 458 /* Check that all instance handles and session handles are the 459 * same */ 460 if (pOpData[i]->dcInstance != pOpData[0]->dcInstance) { 461 QAT_UTILS_LOG( 462 "All instance handles should be the same in the pOpData.\n"); 463 return CPA_STATUS_INVALID_PARAM; 464 } 465 466 if (pOpData[i]->pSessionHandle != pOpData[0]->pSessionHandle) { 467 QAT_UTILS_LOG( 468 "All session handles should be the same in the pOpData.\n"); 469 return CPA_STATUS_INVALID_PARAM; 470 } 471 } 472 473 for (i = 0; i < numberRequests; i++) { 474 if ((CPA_FALSE == pOpData[i]->compressAndVerify) && 475 (CPA_DC_DIR_COMPRESS == pOpData[i]->sessDirection)) { 476 return CPA_STATUS_UNSUPPORTED; 477 } 478 } 479 480 /* Check if SAL is initialised otherwise return an error */ 481 SAL_RUNNING_CHECK(pOpData[0]->dcInstance); 482 483 trans_handle = ((sal_compression_service_t *)pOpData[0]->dcInstance) 484 ->trans_handle_compression_tx; 485 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData[0]->pSessionHandle); 486 487 for (i = 0; i < numberRequests; i++) { 488 if ((CPA_DC_DIR_COMPRESS == pOpData[i]->sessDirection) && 489 (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection)) { 490 QAT_UTILS_LOG( 491 "The session does not support this direction of operation.\n"); 492 return CPA_STATUS_INVALID_PARAM; 493 } else if ((CPA_DC_DIR_DECOMPRESS == 494 pOpData[i]->sessDirection) && 495 (CPA_DC_DIR_COMPRESS == 496 pSessionDesc->sessDirection)) { 497 QAT_UTILS_LOG( 498 "The session does not support this direction of operation.\n"); 499 return CPA_STATUS_INVALID_PARAM; 500 } 501 } 502 503 icp_adf_getQueueMemory(trans_handle, 504 numberRequests, 505 (void **)&pCurrentQatMsg); 506 if (NULL == pCurrentQatMsg) { 507 return CPA_STATUS_RETRY; 508 } 509 510 for (i = 0; i < numberRequests; i++) { 511 dcDpWriteRingMsg(pOpData[i], pCurrentQatMsg); 512 icp_adf_getQueueNext(trans_handle, (void **)&pCurrentQatMsg); 513 } 514 515 pSessionDesc->pendingDpStatelessCbCount += numberRequests; 516 517 if (CPA_TRUE == performOpNow) { 518 SalQatMsg_updateQueueTail(trans_handle); 519 } 520 521 return CPA_STATUS_SUCCESS; 522 } 523 524 CpaStatus 525 icp_sal_DcPollDpInstance(CpaInstanceHandle dcInstance, Cpa32U responseQuota) 526 { 527 icp_comms_trans_handle trans_handle = NULL; 528 529 LAC_CHECK_INSTANCE_HANDLE(dcInstance); 530 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 531 532 /* Check if SAL is initialised otherwise return an error */ 533 SAL_RUNNING_CHECK(dcInstance); 534 535 trans_handle = ((sal_compression_service_t *)dcInstance) 536 ->trans_handle_compression_rx; 537 538 return icp_adf_pollQueue(trans_handle, responseQuota); 539 } 540 541 CpaStatus 542 cpaDcDpPerformOpNow(CpaInstanceHandle dcInstance) 543 { 544 icp_comms_trans_handle trans_handle = NULL; 545 546 LAC_CHECK_NULL_PARAM(dcInstance); 547 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 548 549 /* Check if SAL is initialised otherwise return an error */ 550 SAL_RUNNING_CHECK(dcInstance); 551 552 trans_handle = ((sal_compression_service_t *)dcInstance) 553 ->trans_handle_compression_tx; 554 555 if (CPA_TRUE == icp_adf_queueDataToSend(trans_handle)) { 556 SalQatMsg_updateQueueTail(trans_handle); 557 } 558 559 return CPA_STATUS_SUCCESS; 560 } 561