1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /** 4 ***************************************************************************** 5 * @file dc_dp.c 6 * 7 * @defgroup cpaDcDp Data Compression Data Plane API 8 * 9 * @ingroup cpaDcDp 10 * 11 * @description 12 * Implementation of the Data Compression DP operations. 13 * 14 *****************************************************************************/ 15 16 /* 17 ******************************************************************************* 18 * Include public/global header files 19 ******************************************************************************* 20 */ 21 #include "cpa.h" 22 #include "cpa_dc.h" 23 #include "cpa_dc_dp.h" 24 25 #include "icp_qat_fw_comp.h" 26 27 /* 28 ******************************************************************************* 29 * Include private header files 30 ******************************************************************************* 31 */ 32 #include "dc_session.h" 33 #include "dc_datapath.h" 34 #include "lac_common.h" 35 #include "lac_mem.h" 36 #include "lac_mem_pools.h" 37 #include "sal_types_compression.h" 38 #include "lac_sal.h" 39 #include "lac_sync.h" 40 #include "sal_service_state.h" 41 #include "sal_qat_cmn_msg.h" 42 #include "icp_sal_poll.h" 43 #include "sal_hw_gen.h" 44 45 /** 46 ***************************************************************************** 47 * @ingroup cpaDcDp 48 * Check that pOpData is valid 49 * 50 * @description 51 * Check that all the parameters defined in the pOpData are valid 52 * 53 * @param[in] pOpData Pointer to a structure containing the 54 * request parameters 55 * 56 * @retval CPA_STATUS_SUCCESS Function executed successfully 57 * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in 58 * 59 *****************************************************************************/ 60 static CpaStatus 61 dcDataPlaneParamCheck(const CpaDcDpOpData *pOpData) 62 { 63 sal_compression_service_t *pService = NULL; 64 dc_session_desc_t *pSessionDesc = NULL; 65 66 LAC_CHECK_NULL_PARAM(pOpData); 67 LAC_CHECK_NULL_PARAM(pOpData->dcInstance); 68 LAC_CHECK_NULL_PARAM(pOpData->pSessionHandle); 69 70 /* Ensure this is a compression instance */ 71 SAL_CHECK_INSTANCE_TYPE(pOpData->dcInstance, 72 SAL_SERVICE_TYPE_COMPRESSION); 73 74 pService = (sal_compression_service_t *)(pOpData->dcInstance); 75 76 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 77 if (NULL == pSessionDesc) { 78 QAT_UTILS_LOG("Session handle not as expected.\n"); 79 return CPA_STATUS_INVALID_PARAM; 80 } 81 82 if (CPA_FALSE == pSessionDesc->isDcDp) { 83 QAT_UTILS_LOG("The session type should be data plane.\n"); 84 return CPA_STATUS_INVALID_PARAM; 85 } 86 87 /* Compressing zero byte is not supported */ 88 if ((CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) && 89 (0 == pOpData->bufferLenToCompress)) { 90 QAT_UTILS_LOG("The source buffer length to compress needs to " 91 "be greater than zero byte.\n"); 92 return CPA_STATUS_INVALID_PARAM; 93 } 94 95 if (pOpData->sessDirection > CPA_DC_DIR_DECOMPRESS) { 96 QAT_UTILS_LOG("Invalid direction of operation.\n"); 97 return CPA_STATUS_INVALID_PARAM; 98 } 99 100 if (0 == pOpData->srcBuffer) { 101 QAT_UTILS_LOG("Invalid srcBuffer\n"); 102 return CPA_STATUS_INVALID_PARAM; 103 } 104 if (0 == pOpData->destBuffer) { 105 QAT_UTILS_LOG("Invalid destBuffer\n"); 106 return CPA_STATUS_INVALID_PARAM; 107 } 108 if (pOpData->srcBuffer == pOpData->destBuffer) { 109 QAT_UTILS_LOG("In place operation is not supported.\n"); 110 return CPA_STATUS_INVALID_PARAM; 111 } 112 if (0 == pOpData->thisPhys) { 113 QAT_UTILS_LOG("Invalid thisPhys\n"); 114 return CPA_STATUS_INVALID_PARAM; 115 } 116 117 if ((CPA_TRUE != pOpData->compressAndVerify) && 118 (CPA_FALSE != pOpData->compressAndVerify)) { 119 QAT_UTILS_LOG("Invalid compressAndVerify\n"); 120 return CPA_STATUS_INVALID_PARAM; 121 } 122 if ((CPA_TRUE == pOpData->compressAndVerify) && 123 !(pService->generic_service_info.dcExtendedFeatures & 124 DC_CNV_EXTENDED_CAPABILITY)) { 125 QAT_UTILS_LOG("Invalid compressAndVerify, no CNV capability\n"); 126 return CPA_STATUS_UNSUPPORTED; 127 } 128 if ((CPA_TRUE != pOpData->compressAndVerifyAndRecover) && 129 (CPA_FALSE != pOpData->compressAndVerifyAndRecover)) { 130 QAT_UTILS_LOG("Invalid compressAndVerifyAndRecover\n"); 131 return CPA_STATUS_INVALID_PARAM; 132 } 133 if ((CPA_TRUE == pOpData->compressAndVerifyAndRecover) && 134 (CPA_FALSE == pOpData->compressAndVerify)) { 135 QAT_UTILS_LOG("CnVnR option set without setting CnV\n"); 136 return CPA_STATUS_INVALID_PARAM; 137 } 138 if ((CPA_TRUE == pOpData->compressAndVerifyAndRecover) && 139 !(pService->generic_service_info.dcExtendedFeatures & 140 DC_CNVNR_EXTENDED_CAPABILITY)) { 141 QAT_UTILS_LOG( 142 "Invalid CnVnR option set and no CnVnR capability.\n"); 143 return CPA_STATUS_UNSUPPORTED; 144 } 145 146 if ((CPA_DP_BUFLIST == pOpData->srcBufferLen) && 147 (CPA_DP_BUFLIST != pOpData->destBufferLen)) { 148 QAT_UTILS_LOG( 149 "The source and destination buffers need to be of the same type (both flat buffers or buffer lists).\n"); 150 return CPA_STATUS_INVALID_PARAM; 151 } 152 if ((CPA_DP_BUFLIST != pOpData->srcBufferLen) && 153 (CPA_DP_BUFLIST == pOpData->destBufferLen)) { 154 QAT_UTILS_LOG( 155 "The source and destination buffers need to be of the same type (both flat buffers or buffer lists).\n"); 156 return CPA_STATUS_INVALID_PARAM; 157 } 158 159 if (CPA_DP_BUFLIST != pOpData->srcBufferLen) { 160 if (pOpData->srcBufferLen < pOpData->bufferLenToCompress) { 161 QAT_UTILS_LOG( 162 "srcBufferLen is smaller than bufferLenToCompress.\n"); 163 return CPA_STATUS_INVALID_PARAM; 164 } 165 166 if (pOpData->destBufferLen < pOpData->bufferLenForData) { 167 QAT_UTILS_LOG( 168 "destBufferLen is smaller than bufferLenForData.\n"); 169 return CPA_STATUS_INVALID_PARAM; 170 } 171 } else { 172 /* We are assuming that there is enough memory in the source and 173 * destination buffer lists. We only receive physical addresses 174 * of the buffers so we are unable to test it here */ 175 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->srcBuffer); 176 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->destBuffer); 177 } 178 179 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->thisPhys); 180 181 if ((CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) || 182 (CPA_DC_DIR_COMBINED == pSessionDesc->sessDirection)) { 183 if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) { 184 /* Check if Intermediate Buffer Array pointer is NULL */ 185 if (isDcGen2x(pService) && 186 ((0 == pService->pInterBuffPtrsArrayPhyAddr) || 187 (NULL == pService->pInterBuffPtrsArray))) { 188 QAT_UTILS_LOG( 189 "No intermediate buffer defined for this instance - see cpaDcStartInstance.\n"); 190 return CPA_STATUS_INVALID_PARAM; 191 } 192 193 /* Ensure that the destination buffer length for data is 194 * greater 195 * or equal to 128B */ 196 if (pOpData->bufferLenForData < 197 DC_DEST_BUFFER_DYN_MIN_SIZE) { 198 QAT_UTILS_LOG( 199 "Destination buffer length for data should be greater or equal to 128B.\n"); 200 return CPA_STATUS_INVALID_PARAM; 201 } 202 } else { 203 /* Ensure that the destination buffer length for data is 204 * greater 205 * or equal to min output buffsize */ 206 if (pOpData->bufferLenForData < 207 pService->comp_device_data.minOutputBuffSize) { 208 QAT_UTILS_LOG( 209 "Destination buffer size should be greater or equal to %d bytes.\n", 210 pService->comp_device_data 211 .minOutputBuffSize); 212 return CPA_STATUS_INVALID_PARAM; 213 } 214 } 215 } 216 217 return CPA_STATUS_SUCCESS; 218 } 219 220 CpaStatus 221 cpaDcDpGetSessionSize(CpaInstanceHandle dcInstance, 222 CpaDcSessionSetupData *pSessionData, 223 Cpa32U *pSessionSize) 224 { 225 return dcGetSessionSize(dcInstance, pSessionData, pSessionSize, NULL); 226 } 227 228 CpaStatus 229 cpaDcDpInitSession(CpaInstanceHandle dcInstance, 230 CpaDcSessionHandle pSessionHandle, 231 CpaDcSessionSetupData *pSessionData) 232 { 233 CpaStatus status = CPA_STATUS_SUCCESS; 234 dc_session_desc_t *pSessionDesc = NULL; 235 sal_compression_service_t *pService = NULL; 236 237 LAC_CHECK_INSTANCE_HANDLE(dcInstance); 238 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 239 240 pService = (sal_compression_service_t *)dcInstance; 241 242 /* Check if SAL is initialised otherwise return an error */ 243 SAL_RUNNING_CHECK(pService); 244 245 /* Stateful is not supported */ 246 if (CPA_DC_STATELESS != pSessionData->sessState) { 247 QAT_UTILS_LOG("Invalid sessState value\n"); 248 return CPA_STATUS_INVALID_PARAM; 249 } 250 251 status = 252 dcInitSession(dcInstance, pSessionHandle, pSessionData, NULL, NULL); 253 if (CPA_STATUS_SUCCESS == status) { 254 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); 255 pSessionDesc->isDcDp = CPA_TRUE; 256 257 ICP_QAT_FW_COMN_PTR_TYPE_SET( 258 pSessionDesc->reqCacheDecomp.comn_hdr.comn_req_flags, 259 DC_DP_QAT_PTR_TYPE); 260 ICP_QAT_FW_COMN_PTR_TYPE_SET( 261 pSessionDesc->reqCacheComp.comn_hdr.comn_req_flags, 262 DC_DP_QAT_PTR_TYPE); 263 } 264 265 return status; 266 } 267 268 CpaStatus 269 cpaDcDpRemoveSession(const CpaInstanceHandle dcInstance, 270 CpaDcSessionHandle pSessionHandle) 271 { 272 return cpaDcRemoveSession(dcInstance, pSessionHandle); 273 } 274 275 CpaStatus 276 cpaDcDpUpdateSession(const CpaInstanceHandle dcInstance, 277 CpaDcSessionHandle pSessionHandle, 278 CpaDcSessionUpdateData *pUpdateSessionData) 279 { 280 return CPA_STATUS_UNSUPPORTED; 281 } 282 283 CpaStatus 284 cpaDcDpRegCbFunc(const CpaInstanceHandle dcInstance, 285 const CpaDcDpCallbackFn pNewCb) 286 { 287 sal_compression_service_t *pService = NULL; 288 289 LAC_CHECK_NULL_PARAM(dcInstance); 290 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 291 LAC_CHECK_NULL_PARAM(pNewCb); 292 293 /* Check if SAL is initialised otherwise return an error */ 294 SAL_RUNNING_CHECK(dcInstance); 295 296 pService = (sal_compression_service_t *)dcInstance; 297 pService->pDcDpCb = pNewCb; 298 299 return CPA_STATUS_SUCCESS; 300 } 301 302 /** 303 ***************************************************************************** 304 * @ingroup cpaDcDp 305 * 306 * @description 307 * Writes the message to the ring 308 * 309 * @param[in] pOpData Pointer to a structure containing the 310 * request parameters 311 * @param[in] pCurrentQatMsg Pointer to current QAT message on the ring 312 * 313 *****************************************************************************/ 314 static void 315 dcDpWriteRingMsg(CpaDcDpOpData *pOpData, icp_qat_fw_comp_req_t *pCurrentQatMsg) 316 { 317 icp_qat_fw_comp_req_t *pReqCache = NULL; 318 dc_session_desc_t *pSessionDesc = NULL; 319 Cpa8U bufferFormat; 320 321 Cpa8U cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV; 322 Cpa8U cnvnrCompReq = ICP_QAT_FW_COMP_NO_CNV_RECOVERY; 323 CpaBoolean cnvErrorInjection = ICP_QAT_FW_COMP_NO_CNV_DFX; 324 sal_compression_service_t *pService = NULL; 325 326 pService = (sal_compression_service_t *)(pOpData->dcInstance); 327 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 328 329 if (CPA_DC_DIR_COMPRESS == pOpData->sessDirection) { 330 pReqCache = &(pSessionDesc->reqCacheComp); 331 /* CNV check */ 332 if (CPA_TRUE == pOpData->compressAndVerify) { 333 cnvDecompReq = ICP_QAT_FW_COMP_CNV; 334 if (isDcGen4x(pService)) { 335 cnvErrorInjection = 336 pSessionDesc->cnvErrorInjection; 337 } 338 339 /* CNVNR check */ 340 if (CPA_TRUE == pOpData->compressAndVerifyAndRecover) { 341 cnvnrCompReq = ICP_QAT_FW_COMP_CNV_RECOVERY; 342 } 343 } 344 } else { 345 pReqCache = &(pSessionDesc->reqCacheDecomp); 346 } 347 348 /* Fills in the template DC ET ring message - cached from the 349 * session descriptor */ 350 memcpy((void *)pCurrentQatMsg, 351 (void *)(pReqCache), 352 (LAC_QAT_DC_REQ_SZ_LW * LAC_LONG_WORD_IN_BYTES)); 353 354 if (CPA_DP_BUFLIST == pOpData->srcBufferLen) { 355 bufferFormat = QAT_COMN_PTR_TYPE_SGL; 356 } else { 357 bufferFormat = QAT_COMN_PTR_TYPE_FLAT; 358 } 359 360 pCurrentQatMsg->comp_pars.req_par_flags |= 361 ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD( 362 ICP_QAT_FW_COMP_NOT_SOP, 363 ICP_QAT_FW_COMP_NOT_EOP, 364 ICP_QAT_FW_COMP_NOT_BFINAL, 365 cnvDecompReq, 366 cnvnrCompReq, 367 cnvErrorInjection, 368 ICP_QAT_FW_COMP_CRC_MODE_LEGACY); 369 370 SalQatMsg_CmnMidWrite((icp_qat_fw_la_bulk_req_t *)pCurrentQatMsg, 371 pOpData, 372 bufferFormat, 373 pOpData->srcBuffer, 374 pOpData->destBuffer, 375 pOpData->srcBufferLen, 376 pOpData->destBufferLen); 377 378 pCurrentQatMsg->comp_pars.comp_len = pOpData->bufferLenToCompress; 379 pCurrentQatMsg->comp_pars.out_buffer_sz = pOpData->bufferLenForData; 380 } 381 382 CpaStatus 383 cpaDcDpEnqueueOp(CpaDcDpOpData *pOpData, const CpaBoolean performOpNow) 384 { 385 icp_qat_fw_comp_req_t *pCurrentQatMsg = NULL; 386 icp_comms_trans_handle trans_handle = NULL; 387 dc_session_desc_t *pSessionDesc = NULL; 388 CpaStatus status = CPA_STATUS_SUCCESS; 389 390 status = dcDataPlaneParamCheck(pOpData); 391 if (CPA_STATUS_SUCCESS != status) { 392 return status; 393 } 394 395 if ((CPA_FALSE == pOpData->compressAndVerify) && 396 (CPA_DC_DIR_COMPRESS == pOpData->sessDirection)) { 397 return CPA_STATUS_UNSUPPORTED; 398 } 399 400 /* Check if SAL is initialised otherwise return an error */ 401 SAL_RUNNING_CHECK(pOpData->dcInstance); 402 403 trans_handle = ((sal_compression_service_t *)pOpData->dcInstance) 404 ->trans_handle_compression_tx; 405 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 406 407 if ((CPA_DC_DIR_COMPRESS == pOpData->sessDirection) && 408 (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection)) { 409 QAT_UTILS_LOG( 410 "The session does not support this direction of operation.\n"); 411 return CPA_STATUS_INVALID_PARAM; 412 } else if ((CPA_DC_DIR_DECOMPRESS == pOpData->sessDirection) && 413 (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection)) { 414 QAT_UTILS_LOG( 415 "The session does not support this direction of operation.\n"); 416 return CPA_STATUS_INVALID_PARAM; 417 } 418 419 icp_adf_getSingleQueueAddr(trans_handle, (void **)&pCurrentQatMsg); 420 if (NULL == pCurrentQatMsg) { 421 return CPA_STATUS_RETRY; 422 } 423 424 dcDpWriteRingMsg(pOpData, pCurrentQatMsg); 425 pSessionDesc->pendingDpStatelessCbCount++; 426 427 if (CPA_TRUE == performOpNow) { 428 SalQatMsg_updateQueueTail(trans_handle); 429 } 430 431 return CPA_STATUS_SUCCESS; 432 } 433 434 CpaStatus 435 cpaDcDpEnqueueOpBatch(const Cpa32U numberRequests, 436 CpaDcDpOpData *pOpData[], 437 const CpaBoolean performOpNow) 438 { 439 icp_qat_fw_comp_req_t *pCurrentQatMsg = NULL; 440 icp_comms_trans_handle trans_handle = NULL; 441 dc_session_desc_t *pSessionDesc = NULL; 442 Cpa32U i = 0; 443 CpaStatus status = CPA_STATUS_SUCCESS; 444 sal_compression_service_t *pService = NULL; 445 446 LAC_CHECK_NULL_PARAM(pOpData); 447 LAC_CHECK_NULL_PARAM(pOpData[0]); 448 LAC_CHECK_NULL_PARAM(pOpData[0]->dcInstance); 449 450 pService = (sal_compression_service_t *)(pOpData[0]->dcInstance); 451 if ((numberRequests == 0) || 452 (numberRequests > pService->maxNumCompConcurrentReq)) { 453 QAT_UTILS_LOG( 454 "The number of requests needs to be between 1 and %d.\n", 455 pService->maxNumCompConcurrentReq); 456 return CPA_STATUS_INVALID_PARAM; 457 } 458 459 for (i = 0; i < numberRequests; i++) { 460 status = dcDataPlaneParamCheck(pOpData[i]); 461 if (CPA_STATUS_SUCCESS != status) { 462 return status; 463 } 464 465 /* Check that all instance handles and session handles are the 466 * same */ 467 if (pOpData[i]->dcInstance != pOpData[0]->dcInstance) { 468 QAT_UTILS_LOG( 469 "All instance handles should be the same in the pOpData.\n"); 470 return CPA_STATUS_INVALID_PARAM; 471 } 472 473 if (pOpData[i]->pSessionHandle != pOpData[0]->pSessionHandle) { 474 QAT_UTILS_LOG( 475 "All session handles should be the same in the pOpData.\n"); 476 return CPA_STATUS_INVALID_PARAM; 477 } 478 } 479 480 for (i = 0; i < numberRequests; i++) { 481 if ((CPA_FALSE == pOpData[i]->compressAndVerify) && 482 (CPA_DC_DIR_COMPRESS == pOpData[i]->sessDirection)) { 483 return CPA_STATUS_UNSUPPORTED; 484 } 485 } 486 487 /* Check if SAL is initialised otherwise return an error */ 488 SAL_RUNNING_CHECK(pOpData[0]->dcInstance); 489 490 trans_handle = ((sal_compression_service_t *)pOpData[0]->dcInstance) 491 ->trans_handle_compression_tx; 492 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData[0]->pSessionHandle); 493 494 for (i = 0; i < numberRequests; i++) { 495 if ((CPA_DC_DIR_COMPRESS == pOpData[i]->sessDirection) && 496 (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection)) { 497 QAT_UTILS_LOG( 498 "The session does not support this direction of operation.\n"); 499 return CPA_STATUS_INVALID_PARAM; 500 } else if ((CPA_DC_DIR_DECOMPRESS == 501 pOpData[i]->sessDirection) && 502 (CPA_DC_DIR_COMPRESS == 503 pSessionDesc->sessDirection)) { 504 QAT_UTILS_LOG( 505 "The session does not support this direction of operation.\n"); 506 return CPA_STATUS_INVALID_PARAM; 507 } 508 } 509 510 icp_adf_getQueueMemory(trans_handle, 511 numberRequests, 512 (void **)&pCurrentQatMsg); 513 if (NULL == pCurrentQatMsg) { 514 return CPA_STATUS_RETRY; 515 } 516 517 for (i = 0; i < numberRequests; i++) { 518 dcDpWriteRingMsg(pOpData[i], pCurrentQatMsg); 519 icp_adf_getQueueNext(trans_handle, (void **)&pCurrentQatMsg); 520 } 521 522 pSessionDesc->pendingDpStatelessCbCount += numberRequests; 523 524 if (CPA_TRUE == performOpNow) { 525 SalQatMsg_updateQueueTail(trans_handle); 526 } 527 528 return CPA_STATUS_SUCCESS; 529 } 530 531 CpaStatus 532 icp_sal_DcPollDpInstance(CpaInstanceHandle dcInstance, Cpa32U responseQuota) 533 { 534 icp_comms_trans_handle trans_handle = NULL; 535 536 LAC_CHECK_INSTANCE_HANDLE(dcInstance); 537 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 538 539 /* Check if SAL is initialised otherwise return an error */ 540 SAL_RUNNING_CHECK(dcInstance); 541 542 trans_handle = ((sal_compression_service_t *)dcInstance) 543 ->trans_handle_compression_rx; 544 545 return icp_adf_pollQueue(trans_handle, responseQuota); 546 } 547 548 CpaStatus 549 cpaDcDpPerformOpNow(CpaInstanceHandle dcInstance) 550 { 551 icp_comms_trans_handle trans_handle = NULL; 552 553 LAC_CHECK_NULL_PARAM(dcInstance); 554 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 555 556 /* Check if SAL is initialised otherwise return an error */ 557 SAL_RUNNING_CHECK(dcInstance); 558 559 trans_handle = ((sal_compression_service_t *)dcInstance) 560 ->trans_handle_compression_tx; 561 562 if (CPA_TRUE == icp_adf_queueDataToSend(trans_handle)) { 563 SalQatMsg_updateQueueTail(trans_handle); 564 } 565 566 return CPA_STATUS_SUCCESS; 567 } 568