1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 /** 5 ***************************************************************************** 6 * @file dc_dp.c 7 * 8 * @defgroup cpaDcDp Data Compression Data Plane API 9 * 10 * @ingroup cpaDcDp 11 * 12 * @description 13 * Implementation of the Data Compression DP operations. 14 * 15 *****************************************************************************/ 16 17 /* 18 ******************************************************************************* 19 * Include public/global header files 20 ******************************************************************************* 21 */ 22 #include "cpa.h" 23 #include "cpa_dc.h" 24 #include "cpa_dc_dp.h" 25 26 #include "icp_qat_fw_comp.h" 27 28 /* 29 ******************************************************************************* 30 * Include private header files 31 ******************************************************************************* 32 */ 33 #include "dc_session.h" 34 #include "dc_datapath.h" 35 #include "lac_common.h" 36 #include "lac_mem.h" 37 #include "lac_mem_pools.h" 38 #include "sal_types_compression.h" 39 #include "lac_sal.h" 40 #include "lac_sync.h" 41 #include "sal_service_state.h" 42 #include "sal_qat_cmn_msg.h" 43 #include "icp_sal_poll.h" 44 #include "sal_hw_gen.h" 45 46 /** 47 ***************************************************************************** 48 * @ingroup cpaDcDp 49 * Check that pOpData is valid 50 * 51 * @description 52 * Check that all the parameters defined in the pOpData are valid 53 * 54 * @param[in] pOpData Pointer to a structure containing the 55 * request parameters 56 * 57 * @retval CPA_STATUS_SUCCESS Function executed successfully 58 * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in 59 * 60 *****************************************************************************/ 61 static CpaStatus 62 dcDataPlaneParamCheck(const CpaDcDpOpData *pOpData) 63 { 64 sal_compression_service_t *pService = NULL; 65 dc_session_desc_t *pSessionDesc = NULL; 66 67 LAC_CHECK_NULL_PARAM(pOpData); 68 LAC_CHECK_NULL_PARAM(pOpData->dcInstance); 69 LAC_CHECK_NULL_PARAM(pOpData->pSessionHandle); 70 71 /* Ensure this is a compression instance */ 72 SAL_CHECK_INSTANCE_TYPE(pOpData->dcInstance, 73 SAL_SERVICE_TYPE_COMPRESSION); 74 75 pService = (sal_compression_service_t *)(pOpData->dcInstance); 76 77 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 78 if (NULL == pSessionDesc) { 79 QAT_UTILS_LOG("Session handle not as expected.\n"); 80 return CPA_STATUS_INVALID_PARAM; 81 } 82 83 if (CPA_FALSE == pSessionDesc->isDcDp) { 84 QAT_UTILS_LOG("The session type should be data plane.\n"); 85 return CPA_STATUS_INVALID_PARAM; 86 } 87 88 /* Compressing zero byte is not supported */ 89 if ((CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) && 90 (0 == pOpData->bufferLenToCompress)) { 91 QAT_UTILS_LOG("The source buffer length to compress needs to " 92 "be greater than zero byte.\n"); 93 return CPA_STATUS_INVALID_PARAM; 94 } 95 96 if (pOpData->sessDirection > CPA_DC_DIR_DECOMPRESS) { 97 QAT_UTILS_LOG("Invalid direction of operation.\n"); 98 return CPA_STATUS_INVALID_PARAM; 99 } 100 101 if (0 == pOpData->srcBuffer) { 102 QAT_UTILS_LOG("Invalid srcBuffer\n"); 103 return CPA_STATUS_INVALID_PARAM; 104 } 105 if (0 == pOpData->destBuffer) { 106 QAT_UTILS_LOG("Invalid destBuffer\n"); 107 return CPA_STATUS_INVALID_PARAM; 108 } 109 if (pOpData->srcBuffer == pOpData->destBuffer) { 110 QAT_UTILS_LOG("In place operation is not supported.\n"); 111 return CPA_STATUS_INVALID_PARAM; 112 } 113 if (0 == pOpData->thisPhys) { 114 QAT_UTILS_LOG("Invalid thisPhys\n"); 115 return CPA_STATUS_INVALID_PARAM; 116 } 117 118 if ((CPA_TRUE != pOpData->compressAndVerify) && 119 (CPA_FALSE != pOpData->compressAndVerify)) { 120 QAT_UTILS_LOG("Invalid compressAndVerify\n"); 121 return CPA_STATUS_INVALID_PARAM; 122 } 123 if ((CPA_TRUE == pOpData->compressAndVerify) && 124 !(pService->generic_service_info.dcExtendedFeatures & 125 DC_CNV_EXTENDED_CAPABILITY)) { 126 QAT_UTILS_LOG("Invalid compressAndVerify, no CNV capability\n"); 127 return CPA_STATUS_UNSUPPORTED; 128 } 129 if ((CPA_TRUE != pOpData->compressAndVerifyAndRecover) && 130 (CPA_FALSE != pOpData->compressAndVerifyAndRecover)) { 131 QAT_UTILS_LOG("Invalid compressAndVerifyAndRecover\n"); 132 return CPA_STATUS_INVALID_PARAM; 133 } 134 if ((CPA_TRUE == pOpData->compressAndVerifyAndRecover) && 135 (CPA_FALSE == pOpData->compressAndVerify)) { 136 QAT_UTILS_LOG("CnVnR option set without setting CnV\n"); 137 return CPA_STATUS_INVALID_PARAM; 138 } 139 if ((CPA_TRUE == pOpData->compressAndVerifyAndRecover) && 140 !(pService->generic_service_info.dcExtendedFeatures & 141 DC_CNVNR_EXTENDED_CAPABILITY)) { 142 QAT_UTILS_LOG( 143 "Invalid CnVnR option set and no CnVnR capability.\n"); 144 return CPA_STATUS_UNSUPPORTED; 145 } 146 147 if ((CPA_DP_BUFLIST == pOpData->srcBufferLen) && 148 (CPA_DP_BUFLIST != pOpData->destBufferLen)) { 149 QAT_UTILS_LOG( 150 "The source and destination buffers need to be of the same type (both flat buffers or buffer lists).\n"); 151 return CPA_STATUS_INVALID_PARAM; 152 } 153 if ((CPA_DP_BUFLIST != pOpData->srcBufferLen) && 154 (CPA_DP_BUFLIST == pOpData->destBufferLen)) { 155 QAT_UTILS_LOG( 156 "The source and destination buffers need to be of the same type (both flat buffers or buffer lists).\n"); 157 return CPA_STATUS_INVALID_PARAM; 158 } 159 160 if (CPA_DP_BUFLIST != pOpData->srcBufferLen) { 161 if (pOpData->srcBufferLen < pOpData->bufferLenToCompress) { 162 QAT_UTILS_LOG( 163 "srcBufferLen is smaller than bufferLenToCompress.\n"); 164 return CPA_STATUS_INVALID_PARAM; 165 } 166 167 if (pOpData->destBufferLen < pOpData->bufferLenForData) { 168 QAT_UTILS_LOG( 169 "destBufferLen is smaller than bufferLenForData.\n"); 170 return CPA_STATUS_INVALID_PARAM; 171 } 172 } else { 173 /* We are assuming that there is enough memory in the source and 174 * destination buffer lists. We only receive physical addresses 175 * of the buffers so we are unable to test it here */ 176 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->srcBuffer); 177 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->destBuffer); 178 } 179 180 LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->thisPhys); 181 182 if ((CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) || 183 (CPA_DC_DIR_COMBINED == pSessionDesc->sessDirection)) { 184 if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) { 185 /* Check if Intermediate Buffer Array pointer is NULL */ 186 if (isDcGen2x(pService) && 187 ((0 == pService->pInterBuffPtrsArrayPhyAddr) || 188 (NULL == pService->pInterBuffPtrsArray))) { 189 QAT_UTILS_LOG( 190 "No intermediate buffer defined for this instance - see cpaDcStartInstance.\n"); 191 return CPA_STATUS_INVALID_PARAM; 192 } 193 194 /* Ensure that the destination buffer length for data is 195 * greater 196 * or equal to 128B */ 197 if (pOpData->bufferLenForData < 198 DC_DEST_BUFFER_DYN_MIN_SIZE) { 199 QAT_UTILS_LOG( 200 "Destination buffer length for data should be greater or equal to 128B.\n"); 201 return CPA_STATUS_INVALID_PARAM; 202 } 203 } else { 204 /* Ensure that the destination buffer length for data is 205 * greater 206 * or equal to min output buffsize */ 207 if (pOpData->bufferLenForData < 208 pService->comp_device_data.minOutputBuffSize) { 209 QAT_UTILS_LOG( 210 "Destination buffer size should be greater or equal to %d bytes.\n", 211 pService->comp_device_data 212 .minOutputBuffSize); 213 return CPA_STATUS_INVALID_PARAM; 214 } 215 } 216 } 217 218 return CPA_STATUS_SUCCESS; 219 } 220 221 CpaStatus 222 cpaDcDpGetSessionSize(CpaInstanceHandle dcInstance, 223 CpaDcSessionSetupData *pSessionData, 224 Cpa32U *pSessionSize) 225 { 226 return dcGetSessionSize(dcInstance, pSessionData, pSessionSize, NULL); 227 } 228 229 CpaStatus 230 cpaDcDpInitSession(CpaInstanceHandle dcInstance, 231 CpaDcSessionHandle pSessionHandle, 232 CpaDcSessionSetupData *pSessionData) 233 { 234 CpaStatus status = CPA_STATUS_SUCCESS; 235 dc_session_desc_t *pSessionDesc = NULL; 236 sal_compression_service_t *pService = NULL; 237 238 LAC_CHECK_INSTANCE_HANDLE(dcInstance); 239 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 240 241 pService = (sal_compression_service_t *)dcInstance; 242 243 /* Check if SAL is initialised otherwise return an error */ 244 SAL_RUNNING_CHECK(pService); 245 246 /* Stateful is not supported */ 247 if (CPA_DC_STATELESS != pSessionData->sessState) { 248 QAT_UTILS_LOG("Invalid sessState value\n"); 249 return CPA_STATUS_INVALID_PARAM; 250 } 251 252 status = 253 dcInitSession(dcInstance, pSessionHandle, pSessionData, NULL, NULL); 254 if (CPA_STATUS_SUCCESS == status) { 255 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle); 256 pSessionDesc->isDcDp = CPA_TRUE; 257 258 ICP_QAT_FW_COMN_PTR_TYPE_SET( 259 pSessionDesc->reqCacheDecomp.comn_hdr.comn_req_flags, 260 DC_DP_QAT_PTR_TYPE); 261 ICP_QAT_FW_COMN_PTR_TYPE_SET( 262 pSessionDesc->reqCacheComp.comn_hdr.comn_req_flags, 263 DC_DP_QAT_PTR_TYPE); 264 } 265 266 return status; 267 } 268 269 CpaStatus 270 cpaDcDpRemoveSession(const CpaInstanceHandle dcInstance, 271 CpaDcSessionHandle pSessionHandle) 272 { 273 return cpaDcRemoveSession(dcInstance, pSessionHandle); 274 } 275 276 CpaStatus 277 cpaDcDpUpdateSession(const CpaInstanceHandle dcInstance, 278 CpaDcSessionHandle pSessionHandle, 279 CpaDcSessionUpdateData *pUpdateSessionData) 280 { 281 return CPA_STATUS_UNSUPPORTED; 282 } 283 284 CpaStatus 285 cpaDcDpRegCbFunc(const CpaInstanceHandle dcInstance, 286 const CpaDcDpCallbackFn pNewCb) 287 { 288 sal_compression_service_t *pService = NULL; 289 290 LAC_CHECK_NULL_PARAM(dcInstance); 291 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 292 LAC_CHECK_NULL_PARAM(pNewCb); 293 294 /* Check if SAL is initialised otherwise return an error */ 295 SAL_RUNNING_CHECK(dcInstance); 296 297 pService = (sal_compression_service_t *)dcInstance; 298 pService->pDcDpCb = pNewCb; 299 300 return CPA_STATUS_SUCCESS; 301 } 302 303 /** 304 ***************************************************************************** 305 * @ingroup cpaDcDp 306 * 307 * @description 308 * Writes the message to the ring 309 * 310 * @param[in] pOpData Pointer to a structure containing the 311 * request parameters 312 * @param[in] pCurrentQatMsg Pointer to current QAT message on the ring 313 * 314 *****************************************************************************/ 315 static void 316 dcDpWriteRingMsg(CpaDcDpOpData *pOpData, icp_qat_fw_comp_req_t *pCurrentQatMsg) 317 { 318 icp_qat_fw_comp_req_t *pReqCache = NULL; 319 dc_session_desc_t *pSessionDesc = NULL; 320 Cpa8U bufferFormat; 321 322 Cpa8U cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV; 323 Cpa8U cnvnrCompReq = ICP_QAT_FW_COMP_NO_CNV_RECOVERY; 324 CpaBoolean cnvErrorInjection = ICP_QAT_FW_COMP_NO_CNV_DFX; 325 sal_compression_service_t *pService = NULL; 326 327 pService = (sal_compression_service_t *)(pOpData->dcInstance); 328 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 329 330 if (CPA_DC_DIR_COMPRESS == pOpData->sessDirection) { 331 pReqCache = &(pSessionDesc->reqCacheComp); 332 /* CNV check */ 333 if (CPA_TRUE == pOpData->compressAndVerify) { 334 cnvDecompReq = ICP_QAT_FW_COMP_CNV; 335 if (isDcGen4x(pService)) { 336 cnvErrorInjection = 337 pSessionDesc->cnvErrorInjection; 338 } 339 340 /* CNVNR check */ 341 if (CPA_TRUE == pOpData->compressAndVerifyAndRecover) { 342 cnvnrCompReq = ICP_QAT_FW_COMP_CNV_RECOVERY; 343 } 344 } 345 } else { 346 pReqCache = &(pSessionDesc->reqCacheDecomp); 347 } 348 349 /* Fills in the template DC ET ring message - cached from the 350 * session descriptor */ 351 memcpy((void *)pCurrentQatMsg, 352 (void *)(pReqCache), 353 (LAC_QAT_DC_REQ_SZ_LW * LAC_LONG_WORD_IN_BYTES)); 354 355 if (CPA_DP_BUFLIST == pOpData->srcBufferLen) { 356 bufferFormat = QAT_COMN_PTR_TYPE_SGL; 357 } else { 358 bufferFormat = QAT_COMN_PTR_TYPE_FLAT; 359 } 360 361 pCurrentQatMsg->comp_pars.req_par_flags |= 362 ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD( 363 ICP_QAT_FW_COMP_NOT_SOP, 364 ICP_QAT_FW_COMP_NOT_EOP, 365 ICP_QAT_FW_COMP_NOT_BFINAL, 366 cnvDecompReq, 367 cnvnrCompReq, 368 cnvErrorInjection, 369 ICP_QAT_FW_COMP_CRC_MODE_LEGACY); 370 371 SalQatMsg_CmnMidWrite((icp_qat_fw_la_bulk_req_t *)pCurrentQatMsg, 372 pOpData, 373 bufferFormat, 374 pOpData->srcBuffer, 375 pOpData->destBuffer, 376 pOpData->srcBufferLen, 377 pOpData->destBufferLen); 378 379 pCurrentQatMsg->comp_pars.comp_len = pOpData->bufferLenToCompress; 380 pCurrentQatMsg->comp_pars.out_buffer_sz = pOpData->bufferLenForData; 381 } 382 383 CpaStatus 384 cpaDcDpEnqueueOp(CpaDcDpOpData *pOpData, const CpaBoolean performOpNow) 385 { 386 icp_qat_fw_comp_req_t *pCurrentQatMsg = NULL; 387 icp_comms_trans_handle trans_handle = NULL; 388 dc_session_desc_t *pSessionDesc = NULL; 389 CpaStatus status = CPA_STATUS_SUCCESS; 390 391 status = dcDataPlaneParamCheck(pOpData); 392 if (CPA_STATUS_SUCCESS != status) { 393 return status; 394 } 395 396 if ((CPA_FALSE == pOpData->compressAndVerify) && 397 (CPA_DC_DIR_COMPRESS == pOpData->sessDirection)) { 398 return CPA_STATUS_UNSUPPORTED; 399 } 400 401 /* Check if SAL is initialised otherwise return an error */ 402 SAL_RUNNING_CHECK(pOpData->dcInstance); 403 404 trans_handle = ((sal_compression_service_t *)pOpData->dcInstance) 405 ->trans_handle_compression_tx; 406 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle); 407 408 if ((CPA_DC_DIR_COMPRESS == pOpData->sessDirection) && 409 (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection)) { 410 QAT_UTILS_LOG( 411 "The session does not support this direction of operation.\n"); 412 return CPA_STATUS_INVALID_PARAM; 413 } else if ((CPA_DC_DIR_DECOMPRESS == pOpData->sessDirection) && 414 (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection)) { 415 QAT_UTILS_LOG( 416 "The session does not support this direction of operation.\n"); 417 return CPA_STATUS_INVALID_PARAM; 418 } 419 420 icp_adf_getSingleQueueAddr(trans_handle, (void **)&pCurrentQatMsg); 421 if (NULL == pCurrentQatMsg) { 422 return CPA_STATUS_RETRY; 423 } 424 425 dcDpWriteRingMsg(pOpData, pCurrentQatMsg); 426 pSessionDesc->pendingDpStatelessCbCount++; 427 428 if (CPA_TRUE == performOpNow) { 429 SalQatMsg_updateQueueTail(trans_handle); 430 } 431 432 return CPA_STATUS_SUCCESS; 433 } 434 435 CpaStatus 436 cpaDcDpEnqueueOpBatch(const Cpa32U numberRequests, 437 CpaDcDpOpData *pOpData[], 438 const CpaBoolean performOpNow) 439 { 440 icp_qat_fw_comp_req_t *pCurrentQatMsg = NULL; 441 icp_comms_trans_handle trans_handle = NULL; 442 dc_session_desc_t *pSessionDesc = NULL; 443 Cpa32U i = 0; 444 CpaStatus status = CPA_STATUS_SUCCESS; 445 sal_compression_service_t *pService = NULL; 446 447 LAC_CHECK_NULL_PARAM(pOpData); 448 LAC_CHECK_NULL_PARAM(pOpData[0]); 449 LAC_CHECK_NULL_PARAM(pOpData[0]->dcInstance); 450 451 pService = (sal_compression_service_t *)(pOpData[0]->dcInstance); 452 if ((numberRequests == 0) || 453 (numberRequests > pService->maxNumCompConcurrentReq)) { 454 QAT_UTILS_LOG( 455 "The number of requests needs to be between 1 and %d.\n", 456 pService->maxNumCompConcurrentReq); 457 return CPA_STATUS_INVALID_PARAM; 458 } 459 460 for (i = 0; i < numberRequests; i++) { 461 status = dcDataPlaneParamCheck(pOpData[i]); 462 if (CPA_STATUS_SUCCESS != status) { 463 return status; 464 } 465 466 /* Check that all instance handles and session handles are the 467 * same */ 468 if (pOpData[i]->dcInstance != pOpData[0]->dcInstance) { 469 QAT_UTILS_LOG( 470 "All instance handles should be the same in the pOpData.\n"); 471 return CPA_STATUS_INVALID_PARAM; 472 } 473 474 if (pOpData[i]->pSessionHandle != pOpData[0]->pSessionHandle) { 475 QAT_UTILS_LOG( 476 "All session handles should be the same in the pOpData.\n"); 477 return CPA_STATUS_INVALID_PARAM; 478 } 479 } 480 481 for (i = 0; i < numberRequests; i++) { 482 if ((CPA_FALSE == pOpData[i]->compressAndVerify) && 483 (CPA_DC_DIR_COMPRESS == pOpData[i]->sessDirection)) { 484 return CPA_STATUS_UNSUPPORTED; 485 } 486 } 487 488 /* Check if SAL is initialised otherwise return an error */ 489 SAL_RUNNING_CHECK(pOpData[0]->dcInstance); 490 491 trans_handle = ((sal_compression_service_t *)pOpData[0]->dcInstance) 492 ->trans_handle_compression_tx; 493 pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData[0]->pSessionHandle); 494 495 for (i = 0; i < numberRequests; i++) { 496 if ((CPA_DC_DIR_COMPRESS == pOpData[i]->sessDirection) && 497 (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection)) { 498 QAT_UTILS_LOG( 499 "The session does not support this direction of operation.\n"); 500 return CPA_STATUS_INVALID_PARAM; 501 } else if ((CPA_DC_DIR_DECOMPRESS == 502 pOpData[i]->sessDirection) && 503 (CPA_DC_DIR_COMPRESS == 504 pSessionDesc->sessDirection)) { 505 QAT_UTILS_LOG( 506 "The session does not support this direction of operation.\n"); 507 return CPA_STATUS_INVALID_PARAM; 508 } 509 } 510 511 icp_adf_getQueueMemory(trans_handle, 512 numberRequests, 513 (void **)&pCurrentQatMsg); 514 if (NULL == pCurrentQatMsg) { 515 return CPA_STATUS_RETRY; 516 } 517 518 for (i = 0; i < numberRequests; i++) { 519 dcDpWriteRingMsg(pOpData[i], pCurrentQatMsg); 520 icp_adf_getQueueNext(trans_handle, (void **)&pCurrentQatMsg); 521 } 522 523 pSessionDesc->pendingDpStatelessCbCount += numberRequests; 524 525 if (CPA_TRUE == performOpNow) { 526 SalQatMsg_updateQueueTail(trans_handle); 527 } 528 529 return CPA_STATUS_SUCCESS; 530 } 531 532 CpaStatus 533 icp_sal_DcPollDpInstance(CpaInstanceHandle dcInstance, Cpa32U responseQuota) 534 { 535 icp_comms_trans_handle trans_handle = NULL; 536 537 LAC_CHECK_INSTANCE_HANDLE(dcInstance); 538 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 539 540 /* Check if SAL is initialised otherwise return an error */ 541 SAL_RUNNING_CHECK(dcInstance); 542 543 trans_handle = ((sal_compression_service_t *)dcInstance) 544 ->trans_handle_compression_rx; 545 546 return icp_adf_pollQueue(trans_handle, responseQuota); 547 } 548 549 CpaStatus 550 cpaDcDpPerformOpNow(CpaInstanceHandle dcInstance) 551 { 552 icp_comms_trans_handle trans_handle = NULL; 553 554 LAC_CHECK_NULL_PARAM(dcInstance); 555 SAL_CHECK_INSTANCE_TYPE(dcInstance, SAL_SERVICE_TYPE_COMPRESSION); 556 557 /* Check if SAL is initialised otherwise return an error */ 558 SAL_RUNNING_CHECK(dcInstance); 559 560 trans_handle = ((sal_compression_service_t *)dcInstance) 561 ->trans_handle_compression_tx; 562 563 if (CPA_TRUE == icp_adf_queueDataToSend(trans_handle)) { 564 SalQatMsg_updateQueueTail(trans_handle); 565 } 566 567 return CPA_STATUS_SUCCESS; 568 } 569