1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 #include "adf_transport_access_macros.h" 5 #include "adf_transport_internal.h" 6 7 #include "cpa.h" 8 #include "icp_adf_init.h" 9 #include "icp_adf_transport.h" 10 #include "icp_adf_poll.h" 11 #include "icp_adf_transport_dp.h" 12 #include "icp_sal_poll.h" 13 14 /* 15 * adf_modulo 16 * result = data % ( 2 ^ shift ) 17 */ 18 static inline Cpa32U 19 adf_modulo(Cpa32U data, Cpa32U shift) 20 { 21 Cpa32U div = data >> shift; 22 Cpa32U mult = div << shift; 23 24 return data - mult; 25 } 26 27 /* 28 * icp_adf_transCreateHandle 29 * crete transport handle for a service 30 * call adf_create_ring from adf driver directly with same parameters 31 */ 32 CpaStatus 33 icp_adf_transCreateHandle(icp_accel_dev_t *adf, 34 icp_transport_type trans_type, 35 const char *section, 36 const uint32_t accel_nr, 37 const uint32_t bank_nr, 38 const char *service_name, 39 const icp_adf_ringInfoService_t info, 40 icp_trans_callback callback, 41 icp_resp_deliv_method resp, 42 const uint32_t num_msgs, 43 const uint32_t msg_size, 44 icp_comms_trans_handle *trans_handle) 45 { 46 CpaStatus status; 47 int error; 48 49 ICP_CHECK_FOR_NULL_PARAM(trans_handle); 50 ICP_CHECK_FOR_NULL_PARAM(adf); 51 52 error = adf_create_ring(adf->accel_dev, 53 section, 54 bank_nr, 55 num_msgs, 56 msg_size, 57 service_name, 58 callback, 59 ((resp == ICP_RESP_TYPE_IRQ) ? 0 : 1), 60 (struct adf_etr_ring_data **)trans_handle); 61 if (!error) 62 status = CPA_STATUS_SUCCESS; 63 else 64 status = CPA_STATUS_FAIL; 65 66 return status; 67 } 68 69 /* 70 * icp_adf_transReinitHandle 71 * Reinitialize transport handle for a service 72 */ 73 CpaStatus 74 icp_adf_transReinitHandle(icp_accel_dev_t *adf, 75 icp_transport_type trans_type, 76 const char *section, 77 const uint32_t accel_nr, 78 const uint32_t bank_nr, 79 const char *service_name, 80 const icp_adf_ringInfoService_t info, 81 icp_trans_callback callback, 82 icp_resp_deliv_method resp, 83 const uint32_t num_msgs, 84 const uint32_t msg_size, 85 icp_comms_trans_handle *trans_handle) 86 { 87 return CPA_STATUS_SUCCESS; 88 } 89 /* 90 * icp_adf_transReleaseHandle 91 * destroy a transport handle, call adf_remove_ring from adf driver directly 92 */ 93 CpaStatus 94 icp_adf_transReleaseHandle(icp_comms_trans_handle trans_handle) 95 { 96 struct adf_etr_ring_data *ring = trans_handle; 97 98 ICP_CHECK_FOR_NULL_PARAM(ring); 99 adf_remove_ring(ring); 100 101 return CPA_STATUS_SUCCESS; 102 } 103 104 /* 105 * icp_adf_transResetHandle 106 * clean a transport handle, call adf_remove_ring from adf driver directly 107 */ 108 CpaStatus 109 icp_adf_transResetHandle(icp_comms_trans_handle trans_handle) 110 { 111 return CPA_STATUS_SUCCESS; 112 } 113 114 /* 115 * icp_adf_transGetRingNum 116 * get ring number from a transport handle 117 */ 118 CpaStatus 119 icp_adf_transGetRingNum(icp_comms_trans_handle trans_handle, uint32_t *ringNum) 120 { 121 struct adf_etr_ring_data *ring = trans_handle; 122 123 ICP_CHECK_FOR_NULL_PARAM(ring); 124 ICP_CHECK_FOR_NULL_PARAM(ringNum); 125 *ringNum = (uint32_t)(ring->ring_number); 126 127 return CPA_STATUS_SUCCESS; 128 } 129 130 /* 131 * icp_adf_transPutMsg 132 * send a request to transport handle 133 * call adf_send_message from adf driver directly 134 */ 135 CpaStatus 136 icp_adf_transPutMsg(icp_comms_trans_handle trans_handle, 137 uint32_t *inBuf, 138 uint32_t bufLen) 139 { 140 struct adf_etr_ring_data *ring = trans_handle; 141 CpaStatus status = CPA_STATUS_FAIL; 142 int error = EFAULT; 143 144 ICP_CHECK_FOR_NULL_PARAM(ring); 145 146 error = adf_send_message(ring, inBuf); 147 if (EAGAIN == error) 148 status = CPA_STATUS_RETRY; 149 else if (0 == error) 150 status = CPA_STATUS_SUCCESS; 151 else 152 status = CPA_STATUS_FAIL; 153 154 return status; 155 } 156 157 CpaStatus 158 icp_adf_getInflightRequests(icp_comms_trans_handle trans_handle, 159 Cpa32U *maxInflightRequests, 160 Cpa32U *numInflightRequests) 161 { 162 struct adf_etr_ring_data *ring = trans_handle; 163 ICP_CHECK_FOR_NULL_PARAM(ring); 164 ICP_CHECK_FOR_NULL_PARAM(maxInflightRequests); 165 ICP_CHECK_FOR_NULL_PARAM(numInflightRequests); 166 /* 167 * XXX: The qat_direct version of this routine returns max - 1, not 168 * the absolute max. 169 */ 170 *numInflightRequests = (*(uint32_t *)ring->inflights); 171 *maxInflightRequests = 172 ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size); 173 return CPA_STATUS_SUCCESS; 174 } 175 176 CpaStatus 177 icp_adf_dp_getInflightRequests(icp_comms_trans_handle trans_handle, 178 Cpa32U *maxInflightRequests, 179 Cpa32U *numInflightRequests) 180 { 181 ICP_CHECK_FOR_NULL_PARAM(trans_handle); 182 ICP_CHECK_FOR_NULL_PARAM(maxInflightRequests); 183 ICP_CHECK_FOR_NULL_PARAM(numInflightRequests); 184 185 return icp_adf_getInflightRequests(trans_handle, 186 maxInflightRequests, 187 numInflightRequests); 188 } 189 190 /* 191 * This function allows the user to poll the response ring. The 192 * ring number to be polled is supplied by the user via the 193 * trans handle for that ring. The trans_hnd is a pointer 194 * to an array of trans handles. This ring is 195 * only polled if it contains data. 196 * This method is used as an alternative to the reading messages 197 * via the ISR method. 198 * This function will return RETRY if the ring is empty. 199 */ 200 CpaStatus 201 icp_adf_pollInstance(icp_comms_trans_handle *trans_hnd, 202 Cpa32U num_transHandles, 203 Cpa32U response_quota) 204 { 205 Cpa32U resp_total = 0; 206 Cpa32U num_resp; 207 struct adf_etr_ring_data *ring = NULL; 208 struct adf_etr_bank_data *bank = NULL; 209 Cpa32U i; 210 211 ICP_CHECK_FOR_NULL_PARAM(trans_hnd); 212 213 for (i = 0; i < num_transHandles; i++) { 214 ring = trans_hnd[i]; 215 if (!ring) 216 continue; 217 bank = ring->bank; 218 219 /* If the ring in question is empty try the next ring.*/ 220 if (!bank || !bank->ring_mask) { 221 continue; 222 } 223 224 num_resp = adf_handle_response(ring, response_quota); 225 resp_total += num_resp; 226 } 227 228 /* If any of the rings in the instance had data and was polled 229 * return SUCCESS. */ 230 if (resp_total) 231 return CPA_STATUS_SUCCESS; 232 else 233 return CPA_STATUS_RETRY; 234 } 235 236 /* 237 * This function allows the user to check the response ring. The 238 * ring number to be polled is supplied by the user via the 239 * trans handle for that ring. The trans_hnd is a pointer 240 * to an array of trans handles. 241 * This function now is a empty function. 242 */ 243 CpaStatus 244 icp_adf_check_RespInstance(icp_comms_trans_handle *trans_hnd, 245 Cpa32U num_transHandles) 246 { 247 return CPA_STATUS_SUCCESS; 248 } 249 250 /* 251 * icp_sal_pollBank 252 * poll bank with id bank_number inside acceleration device with id @accelId 253 */ 254 CpaStatus 255 icp_sal_pollBank(Cpa32U accelId, Cpa32U bank_number, Cpa32U response_quota) 256 { 257 int ret; 258 259 ret = adf_poll_bank(accelId, bank_number, response_quota); 260 if (!ret) 261 return CPA_STATUS_SUCCESS; 262 else if (EAGAIN == ret) 263 return CPA_STATUS_RETRY; 264 265 return CPA_STATUS_FAIL; 266 } 267 268 /* 269 * icp_sal_pollAllBanks 270 * poll all banks inside acceleration device with id @accelId 271 */ 272 CpaStatus 273 icp_sal_pollAllBanks(Cpa32U accelId, Cpa32U response_quota) 274 { 275 int ret = 0; 276 277 ret = adf_poll_all_banks(accelId, response_quota); 278 if (!ret) 279 return CPA_STATUS_SUCCESS; 280 else if (ret == EAGAIN) 281 return CPA_STATUS_RETRY; 282 283 return CPA_STATUS_FAIL; 284 } 285 286 /* 287 * icp_adf_getQueueMemory 288 * Data plane support function - returns the pointer to next message on the ring 289 * or NULL if there is not enough space. 290 */ 291 void 292 icp_adf_getQueueMemory(icp_comms_trans_handle trans_handle, 293 Cpa32U numberRequests, 294 void **pCurrentQatMsg) 295 { 296 struct adf_etr_ring_data *ring = trans_handle; 297 Cpa64U flight; 298 299 ICP_CHECK_FOR_NULL_PARAM_VOID(ring); 300 301 /* Check if there is enough space in the ring */ 302 flight = atomic_add_return(numberRequests, ring->inflights); 303 if (flight > ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { 304 atomic_sub(numberRequests, ring->inflights); 305 *pCurrentQatMsg = NULL; 306 return; 307 } 308 309 /* We have enough space - get the address of next message */ 310 *pCurrentQatMsg = (void *)((uintptr_t)ring->base_addr + ring->tail); 311 } 312 313 /* 314 * icp_adf_getSingleQueueAddr 315 * Data plane support function - returns the pointer to next message on the ring 316 * or NULL if there is not enough space - it also updates the shadow tail copy. 317 */ 318 void 319 icp_adf_getSingleQueueAddr(icp_comms_trans_handle trans_handle, 320 void **pCurrentQatMsg) 321 { 322 struct adf_etr_ring_data *ring = trans_handle; 323 Cpa64U flight; 324 325 ICP_CHECK_FOR_NULL_PARAM_VOID(ring); 326 ICP_CHECK_FOR_NULL_PARAM_VOID(pCurrentQatMsg); 327 328 /* Check if there is enough space in the ring */ 329 flight = atomic_add_return(1, ring->inflights); 330 if (flight > ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { 331 atomic_dec(ring->inflights); 332 *pCurrentQatMsg = NULL; 333 return; 334 } 335 336 /* We have enough space - get the address of next message */ 337 *pCurrentQatMsg = (void *)((uintptr_t)ring->base_addr + ring->tail); 338 339 /* Update the shadow tail */ 340 ring->tail = 341 adf_modulo(ring->tail + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), 342 ADF_RING_SIZE_MODULO(ring->ring_size)); 343 } 344 345 /* 346 * icp_adf_getQueueNext 347 * Data plane support function - increments the tail pointer and returns 348 * the pointer to next message on the ring. 349 */ 350 void 351 icp_adf_getQueueNext(icp_comms_trans_handle trans_handle, void **pCurrentQatMsg) 352 { 353 struct adf_etr_ring_data *ring = trans_handle; 354 355 ICP_CHECK_FOR_NULL_PARAM_VOID(ring); 356 ICP_CHECK_FOR_NULL_PARAM_VOID(pCurrentQatMsg); 357 358 /* Increment tail to next message */ 359 ring->tail = 360 adf_modulo(ring->tail + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), 361 ADF_RING_SIZE_MODULO(ring->ring_size)); 362 363 /* Get the address of next message */ 364 *pCurrentQatMsg = (void *)((uintptr_t)ring->base_addr + ring->tail); 365 } 366 367 /* 368 * icp_adf_updateQueueTail 369 * Data plane support function - Writes the tail shadow copy to the device. 370 */ 371 void 372 icp_adf_updateQueueTail(icp_comms_trans_handle trans_handle) 373 { 374 struct adf_etr_ring_data *ring = trans_handle; 375 376 ICP_CHECK_FOR_NULL_PARAM_VOID(ring); 377 378 WRITE_CSR_RING_TAIL(ring->bank->csr_addr, 379 ring->bank->bank_number, 380 ring->ring_number, 381 ring->tail); 382 ring->csr_tail_offset = ring->tail; 383 } 384 385 /* 386 * icp_adf_pollQueue 387 * Data plane support function - Poll messages from the queue. 388 */ 389 CpaStatus 390 icp_adf_pollQueue(icp_comms_trans_handle trans_handle, Cpa32U response_quota) 391 { 392 Cpa32U num_resp; 393 struct adf_etr_ring_data *ring = trans_handle; 394 395 ICP_CHECK_FOR_NULL_PARAM(ring); 396 397 num_resp = adf_handle_response(ring, response_quota); 398 399 if (num_resp) 400 return CPA_STATUS_SUCCESS; 401 else 402 return CPA_STATUS_RETRY; 403 } 404 405 /* 406 * icp_adf_queueDataToSend 407 * Data-plane support function - Indicates if there is data on the ring to be 408 * sent. This should only be called on request rings. If the function returns 409 * true then it is ok to call icp_adf_updateQueueTail() function on this ring. 410 */ 411 CpaBoolean 412 icp_adf_queueDataToSend(icp_comms_trans_handle trans_handle) 413 { 414 struct adf_etr_ring_data *ring = trans_handle; 415 416 if (ring->tail != ring->csr_tail_offset) 417 return CPA_TRUE; 418 else 419 return CPA_FALSE; 420 } 421 422 /* 423 * This icp API won't be supported in kernel space currently 424 */ 425 CpaStatus 426 icp_adf_transGetFdForHandle(icp_comms_trans_handle trans_hnd, int *fd) 427 { 428 return CPA_STATUS_UNSUPPORTED; 429 } 430