1 /* SPDX-License-Identifier: GPL-2.0-only 2 * Copyright (C) 2020 Marvell. 3 */ 4 5 #ifndef __OTX2_CPT_REQMGR_H 6 #define __OTX2_CPT_REQMGR_H 7 8 #include "otx2_cpt_common.h" 9 10 /* Completion code size and initial value */ 11 #define OTX2_CPT_COMPLETION_CODE_SIZE 8 12 #define OTX2_CPT_COMPLETION_CODE_INIT OTX2_CPT_COMP_E_NOTDONE 13 /* 14 * Maximum total number of SG buffers is 100, we divide it equally 15 * between input and output 16 */ 17 #define OTX2_CPT_MAX_SG_IN_CNT 50 18 #define OTX2_CPT_MAX_SG_OUT_CNT 50 19 20 /* DMA mode direct or SG */ 21 #define OTX2_CPT_DMA_MODE_DIRECT 0 22 #define OTX2_CPT_DMA_MODE_SG 1 23 24 /* Context source CPTR or DPTR */ 25 #define OTX2_CPT_FROM_CPTR 0 26 #define OTX2_CPT_FROM_DPTR 1 27 28 #define OTX2_CPT_MAX_REQ_SIZE 65535 29 30 #define SG_COMPS_MAX 4 31 #define SGV2_COMPS_MAX 3 32 33 #define SG_COMP_3 3 34 #define SG_COMP_2 2 35 #define SG_COMP_1 1 36 37 union otx2_cpt_opcode { 38 u16 flags; 39 struct { 40 u8 major; 41 u8 minor; 42 } s; 43 }; 44 45 struct otx2_cptvf_request { 46 u32 param1; 47 u32 param2; 48 u16 dlen; 49 union otx2_cpt_opcode opcode; 50 dma_addr_t cptr_dma; 51 void *cptr; 52 }; 53 54 /* 55 * CPT_INST_S software command definitions 56 * Words EI (0-3) 57 */ 58 union otx2_cpt_iq_cmd_word0 { 59 u64 u; 60 struct { 61 __be16 opcode; 62 __be16 param1; 63 __be16 param2; 64 __be16 dlen; 65 } s; 66 }; 67 68 union otx2_cpt_iq_cmd_word3 { 69 u64 u; 70 struct { 71 u64 cptr:61; 72 u64 grp:3; 73 } s; 74 }; 75 76 struct otx2_cpt_iq_command { 77 union otx2_cpt_iq_cmd_word0 cmd; 78 u64 dptr; 79 u64 rptr; 80 union otx2_cpt_iq_cmd_word3 cptr; 81 }; 82 83 struct otx2_cpt_pending_entry { 84 void *completion_addr; /* Completion address */ 85 void *info; 86 /* Kernel async request callback */ 87 void (*callback)(int status, void *arg1, void *arg2); 88 struct crypto_async_request *areq; /* Async request callback arg */ 89 u8 resume_sender; /* Notify sender to resume sending requests */ 90 u8 busy; /* Entry status (free/busy) */ 91 }; 92 93 struct otx2_cpt_pending_queue { 94 struct otx2_cpt_pending_entry *head; /* Head of the queue */ 95 u32 front; /* Process work from here */ 96 u32 rear; /* Append new work here */ 97 u32 pending_count; /* Pending requests count */ 98 u32 qlen; /* Queue length */ 99 spinlock_t lock; /* Queue lock */ 100 }; 101 102 struct otx2_cpt_buf_ptr { 103 u8 *vptr; 104 dma_addr_t dma_addr; 105 u16 size; 106 }; 107 108 union otx2_cpt_ctrl_info { 109 u32 flags; 110 struct { 111 #if defined(__BIG_ENDIAN_BITFIELD) 112 u32 reserved_6_31:26; 113 u32 grp:3; /* Group bits */ 114 u32 dma_mode:2; /* DMA mode */ 115 u32 se_req:1; /* To SE core */ 116 #else 117 u32 se_req:1; /* To SE core */ 118 u32 dma_mode:2; /* DMA mode */ 119 u32 grp:3; /* Group bits */ 120 u32 reserved_6_31:26; 121 #endif 122 } s; 123 }; 124 125 struct otx2_cpt_req_info { 126 /* Kernel async request callback */ 127 void (*callback)(int status, void *arg1, void *arg2); 128 struct crypto_async_request *areq; /* Async request callback arg */ 129 struct otx2_cptvf_request req;/* Request information (core specific) */ 130 union otx2_cpt_ctrl_info ctrl;/* User control information */ 131 struct otx2_cpt_buf_ptr in[OTX2_CPT_MAX_SG_IN_CNT]; 132 struct otx2_cpt_buf_ptr out[OTX2_CPT_MAX_SG_OUT_CNT]; 133 u8 *iv_out; /* IV to send back */ 134 u16 rlen; /* Output length */ 135 u8 in_cnt; /* Number of input buffers */ 136 u8 out_cnt; /* Number of output buffers */ 137 u8 req_type; /* Type of request */ 138 u8 is_enc; /* Is a request an encryption request */ 139 u8 is_trunc_hmac;/* Is truncated hmac used */ 140 }; 141 142 struct otx2_cpt_inst_info { 143 struct otx2_cpt_pending_entry *pentry; 144 struct otx2_cpt_req_info *req; 145 struct pci_dev *pdev; 146 void *completion_addr; 147 u8 *out_buffer; 148 u8 *in_buffer; 149 dma_addr_t dptr_baddr; 150 dma_addr_t rptr_baddr; 151 dma_addr_t comp_baddr; 152 unsigned long time_in; 153 u32 dlen; 154 u32 dma_len; 155 u64 gthr_sz; 156 u64 sctr_sz; 157 u8 extra_time; 158 }; 159 160 struct otx2_cpt_sglist_component { 161 __be16 len0; 162 __be16 len1; 163 __be16 len2; 164 __be16 len3; 165 __be64 ptr0; 166 __be64 ptr1; 167 __be64 ptr2; 168 __be64 ptr3; 169 }; 170 171 struct cn10kb_cpt_sglist_component { 172 u16 len0; 173 u16 len1; 174 u16 len2; 175 u16 valid_segs; 176 u64 ptr0; 177 u64 ptr1; 178 u64 ptr2; 179 }; 180 181 static inline void otx2_cpt_info_destroy(struct pci_dev *pdev, 182 struct otx2_cpt_inst_info *info) 183 { 184 struct otx2_cpt_req_info *req; 185 int i; 186 187 if (info->dptr_baddr) 188 dma_unmap_single(&pdev->dev, info->dptr_baddr, 189 info->dma_len, DMA_BIDIRECTIONAL); 190 191 if (info->req) { 192 req = info->req; 193 for (i = 0; i < req->out_cnt; i++) { 194 if (req->out[i].dma_addr) 195 dma_unmap_single(&pdev->dev, 196 req->out[i].dma_addr, 197 req->out[i].size, 198 DMA_BIDIRECTIONAL); 199 } 200 201 for (i = 0; i < req->in_cnt; i++) { 202 if (req->in[i].dma_addr) 203 dma_unmap_single(&pdev->dev, 204 req->in[i].dma_addr, 205 req->in[i].size, 206 DMA_BIDIRECTIONAL); 207 } 208 } 209 kfree(info); 210 } 211 212 static inline int setup_sgio_components(struct pci_dev *pdev, 213 struct otx2_cpt_buf_ptr *list, 214 int buf_count, u8 *buffer) 215 { 216 struct otx2_cpt_sglist_component *sg_ptr; 217 int components; 218 int i, j; 219 220 if (unlikely(!list)) { 221 dev_err(&pdev->dev, "Input list pointer is NULL\n"); 222 return -EINVAL; 223 } 224 225 for (i = 0; i < buf_count; i++) { 226 if (unlikely(!list[i].vptr)) 227 continue; 228 list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr, 229 list[i].size, 230 DMA_BIDIRECTIONAL); 231 if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) { 232 dev_err(&pdev->dev, "Dma mapping failed\n"); 233 goto sg_cleanup; 234 } 235 } 236 components = buf_count / SG_COMPS_MAX; 237 sg_ptr = (struct otx2_cpt_sglist_component *)buffer; 238 for (i = 0; i < components; i++) { 239 sg_ptr->len0 = cpu_to_be16(list[i * SG_COMPS_MAX + 0].size); 240 sg_ptr->len1 = cpu_to_be16(list[i * SG_COMPS_MAX + 1].size); 241 sg_ptr->len2 = cpu_to_be16(list[i * SG_COMPS_MAX + 2].size); 242 sg_ptr->len3 = cpu_to_be16(list[i * SG_COMPS_MAX + 3].size); 243 sg_ptr->ptr0 = cpu_to_be64(list[i * SG_COMPS_MAX + 0].dma_addr); 244 sg_ptr->ptr1 = cpu_to_be64(list[i * SG_COMPS_MAX + 1].dma_addr); 245 sg_ptr->ptr2 = cpu_to_be64(list[i * SG_COMPS_MAX + 2].dma_addr); 246 sg_ptr->ptr3 = cpu_to_be64(list[i * SG_COMPS_MAX + 3].dma_addr); 247 sg_ptr++; 248 } 249 components = buf_count % SG_COMPS_MAX; 250 251 switch (components) { 252 case SG_COMP_3: 253 sg_ptr->len2 = cpu_to_be16(list[i * SG_COMPS_MAX + 2].size); 254 sg_ptr->ptr2 = cpu_to_be64(list[i * SG_COMPS_MAX + 2].dma_addr); 255 fallthrough; 256 case SG_COMP_2: 257 sg_ptr->len1 = cpu_to_be16(list[i * SG_COMPS_MAX + 1].size); 258 sg_ptr->ptr1 = cpu_to_be64(list[i * SG_COMPS_MAX + 1].dma_addr); 259 fallthrough; 260 case SG_COMP_1: 261 sg_ptr->len0 = cpu_to_be16(list[i * SG_COMPS_MAX + 0].size); 262 sg_ptr->ptr0 = cpu_to_be64(list[i * SG_COMPS_MAX + 0].dma_addr); 263 break; 264 default: 265 break; 266 } 267 return 0; 268 269 sg_cleanup: 270 for (j = 0; j < i; j++) { 271 if (list[j].dma_addr) { 272 dma_unmap_single(&pdev->dev, list[j].dma_addr, 273 list[j].size, DMA_BIDIRECTIONAL); 274 } 275 276 list[j].dma_addr = 0; 277 } 278 return -EIO; 279 } 280 281 static inline int sgv2io_components_setup(struct pci_dev *pdev, 282 struct otx2_cpt_buf_ptr *list, 283 int buf_count, u8 *buffer) 284 { 285 struct cn10kb_cpt_sglist_component *sg_ptr; 286 int components; 287 int i, j; 288 289 if (unlikely(!list)) { 290 dev_err(&pdev->dev, "Input list pointer is NULL\n"); 291 return -EFAULT; 292 } 293 294 for (i = 0; i < buf_count; i++) { 295 if (unlikely(!list[i].vptr)) 296 continue; 297 list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr, 298 list[i].size, 299 DMA_BIDIRECTIONAL); 300 if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) { 301 dev_err(&pdev->dev, "Dma mapping failed\n"); 302 goto sg_cleanup; 303 } 304 } 305 components = buf_count / SGV2_COMPS_MAX; 306 sg_ptr = (struct cn10kb_cpt_sglist_component *)buffer; 307 for (i = 0; i < components; i++) { 308 sg_ptr->len0 = list[i * SGV2_COMPS_MAX + 0].size; 309 sg_ptr->len1 = list[i * SGV2_COMPS_MAX + 1].size; 310 sg_ptr->len2 = list[i * SGV2_COMPS_MAX + 2].size; 311 sg_ptr->ptr0 = list[i * SGV2_COMPS_MAX + 0].dma_addr; 312 sg_ptr->ptr1 = list[i * SGV2_COMPS_MAX + 1].dma_addr; 313 sg_ptr->ptr2 = list[i * SGV2_COMPS_MAX + 2].dma_addr; 314 sg_ptr->valid_segs = SGV2_COMPS_MAX; 315 sg_ptr++; 316 } 317 components = buf_count % SGV2_COMPS_MAX; 318 319 sg_ptr->valid_segs = components; 320 switch (components) { 321 case SG_COMP_2: 322 sg_ptr->len1 = list[i * SGV2_COMPS_MAX + 1].size; 323 sg_ptr->ptr1 = list[i * SGV2_COMPS_MAX + 1].dma_addr; 324 fallthrough; 325 case SG_COMP_1: 326 sg_ptr->len0 = list[i * SGV2_COMPS_MAX + 0].size; 327 sg_ptr->ptr0 = list[i * SGV2_COMPS_MAX + 0].dma_addr; 328 break; 329 default: 330 break; 331 } 332 return 0; 333 334 sg_cleanup: 335 for (j = 0; j < i; j++) { 336 if (list[j].dma_addr) { 337 dma_unmap_single(&pdev->dev, list[j].dma_addr, 338 list[j].size, DMA_BIDIRECTIONAL); 339 } 340 341 list[j].dma_addr = 0; 342 } 343 return -EIO; 344 } 345 346 static inline struct otx2_cpt_inst_info * 347 cn10k_sgv2_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, 348 gfp_t gfp) 349 { 350 u32 dlen = 0, g_len, sg_len, info_len; 351 int align = OTX2_CPT_DMA_MINALIGN; 352 struct otx2_cpt_inst_info *info; 353 u16 g_sz_bytes, s_sz_bytes; 354 u32 total_mem_len; 355 int i; 356 357 g_sz_bytes = ((req->in_cnt + 2) / 3) * 358 sizeof(struct cn10kb_cpt_sglist_component); 359 s_sz_bytes = ((req->out_cnt + 2) / 3) * 360 sizeof(struct cn10kb_cpt_sglist_component); 361 362 g_len = ALIGN(g_sz_bytes, align); 363 sg_len = ALIGN(g_len + s_sz_bytes, align); 364 info_len = ALIGN(sizeof(*info), align); 365 total_mem_len = sg_len + info_len + sizeof(union otx2_cpt_res_s); 366 367 info = kzalloc(total_mem_len, gfp); 368 if (unlikely(!info)) 369 return NULL; 370 371 for (i = 0; i < req->in_cnt; i++) 372 dlen += req->in[i].size; 373 374 info->dlen = dlen; 375 info->in_buffer = (u8 *)info + info_len; 376 info->gthr_sz = req->in_cnt; 377 info->sctr_sz = req->out_cnt; 378 379 /* Setup gather (input) components */ 380 if (sgv2io_components_setup(pdev, req->in, req->in_cnt, 381 info->in_buffer)) { 382 dev_err(&pdev->dev, "Failed to setup gather list\n"); 383 goto destroy_info; 384 } 385 386 if (sgv2io_components_setup(pdev, req->out, req->out_cnt, 387 &info->in_buffer[g_len])) { 388 dev_err(&pdev->dev, "Failed to setup scatter list\n"); 389 goto destroy_info; 390 } 391 392 info->dma_len = total_mem_len - info_len; 393 info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer, 394 info->dma_len, DMA_BIDIRECTIONAL); 395 if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { 396 dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); 397 goto destroy_info; 398 } 399 info->rptr_baddr = info->dptr_baddr + g_len; 400 /* 401 * Get buffer for union otx2_cpt_res_s response 402 * structure and its physical address 403 */ 404 info->completion_addr = info->in_buffer + sg_len; 405 info->comp_baddr = info->dptr_baddr + sg_len; 406 407 return info; 408 409 destroy_info: 410 otx2_cpt_info_destroy(pdev, info); 411 return NULL; 412 } 413 414 /* SG list header size in bytes */ 415 #define SG_LIST_HDR_SIZE 8 416 static inline struct otx2_cpt_inst_info * 417 otx2_sg_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, 418 gfp_t gfp) 419 { 420 int align = OTX2_CPT_DMA_MINALIGN; 421 struct otx2_cpt_inst_info *info; 422 u32 dlen, align_dlen, info_len; 423 u16 g_sz_bytes, s_sz_bytes; 424 u32 total_mem_len; 425 426 if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT || 427 req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) { 428 dev_err(&pdev->dev, "Error too many sg components\n"); 429 return NULL; 430 } 431 432 g_sz_bytes = ((req->in_cnt + 3) / 4) * 433 sizeof(struct otx2_cpt_sglist_component); 434 s_sz_bytes = ((req->out_cnt + 3) / 4) * 435 sizeof(struct otx2_cpt_sglist_component); 436 437 dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; 438 align_dlen = ALIGN(dlen, align); 439 info_len = ALIGN(sizeof(*info), align); 440 total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s); 441 442 info = kzalloc(total_mem_len, gfp); 443 if (unlikely(!info)) 444 return NULL; 445 446 info->dlen = dlen; 447 info->in_buffer = (u8 *)info + info_len; 448 449 ((u16 *)info->in_buffer)[0] = req->out_cnt; 450 ((u16 *)info->in_buffer)[1] = req->in_cnt; 451 ((u16 *)info->in_buffer)[2] = 0; 452 ((u16 *)info->in_buffer)[3] = 0; 453 cpu_to_be64s((u64 *)info->in_buffer); 454 455 /* Setup gather (input) components */ 456 if (setup_sgio_components(pdev, req->in, req->in_cnt, 457 &info->in_buffer[8])) { 458 dev_err(&pdev->dev, "Failed to setup gather list\n"); 459 goto destroy_info; 460 } 461 462 if (setup_sgio_components(pdev, req->out, req->out_cnt, 463 &info->in_buffer[8 + g_sz_bytes])) { 464 dev_err(&pdev->dev, "Failed to setup scatter list\n"); 465 goto destroy_info; 466 } 467 468 info->dma_len = total_mem_len - info_len; 469 info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer, 470 info->dma_len, DMA_BIDIRECTIONAL); 471 if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { 472 dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); 473 goto destroy_info; 474 } 475 /* 476 * Get buffer for union otx2_cpt_res_s response 477 * structure and its physical address 478 */ 479 info->completion_addr = info->in_buffer + align_dlen; 480 info->comp_baddr = info->dptr_baddr + align_dlen; 481 482 return info; 483 484 destroy_info: 485 otx2_cpt_info_destroy(pdev, info); 486 return NULL; 487 } 488 489 struct otx2_cptlf_wqe; 490 int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, 491 int cpu_num); 492 void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe); 493 int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev); 494 495 #endif /* __OTX2_CPT_REQMGR_H */ 496