1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2021 Google, Inc. 5 */ 6 7 #include <linux/etherdevice.h> 8 #include <linux/pci.h> 9 #include "gve.h" 10 #include "gve_adminq.h" 11 #include "gve_register.h" 12 13 #define GVE_MAX_ADMINQ_RELEASE_CHECK 500 14 #define GVE_ADMINQ_SLEEP_LEN 20 15 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK 100 16 17 #define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \ 18 "Expected: length=%d, feature_mask=%x.\n" \ 19 "Actual: length=%d, feature_mask=%x.\n" 20 21 #define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n" 22 23 static 24 struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor, 25 struct gve_device_option *option) 26 { 27 void *option_end, *descriptor_end; 28 29 option_end = (void *)(option + 1) + be16_to_cpu(option->option_length); 30 descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length); 31 32 return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end; 33 } 34 35 static 36 void gve_parse_device_option(struct gve_priv *priv, 37 struct gve_device_descriptor *device_descriptor, 38 struct gve_device_option *option, 39 struct gve_device_option_gqi_rda **dev_op_gqi_rda, 40 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, 41 struct gve_device_option_dqo_rda **dev_op_dqo_rda, 42 struct gve_device_option_jumbo_frames **dev_op_jumbo_frames) 43 { 44 u32 req_feat_mask = be32_to_cpu(option->required_features_mask); 45 u16 option_length = be16_to_cpu(option->option_length); 46 u16 option_id = be16_to_cpu(option->option_id); 47 48 /* If the length or feature mask doesn't match, continue without 49 * enabling the feature. 50 */ 51 switch (option_id) { 52 case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING: 53 if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING || 54 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) { 55 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 56 "Raw Addressing", 57 GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING, 58 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING, 59 option_length, req_feat_mask); 60 break; 61 } 62 63 dev_info(&priv->pdev->dev, 64 "Gqi raw addressing device option enabled.\n"); 65 priv->queue_format = GVE_GQI_RDA_FORMAT; 66 break; 67 case GVE_DEV_OPT_ID_GQI_RDA: 68 if (option_length < sizeof(**dev_op_gqi_rda) || 69 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) { 70 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 71 "GQI RDA", (int)sizeof(**dev_op_gqi_rda), 72 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA, 73 option_length, req_feat_mask); 74 break; 75 } 76 77 if (option_length > sizeof(**dev_op_gqi_rda)) { 78 dev_warn(&priv->pdev->dev, 79 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA"); 80 } 81 *dev_op_gqi_rda = (void *)(option + 1); 82 break; 83 case GVE_DEV_OPT_ID_GQI_QPL: 84 if (option_length < sizeof(**dev_op_gqi_qpl) || 85 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) { 86 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 87 "GQI QPL", (int)sizeof(**dev_op_gqi_qpl), 88 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL, 89 option_length, req_feat_mask); 90 break; 91 } 92 93 if (option_length > sizeof(**dev_op_gqi_qpl)) { 94 dev_warn(&priv->pdev->dev, 95 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL"); 96 } 97 *dev_op_gqi_qpl = (void *)(option + 1); 98 break; 99 case GVE_DEV_OPT_ID_DQO_RDA: 100 if (option_length < sizeof(**dev_op_dqo_rda) || 101 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) { 102 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 103 "DQO RDA", (int)sizeof(**dev_op_dqo_rda), 104 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA, 105 option_length, req_feat_mask); 106 break; 107 } 108 109 if (option_length > sizeof(**dev_op_dqo_rda)) { 110 dev_warn(&priv->pdev->dev, 111 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA"); 112 } 113 *dev_op_dqo_rda = (void *)(option + 1); 114 break; 115 case GVE_DEV_OPT_ID_JUMBO_FRAMES: 116 if (option_length < sizeof(**dev_op_jumbo_frames) || 117 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) { 118 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 119 "Jumbo Frames", 120 (int)sizeof(**dev_op_jumbo_frames), 121 GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES, 122 option_length, req_feat_mask); 123 break; 124 } 125 126 if (option_length > sizeof(**dev_op_jumbo_frames)) { 127 dev_warn(&priv->pdev->dev, 128 GVE_DEVICE_OPTION_TOO_BIG_FMT, 129 "Jumbo Frames"); 130 } 131 *dev_op_jumbo_frames = (void *)(option + 1); 132 break; 133 default: 134 /* If we don't recognize the option just continue 135 * without doing anything. 136 */ 137 dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n", 138 option_id); 139 } 140 } 141 142 /* Process all device options for a given describe device call. */ 143 static int 144 gve_process_device_options(struct gve_priv *priv, 145 struct gve_device_descriptor *descriptor, 146 struct gve_device_option_gqi_rda **dev_op_gqi_rda, 147 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, 148 struct gve_device_option_dqo_rda **dev_op_dqo_rda, 149 struct gve_device_option_jumbo_frames **dev_op_jumbo_frames) 150 { 151 const int num_options = be16_to_cpu(descriptor->num_device_options); 152 struct gve_device_option *dev_opt; 153 int i; 154 155 /* The options struct directly follows the device descriptor. */ 156 dev_opt = (void *)(descriptor + 1); 157 for (i = 0; i < num_options; i++) { 158 struct gve_device_option *next_opt; 159 160 next_opt = gve_get_next_option(descriptor, dev_opt); 161 if (!next_opt) { 162 dev_err(&priv->dev->dev, 163 "options exceed device_descriptor's total length.\n"); 164 return -EINVAL; 165 } 166 167 gve_parse_device_option(priv, descriptor, dev_opt, 168 dev_op_gqi_rda, dev_op_gqi_qpl, 169 dev_op_dqo_rda, dev_op_jumbo_frames); 170 dev_opt = next_opt; 171 } 172 173 return 0; 174 } 175 176 int gve_adminq_alloc(struct device *dev, struct gve_priv *priv) 177 { 178 priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE, 179 &priv->adminq_bus_addr, GFP_KERNEL); 180 if (unlikely(!priv->adminq)) 181 return -ENOMEM; 182 183 priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1; 184 priv->adminq_prod_cnt = 0; 185 priv->adminq_cmd_fail = 0; 186 priv->adminq_timeouts = 0; 187 priv->adminq_describe_device_cnt = 0; 188 priv->adminq_cfg_device_resources_cnt = 0; 189 priv->adminq_register_page_list_cnt = 0; 190 priv->adminq_unregister_page_list_cnt = 0; 191 priv->adminq_create_tx_queue_cnt = 0; 192 priv->adminq_create_rx_queue_cnt = 0; 193 priv->adminq_destroy_tx_queue_cnt = 0; 194 priv->adminq_destroy_rx_queue_cnt = 0; 195 priv->adminq_dcfg_device_resources_cnt = 0; 196 priv->adminq_set_driver_parameter_cnt = 0; 197 priv->adminq_report_stats_cnt = 0; 198 priv->adminq_report_link_speed_cnt = 0; 199 priv->adminq_get_ptype_map_cnt = 0; 200 201 /* Setup Admin queue with the device */ 202 iowrite32be(priv->adminq_bus_addr / PAGE_SIZE, 203 &priv->reg_bar0->adminq_pfn); 204 205 gve_set_admin_queue_ok(priv); 206 return 0; 207 } 208 209 void gve_adminq_release(struct gve_priv *priv) 210 { 211 int i = 0; 212 213 /* Tell the device the adminq is leaving */ 214 iowrite32be(0x0, &priv->reg_bar0->adminq_pfn); 215 while (ioread32be(&priv->reg_bar0->adminq_pfn)) { 216 /* If this is reached the device is unrecoverable and still 217 * holding memory. Continue looping to avoid memory corruption, 218 * but WARN so it is visible what is going on. 219 */ 220 if (i == GVE_MAX_ADMINQ_RELEASE_CHECK) 221 WARN(1, "Unrecoverable platform error!"); 222 i++; 223 msleep(GVE_ADMINQ_SLEEP_LEN); 224 } 225 gve_clear_device_rings_ok(priv); 226 gve_clear_device_resources_ok(priv); 227 gve_clear_admin_queue_ok(priv); 228 } 229 230 void gve_adminq_free(struct device *dev, struct gve_priv *priv) 231 { 232 if (!gve_get_admin_queue_ok(priv)) 233 return; 234 gve_adminq_release(priv); 235 dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr); 236 gve_clear_admin_queue_ok(priv); 237 } 238 239 static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt) 240 { 241 iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell); 242 } 243 244 static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt) 245 { 246 int i; 247 248 for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) { 249 if (ioread32be(&priv->reg_bar0->adminq_event_counter) 250 == prod_cnt) 251 return true; 252 msleep(GVE_ADMINQ_SLEEP_LEN); 253 } 254 255 return false; 256 } 257 258 static int gve_adminq_parse_err(struct gve_priv *priv, u32 status) 259 { 260 if (status != GVE_ADMINQ_COMMAND_PASSED && 261 status != GVE_ADMINQ_COMMAND_UNSET) { 262 dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status); 263 priv->adminq_cmd_fail++; 264 } 265 switch (status) { 266 case GVE_ADMINQ_COMMAND_PASSED: 267 return 0; 268 case GVE_ADMINQ_COMMAND_UNSET: 269 dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n"); 270 return -EINVAL; 271 case GVE_ADMINQ_COMMAND_ERROR_ABORTED: 272 case GVE_ADMINQ_COMMAND_ERROR_CANCELLED: 273 case GVE_ADMINQ_COMMAND_ERROR_DATALOSS: 274 case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION: 275 case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE: 276 return -EAGAIN; 277 case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS: 278 case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR: 279 case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT: 280 case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND: 281 case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE: 282 case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR: 283 return -EINVAL; 284 case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED: 285 return -ETIME; 286 case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED: 287 case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED: 288 return -EACCES; 289 case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED: 290 return -ENOMEM; 291 case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED: 292 return -ENOTSUPP; 293 default: 294 dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status); 295 return -EINVAL; 296 } 297 } 298 299 /* Flushes all AQ commands currently queued and waits for them to complete. 300 * If there are failures, it will return the first error. 301 */ 302 static int gve_adminq_kick_and_wait(struct gve_priv *priv) 303 { 304 int tail, head; 305 int i; 306 307 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 308 head = priv->adminq_prod_cnt; 309 310 gve_adminq_kick_cmd(priv, head); 311 if (!gve_adminq_wait_for_cmd(priv, head)) { 312 dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n"); 313 priv->adminq_timeouts++; 314 return -ENOTRECOVERABLE; 315 } 316 317 for (i = tail; i < head; i++) { 318 union gve_adminq_command *cmd; 319 u32 status, err; 320 321 cmd = &priv->adminq[i & priv->adminq_mask]; 322 status = be32_to_cpu(READ_ONCE(cmd->status)); 323 err = gve_adminq_parse_err(priv, status); 324 if (err) 325 // Return the first error if we failed. 326 return err; 327 } 328 329 return 0; 330 } 331 332 /* This function is not threadsafe - the caller is responsible for any 333 * necessary locks. 334 */ 335 static int gve_adminq_issue_cmd(struct gve_priv *priv, 336 union gve_adminq_command *cmd_orig) 337 { 338 union gve_adminq_command *cmd; 339 u32 opcode; 340 u32 tail; 341 342 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 343 344 // Check if next command will overflow the buffer. 345 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) == 346 (tail & priv->adminq_mask)) { 347 int err; 348 349 // Flush existing commands to make room. 350 err = gve_adminq_kick_and_wait(priv); 351 if (err) 352 return err; 353 354 // Retry. 355 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 356 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) == 357 (tail & priv->adminq_mask)) { 358 // This should never happen. We just flushed the 359 // command queue so there should be enough space. 360 return -ENOMEM; 361 } 362 } 363 364 cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask]; 365 priv->adminq_prod_cnt++; 366 367 memcpy(cmd, cmd_orig, sizeof(*cmd_orig)); 368 opcode = be32_to_cpu(READ_ONCE(cmd->opcode)); 369 370 switch (opcode) { 371 case GVE_ADMINQ_DESCRIBE_DEVICE: 372 priv->adminq_describe_device_cnt++; 373 break; 374 case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES: 375 priv->adminq_cfg_device_resources_cnt++; 376 break; 377 case GVE_ADMINQ_REGISTER_PAGE_LIST: 378 priv->adminq_register_page_list_cnt++; 379 break; 380 case GVE_ADMINQ_UNREGISTER_PAGE_LIST: 381 priv->adminq_unregister_page_list_cnt++; 382 break; 383 case GVE_ADMINQ_CREATE_TX_QUEUE: 384 priv->adminq_create_tx_queue_cnt++; 385 break; 386 case GVE_ADMINQ_CREATE_RX_QUEUE: 387 priv->adminq_create_rx_queue_cnt++; 388 break; 389 case GVE_ADMINQ_DESTROY_TX_QUEUE: 390 priv->adminq_destroy_tx_queue_cnt++; 391 break; 392 case GVE_ADMINQ_DESTROY_RX_QUEUE: 393 priv->adminq_destroy_rx_queue_cnt++; 394 break; 395 case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES: 396 priv->adminq_dcfg_device_resources_cnt++; 397 break; 398 case GVE_ADMINQ_SET_DRIVER_PARAMETER: 399 priv->adminq_set_driver_parameter_cnt++; 400 break; 401 case GVE_ADMINQ_REPORT_STATS: 402 priv->adminq_report_stats_cnt++; 403 break; 404 case GVE_ADMINQ_REPORT_LINK_SPEED: 405 priv->adminq_report_link_speed_cnt++; 406 break; 407 case GVE_ADMINQ_GET_PTYPE_MAP: 408 priv->adminq_get_ptype_map_cnt++; 409 break; 410 default: 411 dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode); 412 } 413 414 return 0; 415 } 416 417 /* This function is not threadsafe - the caller is responsible for any 418 * necessary locks. 419 * The caller is also responsible for making sure there are no commands 420 * waiting to be executed. 421 */ 422 static int gve_adminq_execute_cmd(struct gve_priv *priv, 423 union gve_adminq_command *cmd_orig) 424 { 425 u32 tail, head; 426 int err; 427 428 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 429 head = priv->adminq_prod_cnt; 430 if (tail != head) 431 // This is not a valid path 432 return -EINVAL; 433 434 err = gve_adminq_issue_cmd(priv, cmd_orig); 435 if (err) 436 return err; 437 438 return gve_adminq_kick_and_wait(priv); 439 } 440 441 /* The device specifies that the management vector can either be the first irq 442 * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to 443 * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then 444 * the management vector is first. 445 * 446 * gve arranges the msix vectors so that the management vector is last. 447 */ 448 #define GVE_NTFY_BLK_BASE_MSIX_IDX 0 449 int gve_adminq_configure_device_resources(struct gve_priv *priv, 450 dma_addr_t counter_array_bus_addr, 451 u32 num_counters, 452 dma_addr_t db_array_bus_addr, 453 u32 num_ntfy_blks) 454 { 455 union gve_adminq_command cmd; 456 457 memset(&cmd, 0, sizeof(cmd)); 458 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES); 459 cmd.configure_device_resources = 460 (struct gve_adminq_configure_device_resources) { 461 .counter_array = cpu_to_be64(counter_array_bus_addr), 462 .num_counters = cpu_to_be32(num_counters), 463 .irq_db_addr = cpu_to_be64(db_array_bus_addr), 464 .num_irq_dbs = cpu_to_be32(num_ntfy_blks), 465 .irq_db_stride = cpu_to_be32(sizeof(*priv->irq_db_indices)), 466 .ntfy_blk_msix_base_idx = 467 cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX), 468 .queue_format = priv->queue_format, 469 }; 470 471 return gve_adminq_execute_cmd(priv, &cmd); 472 } 473 474 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv) 475 { 476 union gve_adminq_command cmd; 477 478 memset(&cmd, 0, sizeof(cmd)); 479 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES); 480 481 return gve_adminq_execute_cmd(priv, &cmd); 482 } 483 484 static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index) 485 { 486 struct gve_tx_ring *tx = &priv->tx[queue_index]; 487 union gve_adminq_command cmd; 488 489 memset(&cmd, 0, sizeof(cmd)); 490 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE); 491 cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) { 492 .queue_id = cpu_to_be32(queue_index), 493 .queue_resources_addr = 494 cpu_to_be64(tx->q_resources_bus), 495 .tx_ring_addr = cpu_to_be64(tx->bus), 496 .ntfy_id = cpu_to_be32(tx->ntfy_id), 497 }; 498 499 if (gve_is_gqi(priv)) { 500 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? 501 GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id; 502 503 cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 504 } else { 505 cmd.create_tx_queue.tx_ring_size = 506 cpu_to_be16(priv->tx_desc_cnt); 507 cmd.create_tx_queue.tx_comp_ring_addr = 508 cpu_to_be64(tx->complq_bus_dqo); 509 cmd.create_tx_queue.tx_comp_ring_size = 510 cpu_to_be16(priv->options_dqo_rda.tx_comp_ring_entries); 511 } 512 513 return gve_adminq_issue_cmd(priv, &cmd); 514 } 515 516 int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 num_queues) 517 { 518 int err; 519 int i; 520 521 for (i = 0; i < num_queues; i++) { 522 err = gve_adminq_create_tx_queue(priv, i); 523 if (err) 524 return err; 525 } 526 527 return gve_adminq_kick_and_wait(priv); 528 } 529 530 static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index) 531 { 532 struct gve_rx_ring *rx = &priv->rx[queue_index]; 533 union gve_adminq_command cmd; 534 535 memset(&cmd, 0, sizeof(cmd)); 536 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE); 537 cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) { 538 .queue_id = cpu_to_be32(queue_index), 539 .ntfy_id = cpu_to_be32(rx->ntfy_id), 540 .queue_resources_addr = cpu_to_be64(rx->q_resources_bus), 541 }; 542 543 if (gve_is_gqi(priv)) { 544 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? 545 GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id; 546 547 cmd.create_rx_queue.rx_desc_ring_addr = 548 cpu_to_be64(rx->desc.bus), 549 cmd.create_rx_queue.rx_data_ring_addr = 550 cpu_to_be64(rx->data.data_bus), 551 cmd.create_rx_queue.index = cpu_to_be32(queue_index); 552 cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 553 cmd.create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size); 554 } else { 555 cmd.create_rx_queue.rx_ring_size = 556 cpu_to_be16(priv->rx_desc_cnt); 557 cmd.create_rx_queue.rx_desc_ring_addr = 558 cpu_to_be64(rx->dqo.complq.bus); 559 cmd.create_rx_queue.rx_data_ring_addr = 560 cpu_to_be64(rx->dqo.bufq.bus); 561 cmd.create_rx_queue.packet_buffer_size = 562 cpu_to_be16(priv->data_buffer_size_dqo); 563 cmd.create_rx_queue.rx_buff_ring_size = 564 cpu_to_be16(priv->options_dqo_rda.rx_buff_ring_entries); 565 cmd.create_rx_queue.enable_rsc = 566 !!(priv->dev->features & NETIF_F_LRO); 567 } 568 569 return gve_adminq_issue_cmd(priv, &cmd); 570 } 571 572 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues) 573 { 574 int err; 575 int i; 576 577 for (i = 0; i < num_queues; i++) { 578 err = gve_adminq_create_rx_queue(priv, i); 579 if (err) 580 return err; 581 } 582 583 return gve_adminq_kick_and_wait(priv); 584 } 585 586 static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index) 587 { 588 union gve_adminq_command cmd; 589 int err; 590 591 memset(&cmd, 0, sizeof(cmd)); 592 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE); 593 cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) { 594 .queue_id = cpu_to_be32(queue_index), 595 }; 596 597 err = gve_adminq_issue_cmd(priv, &cmd); 598 if (err) 599 return err; 600 601 return 0; 602 } 603 604 int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 num_queues) 605 { 606 int err; 607 int i; 608 609 for (i = 0; i < num_queues; i++) { 610 err = gve_adminq_destroy_tx_queue(priv, i); 611 if (err) 612 return err; 613 } 614 615 return gve_adminq_kick_and_wait(priv); 616 } 617 618 static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index) 619 { 620 union gve_adminq_command cmd; 621 int err; 622 623 memset(&cmd, 0, sizeof(cmd)); 624 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE); 625 cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) { 626 .queue_id = cpu_to_be32(queue_index), 627 }; 628 629 err = gve_adminq_issue_cmd(priv, &cmd); 630 if (err) 631 return err; 632 633 return 0; 634 } 635 636 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues) 637 { 638 int err; 639 int i; 640 641 for (i = 0; i < num_queues; i++) { 642 err = gve_adminq_destroy_rx_queue(priv, i); 643 if (err) 644 return err; 645 } 646 647 return gve_adminq_kick_and_wait(priv); 648 } 649 650 static int gve_set_desc_cnt(struct gve_priv *priv, 651 struct gve_device_descriptor *descriptor) 652 { 653 priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); 654 if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) { 655 dev_err(&priv->pdev->dev, "Tx desc count %d too low\n", 656 priv->tx_desc_cnt); 657 return -EINVAL; 658 } 659 priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); 660 if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0]) 661 < PAGE_SIZE) { 662 dev_err(&priv->pdev->dev, "Rx desc count %d too low\n", 663 priv->rx_desc_cnt); 664 return -EINVAL; 665 } 666 return 0; 667 } 668 669 static int 670 gve_set_desc_cnt_dqo(struct gve_priv *priv, 671 const struct gve_device_descriptor *descriptor, 672 const struct gve_device_option_dqo_rda *dev_op_dqo_rda) 673 { 674 priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); 675 priv->options_dqo_rda.tx_comp_ring_entries = 676 be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries); 677 priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); 678 priv->options_dqo_rda.rx_buff_ring_entries = 679 be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries); 680 681 return 0; 682 } 683 684 static void gve_enable_supported_features(struct gve_priv *priv, 685 u32 supported_features_mask, 686 const struct gve_device_option_jumbo_frames 687 *dev_op_jumbo_frames) 688 { 689 /* Before control reaches this point, the page-size-capped max MTU from 690 * the gve_device_descriptor field has already been stored in 691 * priv->dev->max_mtu. We overwrite it with the true max MTU below. 692 */ 693 if (dev_op_jumbo_frames && 694 (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) { 695 dev_info(&priv->pdev->dev, 696 "JUMBO FRAMES device option enabled.\n"); 697 priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu); 698 } 699 } 700 701 int gve_adminq_describe_device(struct gve_priv *priv) 702 { 703 struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL; 704 struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL; 705 struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL; 706 struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL; 707 struct gve_device_descriptor *descriptor; 708 u32 supported_features_mask = 0; 709 union gve_adminq_command cmd; 710 dma_addr_t descriptor_bus; 711 int err = 0; 712 u8 *mac; 713 u16 mtu; 714 715 memset(&cmd, 0, sizeof(cmd)); 716 descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE, 717 &descriptor_bus, GFP_KERNEL); 718 if (!descriptor) 719 return -ENOMEM; 720 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE); 721 cmd.describe_device.device_descriptor_addr = 722 cpu_to_be64(descriptor_bus); 723 cmd.describe_device.device_descriptor_version = 724 cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION); 725 cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE); 726 727 err = gve_adminq_execute_cmd(priv, &cmd); 728 if (err) 729 goto free_device_descriptor; 730 731 err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda, 732 &dev_op_gqi_qpl, &dev_op_dqo_rda, 733 &dev_op_jumbo_frames); 734 if (err) 735 goto free_device_descriptor; 736 737 /* If the GQI_RAW_ADDRESSING option is not enabled and the queue format 738 * is not set to GqiRda, choose the queue format in a priority order: 739 * DqoRda, GqiRda, GqiQpl. Use GqiQpl as default. 740 */ 741 if (dev_op_dqo_rda) { 742 priv->queue_format = GVE_DQO_RDA_FORMAT; 743 dev_info(&priv->pdev->dev, 744 "Driver is running with DQO RDA queue format.\n"); 745 supported_features_mask = 746 be32_to_cpu(dev_op_dqo_rda->supported_features_mask); 747 } else if (dev_op_gqi_rda) { 748 priv->queue_format = GVE_GQI_RDA_FORMAT; 749 dev_info(&priv->pdev->dev, 750 "Driver is running with GQI RDA queue format.\n"); 751 supported_features_mask = 752 be32_to_cpu(dev_op_gqi_rda->supported_features_mask); 753 } else if (priv->queue_format == GVE_GQI_RDA_FORMAT) { 754 dev_info(&priv->pdev->dev, 755 "Driver is running with GQI RDA queue format.\n"); 756 } else { 757 priv->queue_format = GVE_GQI_QPL_FORMAT; 758 if (dev_op_gqi_qpl) 759 supported_features_mask = 760 be32_to_cpu(dev_op_gqi_qpl->supported_features_mask); 761 dev_info(&priv->pdev->dev, 762 "Driver is running with GQI QPL queue format.\n"); 763 } 764 if (gve_is_gqi(priv)) { 765 err = gve_set_desc_cnt(priv, descriptor); 766 } else { 767 /* DQO supports LRO. */ 768 priv->dev->hw_features |= NETIF_F_LRO; 769 err = gve_set_desc_cnt_dqo(priv, descriptor, dev_op_dqo_rda); 770 } 771 if (err) 772 goto free_device_descriptor; 773 774 priv->max_registered_pages = 775 be64_to_cpu(descriptor->max_registered_pages); 776 mtu = be16_to_cpu(descriptor->mtu); 777 if (mtu < ETH_MIN_MTU) { 778 dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu); 779 err = -EINVAL; 780 goto free_device_descriptor; 781 } 782 priv->dev->max_mtu = mtu; 783 priv->num_event_counters = be16_to_cpu(descriptor->counters); 784 eth_hw_addr_set(priv->dev, descriptor->mac); 785 mac = descriptor->mac; 786 dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac); 787 priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl); 788 priv->rx_data_slot_cnt = be16_to_cpu(descriptor->rx_pages_per_qpl); 789 790 if (gve_is_gqi(priv) && priv->rx_data_slot_cnt < priv->rx_desc_cnt) { 791 dev_err(&priv->pdev->dev, "rx_data_slot_cnt cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n", 792 priv->rx_data_slot_cnt); 793 priv->rx_desc_cnt = priv->rx_data_slot_cnt; 794 } 795 priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues); 796 797 gve_enable_supported_features(priv, supported_features_mask, 798 dev_op_jumbo_frames); 799 800 free_device_descriptor: 801 dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor, 802 descriptor_bus); 803 return err; 804 } 805 806 int gve_adminq_register_page_list(struct gve_priv *priv, 807 struct gve_queue_page_list *qpl) 808 { 809 struct device *hdev = &priv->pdev->dev; 810 u32 num_entries = qpl->num_entries; 811 u32 size = num_entries * sizeof(qpl->page_buses[0]); 812 union gve_adminq_command cmd; 813 dma_addr_t page_list_bus; 814 __be64 *page_list; 815 int err; 816 int i; 817 818 memset(&cmd, 0, sizeof(cmd)); 819 page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL); 820 if (!page_list) 821 return -ENOMEM; 822 823 for (i = 0; i < num_entries; i++) 824 page_list[i] = cpu_to_be64(qpl->page_buses[i]); 825 826 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST); 827 cmd.reg_page_list = (struct gve_adminq_register_page_list) { 828 .page_list_id = cpu_to_be32(qpl->id), 829 .num_pages = cpu_to_be32(num_entries), 830 .page_address_list_addr = cpu_to_be64(page_list_bus), 831 }; 832 833 err = gve_adminq_execute_cmd(priv, &cmd); 834 dma_free_coherent(hdev, size, page_list, page_list_bus); 835 return err; 836 } 837 838 int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id) 839 { 840 union gve_adminq_command cmd; 841 842 memset(&cmd, 0, sizeof(cmd)); 843 cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST); 844 cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) { 845 .page_list_id = cpu_to_be32(page_list_id), 846 }; 847 848 return gve_adminq_execute_cmd(priv, &cmd); 849 } 850 851 int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu) 852 { 853 union gve_adminq_command cmd; 854 855 memset(&cmd, 0, sizeof(cmd)); 856 cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER); 857 cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) { 858 .parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU), 859 .parameter_value = cpu_to_be64(mtu), 860 }; 861 862 return gve_adminq_execute_cmd(priv, &cmd); 863 } 864 865 int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len, 866 dma_addr_t stats_report_addr, u64 interval) 867 { 868 union gve_adminq_command cmd; 869 870 memset(&cmd, 0, sizeof(cmd)); 871 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS); 872 cmd.report_stats = (struct gve_adminq_report_stats) { 873 .stats_report_len = cpu_to_be64(stats_report_len), 874 .stats_report_addr = cpu_to_be64(stats_report_addr), 875 .interval = cpu_to_be64(interval), 876 }; 877 878 return gve_adminq_execute_cmd(priv, &cmd); 879 } 880 881 int gve_adminq_report_link_speed(struct gve_priv *priv) 882 { 883 union gve_adminq_command gvnic_cmd; 884 dma_addr_t link_speed_region_bus; 885 __be64 *link_speed_region; 886 int err; 887 888 link_speed_region = 889 dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region), 890 &link_speed_region_bus, GFP_KERNEL); 891 892 if (!link_speed_region) 893 return -ENOMEM; 894 895 memset(&gvnic_cmd, 0, sizeof(gvnic_cmd)); 896 gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED); 897 gvnic_cmd.report_link_speed.link_speed_address = 898 cpu_to_be64(link_speed_region_bus); 899 900 err = gve_adminq_execute_cmd(priv, &gvnic_cmd); 901 902 priv->link_speed = be64_to_cpu(*link_speed_region); 903 dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region, 904 link_speed_region_bus); 905 return err; 906 } 907 908 int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv, 909 struct gve_ptype_lut *ptype_lut) 910 { 911 struct gve_ptype_map *ptype_map; 912 union gve_adminq_command cmd; 913 dma_addr_t ptype_map_bus; 914 int err = 0; 915 int i; 916 917 memset(&cmd, 0, sizeof(cmd)); 918 ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map), 919 &ptype_map_bus, GFP_KERNEL); 920 if (!ptype_map) 921 return -ENOMEM; 922 923 cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP); 924 cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) { 925 .ptype_map_len = cpu_to_be64(sizeof(*ptype_map)), 926 .ptype_map_addr = cpu_to_be64(ptype_map_bus), 927 }; 928 929 err = gve_adminq_execute_cmd(priv, &cmd); 930 if (err) 931 goto err; 932 933 /* Populate ptype_lut. */ 934 for (i = 0; i < GVE_NUM_PTYPES; i++) { 935 ptype_lut->ptypes[i].l3_type = 936 ptype_map->ptypes[i].l3_type; 937 ptype_lut->ptypes[i].l4_type = 938 ptype_map->ptypes[i].l4_type; 939 } 940 err: 941 dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map, 942 ptype_map_bus); 943 return err; 944 } 945