56 void gve_parse_device_option(struct gve_priv *priv,  in gve_parse_device_option()  argument
74 			device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,  in gve_parse_device_option()
82 			device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,  in gve_parse_device_option()
91 			device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,  in gve_parse_device_option()
99 			device_printf(priv->dev,  in gve_parse_device_option()
110 		device_printf(priv->dev, "Unrecognized device option 0x%hx not enabled.\n",  in gve_parse_device_option()
117 gve_process_device_options(struct gve_priv *priv,  in gve_process_device_options()  argument
132 			device_printf(priv->dev,  in gve_process_device_options()
137 		gve_parse_device_option(priv, descriptor, dev_opt,  in gve_process_device_options()
145 static int gve_adminq_execute_cmd(struct gve_priv *priv,
149 gve_adminq_destroy_tx_queue(struct gve_priv *priv, uint32_t id)  in gve_adminq_destroy_tx_queue()  argument
156 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_destroy_tx_queue()
160 gve_adminq_destroy_rx_queue(struct gve_priv *priv, uint32_t id)  in gve_adminq_destroy_rx_queue()  argument
167 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_destroy_rx_queue()
171 gve_adminq_destroy_rx_queues(struct gve_priv *priv, uint32_t num_queues)  in gve_adminq_destroy_rx_queues()  argument
177 		err = gve_adminq_destroy_rx_queue(priv, i);  in gve_adminq_destroy_rx_queues()
179 			device_printf(priv->dev, "Failed to destroy rxq %d, err: %d\n",  in gve_adminq_destroy_rx_queues()
187 	device_printf(priv->dev, "Destroyed %d rx queues\n", num_queues);  in gve_adminq_destroy_rx_queues()
192 gve_adminq_destroy_tx_queues(struct gve_priv *priv, uint32_t num_queues)  in gve_adminq_destroy_tx_queues()  argument
198 		err = gve_adminq_destroy_tx_queue(priv, i);  in gve_adminq_destroy_tx_queues()
200 			device_printf(priv->dev, "Failed to destroy txq %d, err: %d\n",  in gve_adminq_destroy_tx_queues()
208 	device_printf(priv->dev, "Destroyed %d tx queues\n", num_queues);  in gve_adminq_destroy_tx_queues()
213 gve_adminq_create_rx_queue(struct gve_priv *priv, uint32_t queue_index)  in gve_adminq_create_rx_queue()  argument
216 	struct gve_rx_ring *rx = &priv->rx[queue_index];  in gve_adminq_create_rx_queue()
230 		.rx_ring_size = htobe16(priv->rx_desc_cnt),  in gve_adminq_create_rx_queue()
234 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_create_rx_queue()
238 gve_adminq_create_rx_queues(struct gve_priv *priv, uint32_t num_queues)  in gve_adminq_create_rx_queues()  argument
244 		err = gve_adminq_create_rx_queue(priv, i);  in gve_adminq_create_rx_queues()
246 			device_printf(priv->dev, "Failed to create rxq %d, err: %d\n",  in gve_adminq_create_rx_queues()
253 		device_printf(priv->dev, "Created %d rx queues\n", num_queues);  in gve_adminq_create_rx_queues()
257 	gve_adminq_destroy_rx_queues(priv, i);  in gve_adminq_create_rx_queues()
262 gve_adminq_create_tx_queue(struct gve_priv *priv, uint32_t queue_index)  in gve_adminq_create_tx_queue()  argument
265 	struct gve_tx_ring *tx = &priv->tx[queue_index];  in gve_adminq_create_tx_queue()
277 		.tx_ring_size = htobe16(priv->tx_desc_cnt),  in gve_adminq_create_tx_queue()
280 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_create_tx_queue()
284 gve_adminq_create_tx_queues(struct gve_priv *priv, uint32_t num_queues)  in gve_adminq_create_tx_queues()  argument
290 		err = gve_adminq_create_tx_queue(priv, i);  in gve_adminq_create_tx_queues()
292 			device_printf(priv->dev, "Failed to create txq %d, err: %d\n",  in gve_adminq_create_tx_queues()
299 		device_printf(priv->dev, "Created %d tx queues\n", num_queues);  in gve_adminq_create_tx_queues()
303 	gve_adminq_destroy_tx_queues(priv, i);  in gve_adminq_create_tx_queues()
308 gve_adminq_set_mtu(struct gve_priv *priv, uint32_t mtu) {  in gve_adminq_set_mtu()  argument
317 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_set_mtu()
321 gve_enable_supported_features(struct gve_priv *priv,  in gve_enable_supported_features()  argument
328 			device_printf(priv->dev, "JUMBO FRAMES device option enabled: %u.\n",  in gve_enable_supported_features()
330 		priv->max_mtu = be16toh(dev_op_jumbo_frames->max_mtu);  in gve_enable_supported_features()
335 gve_adminq_describe_device(struct gve_priv *priv)  in gve_adminq_describe_device()  argument
346 	rc = gve_dma_alloc_coherent(priv, ADMINQ_SIZE, ADMINQ_SIZE, &desc_mem);  in gve_adminq_describe_device()
348 		device_printf(priv->dev, "Failed to alloc DMA mem for DescribeDevice.\n");  in gve_adminq_describe_device()
363 	rc = gve_adminq_execute_cmd(priv, &aq_cmd);  in gve_adminq_describe_device()
369 	rc = gve_process_device_options(priv, desc, &dev_op_gqi_qpl,  in gve_adminq_describe_device()
375 		priv->queue_format = GVE_GQI_QPL_FORMAT;  in gve_adminq_describe_device()
379 			device_printf(priv->dev,  in gve_adminq_describe_device()
382 		device_printf(priv->dev, "No compatible queue formats\n");  in gve_adminq_describe_device()
387         priv->num_event_counters = be16toh(desc->counters);  in gve_adminq_describe_device()
388 	priv->default_num_queues = be16toh(desc->default_num_queues);  in gve_adminq_describe_device()
389 	priv->tx_desc_cnt = be16toh(desc->tx_queue_entries);  in gve_adminq_describe_device()
390 	priv->rx_desc_cnt = be16toh(desc->rx_queue_entries);  in gve_adminq_describe_device()
391 	priv->rx_pages_per_qpl = be16toh(desc->rx_pages_per_qpl);  in gve_adminq_describe_device()
392 	priv->max_registered_pages = be64toh(desc->max_registered_pages);  in gve_adminq_describe_device()
393 	priv->max_mtu = be16toh(desc->mtu);  in gve_adminq_describe_device()
394 	priv->default_num_queues = be16toh(desc->default_num_queues);  in gve_adminq_describe_device()
395 	priv->supported_features =  supported_features_mask;  in gve_adminq_describe_device()
397 	gve_enable_supported_features(priv, supported_features_mask,  in gve_adminq_describe_device()
401 		priv->mac[i] = desc->mac[i];  in gve_adminq_describe_device()
410 gve_adminq_register_page_list(struct gve_priv *priv,  in gve_adminq_register_page_list()  argument
421 	err = gve_dma_alloc_coherent(priv, size, PAGE_SIZE, &dma);  in gve_adminq_register_page_list()
440 	err = gve_adminq_execute_cmd(priv, &cmd);  in gve_adminq_register_page_list()
446 gve_adminq_unregister_page_list(struct gve_priv *priv, uint32_t page_list_id)  in gve_adminq_unregister_page_list()  argument
455 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_unregister_page_list()
460 gve_adminq_configure_device_resources(struct gve_priv *priv)  in gve_adminq_configure_device_resources()  argument
464 	bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map,  in gve_adminq_configure_device_resources()
466 	bus_dmamap_sync(priv->counter_array_mem.tag,  in gve_adminq_configure_device_resources()
467 	    priv->counter_array_mem.map, BUS_DMASYNC_PREREAD);  in gve_adminq_configure_device_resources()
472 		.counter_array = htobe64(priv->counter_array_mem.bus_addr),  in gve_adminq_configure_device_resources()
473 		.irq_db_addr = htobe64(priv->irqs_db_mem.bus_addr),  in gve_adminq_configure_device_resources()
474 		.num_counters = htobe32(priv->num_event_counters),  in gve_adminq_configure_device_resources()
475 		.num_irq_dbs = htobe32(priv->num_queues),  in gve_adminq_configure_device_resources()
478 		.queue_format = priv->queue_format,  in gve_adminq_configure_device_resources()
481 	return (gve_adminq_execute_cmd(priv, &aq_cmd));  in gve_adminq_configure_device_resources()
485 gve_adminq_deconfigure_device_resources(struct gve_priv *priv)  in gve_adminq_deconfigure_device_resources()  argument
490 	return (gve_adminq_execute_cmd(priv, &aq_cmd));  in gve_adminq_deconfigure_device_resources()
494 gve_adminq_verify_driver_compatibility(struct gve_priv *priv,  in gve_adminq_verify_driver_compatibility()  argument
506 	return (gve_adminq_execute_cmd(priv, &aq_cmd));  in gve_adminq_verify_driver_compatibility()
510 gve_adminq_alloc(struct gve_priv *priv)  in gve_adminq_alloc()  argument
514 	if (gve_get_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK))  in gve_adminq_alloc()
517 	if (priv->aq_mem.cpu_addr == NULL) {  in gve_adminq_alloc()
518 		rc = gve_dma_alloc_coherent(priv, ADMINQ_SIZE, ADMINQ_SIZE,  in gve_adminq_alloc()
519 		    &priv->aq_mem);  in gve_adminq_alloc()
521 			device_printf(priv->dev, "Failed to allocate admin queue mem\n");  in gve_adminq_alloc()
526 	priv->adminq = priv->aq_mem.cpu_addr;  in gve_adminq_alloc()
527 	priv->adminq_bus_addr = priv->aq_mem.bus_addr;  in gve_adminq_alloc()
529 	if (priv->adminq == NULL)  in gve_adminq_alloc()
532 	priv->adminq_mask = ADMINQ_SLOTS - 1;  in gve_adminq_alloc()
533 	priv->adminq_prod_cnt = 0;  in gve_adminq_alloc()
534 	priv->adminq_cmd_fail = 0;  in gve_adminq_alloc()
535 	priv->adminq_timeouts = 0;  in gve_adminq_alloc()
536 	priv->adminq_describe_device_cnt = 0;  in gve_adminq_alloc()
537 	priv->adminq_cfg_device_resources_cnt = 0;  in gve_adminq_alloc()
538 	priv->adminq_register_page_list_cnt = 0;  in gve_adminq_alloc()
539 	priv->adminq_unregister_page_list_cnt = 0;  in gve_adminq_alloc()
540 	priv->adminq_create_tx_queue_cnt = 0;  in gve_adminq_alloc()
541 	priv->adminq_create_rx_queue_cnt = 0;  in gve_adminq_alloc()
542 	priv->adminq_destroy_tx_queue_cnt = 0;  in gve_adminq_alloc()
543 	priv->adminq_destroy_rx_queue_cnt = 0;  in gve_adminq_alloc()
544 	priv->adminq_dcfg_device_resources_cnt = 0;  in gve_adminq_alloc()
545 	priv->adminq_set_driver_parameter_cnt = 0;  in gve_adminq_alloc()
547 	gve_reg_bar_write_4(priv, GVE_REG_ADMINQ_ADDR,  in gve_adminq_alloc()
548 	    priv->adminq_bus_addr / ADMINQ_SIZE);  in gve_adminq_alloc()
550 	gve_set_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK);  in gve_adminq_alloc()
555 gve_release_adminq(struct gve_priv *priv)  in gve_release_adminq()  argument
557 	if (!gve_get_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK))  in gve_release_adminq()
560 	gve_reg_bar_write_4(priv, GVE_REG_ADMINQ_ADDR, 0);  in gve_release_adminq()
561 	while (gve_reg_bar_read_4(priv, GVE_REG_ADMINQ_ADDR)) {  in gve_release_adminq()
562 		device_printf(priv->dev, "Waiting until admin queue is released.\n");  in gve_release_adminq()
566 	gve_dma_free_coherent(&priv->aq_mem);  in gve_release_adminq()
567 	priv->aq_mem = (struct gve_dma_handle){};  in gve_release_adminq()
568 	priv->adminq = 0;  in gve_release_adminq()
569 	priv->adminq_bus_addr = 0;  in gve_release_adminq()
571 	gve_clear_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK);  in gve_release_adminq()
574 		device_printf(priv->dev, "Admin queue released\n");  in gve_release_adminq()
578 gve_adminq_parse_err(struct gve_priv *priv, uint32_t opcode, uint32_t status)  in gve_adminq_parse_err()  argument
582 		device_printf(priv->dev, "AQ command(%u): failed with status %d\n", opcode, status);  in gve_adminq_parse_err()
583 		priv->adminq_cmd_fail++;  in gve_adminq_parse_err()
590 		device_printf(priv->dev,  in gve_adminq_parse_err()
624 		device_printf(priv->dev, "AQ command(%u): unknown status code %d\n",  in gve_adminq_parse_err()
631 gve_adminq_kick_cmd(struct gve_priv *priv, uint32_t prod_cnt)  in gve_adminq_kick_cmd()  argument
633 	gve_reg_bar_write_4(priv, ADMINQ_DOORBELL, prod_cnt);  in gve_adminq_kick_cmd()
638 gve_adminq_wait_for_cmd(struct gve_priv *priv, uint32_t prod_cnt)  in gve_adminq_wait_for_cmd()  argument
643 		if (gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER) == prod_cnt)  in gve_adminq_wait_for_cmd()
656 gve_adminq_kick_and_wait(struct gve_priv *priv)  in gve_adminq_kick_and_wait()  argument
664 	tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);  in gve_adminq_kick_and_wait()
665 	head = priv->adminq_prod_cnt;  in gve_adminq_kick_and_wait()
667 	gve_adminq_kick_cmd(priv, head);  in gve_adminq_kick_and_wait()
668 	if (!gve_adminq_wait_for_cmd(priv, head)) {  in gve_adminq_kick_and_wait()
669 		device_printf(priv->dev, "AQ commands timed out, need to reset AQ\n");  in gve_adminq_kick_and_wait()
670 		priv->adminq_timeouts++;  in gve_adminq_kick_and_wait()
674 	    priv->aq_mem.tag, priv->aq_mem.map, BUS_DMASYNC_POSTREAD);  in gve_adminq_kick_and_wait()
677 		cmd = &priv->adminq[i & priv->adminq_mask];  in gve_adminq_kick_and_wait()
680 		err = gve_adminq_parse_err(priv, opcode, status);  in gve_adminq_kick_and_wait()
693 gve_adminq_issue_cmd(struct gve_priv *priv, struct gve_adminq_command *cmd_orig)  in gve_adminq_issue_cmd()  argument
700 	tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);  in gve_adminq_issue_cmd()
703 	if ((priv->adminq_prod_cnt - tail) > priv->adminq_mask) {  in gve_adminq_issue_cmd()
705 		err = gve_adminq_kick_and_wait(priv);  in gve_adminq_issue_cmd()
710 		tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);  in gve_adminq_issue_cmd()
711 		if ((priv->adminq_prod_cnt - tail) > priv->adminq_mask) {  in gve_adminq_issue_cmd()
720 	cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];  in gve_adminq_issue_cmd()
721 	priv->adminq_prod_cnt++;  in gve_adminq_issue_cmd()
726 	    priv->aq_mem.tag, priv->aq_mem.map, BUS_DMASYNC_PREWRITE);  in gve_adminq_issue_cmd()
732 		priv->adminq_describe_device_cnt++;  in gve_adminq_issue_cmd()
736 		priv->adminq_cfg_device_resources_cnt++;  in gve_adminq_issue_cmd()
740 		priv->adminq_register_page_list_cnt++;  in gve_adminq_issue_cmd()
744 		priv->adminq_unregister_page_list_cnt++;  in gve_adminq_issue_cmd()
748 		priv->adminq_create_tx_queue_cnt++;  in gve_adminq_issue_cmd()
752 		priv->adminq_create_rx_queue_cnt++;  in gve_adminq_issue_cmd()
756 		priv->adminq_destroy_tx_queue_cnt++;  in gve_adminq_issue_cmd()
760 		priv->adminq_destroy_rx_queue_cnt++;  in gve_adminq_issue_cmd()
764 		priv->adminq_dcfg_device_resources_cnt++;  in gve_adminq_issue_cmd()
768 		priv->adminq_set_driver_parameter_cnt++;  in gve_adminq_issue_cmd()
772 		priv->adminq_verify_driver_compatibility_cnt++;  in gve_adminq_issue_cmd()
776 		device_printf(priv->dev, "Unknown AQ command opcode %d\n", opcode);  in gve_adminq_issue_cmd()
789 gve_adminq_execute_cmd(struct gve_priv *priv, struct gve_adminq_command *cmd_orig)  in gve_adminq_execute_cmd()  argument
794 	tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);  in gve_adminq_execute_cmd()
795 	head = priv->adminq_prod_cnt;  in gve_adminq_execute_cmd()
799 	err = gve_adminq_issue_cmd(priv, cmd_orig);  in gve_adminq_execute_cmd()
802 	return (gve_adminq_kick_and_wait(priv));  in gve_adminq_execute_cmd()