103 	struct mlx5_cmd_work_ent *ent;  in alloc_cmd()  local
105 	ent = kzalloc(sizeof(*ent), alloc_flags);  in alloc_cmd()
106 	if (!ent)  in alloc_cmd()
109 	ent->in		= in;  in alloc_cmd()
110 	ent->uin_size	= uin_size;  in alloc_cmd()
111 	ent->out	= out;  in alloc_cmd()
112 	ent->uout	= uout;  in alloc_cmd()
113 	ent->uout_size	= uout_size;  in alloc_cmd()
114 	ent->callback	= cbk;  in alloc_cmd()
115 	ent->context	= context;  in alloc_cmd()
116 	ent->cmd	= cmd;  in alloc_cmd()
117 	ent->page_queue = page_queue;  in alloc_cmd()
119 	return ent;  in alloc_cmd()
136 static int alloc_ent(struct mlx5_cmd_work_ent *ent)  in alloc_ent()  argument
139 	struct mlx5_cmd *cmd = ent->cmd;  in alloc_ent()
145 	if (!ent->page_queue) {  in alloc_ent()
155 		ent->busy = 1;  in alloc_ent()
156 		ent->idx = ret;  in alloc_ent()
157 		clear_bit(ent->idx, &cmd->bitmask);  in alloc_ent()
158 		cmd->ent_mode[ent->idx] =  in alloc_ent()
159 		    ent->polling ? MLX5_CMD_MODE_POLLING : MLX5_CMD_MODE_EVENTS;  in alloc_ent()
160 		cmd->ent_arr[ent->idx] = ent;  in alloc_ent()
239 static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)  in set_signature()  argument
241 	ent->lay->sig = ~xor8_buf(ent->lay, sizeof(*ent->lay));  in set_signature()
242 	calc_chain_sig(ent->in, ent->token, csum);  in set_signature()
243 	calc_chain_sig(ent->out, ent->token, csum);  in set_signature()
246 static void poll_timeout(struct mlx5_cmd_work_ent *ent)  in poll_timeout()  argument
248 	struct mlx5_core_dev *dev = container_of(ent->cmd,  in poll_timeout()
255 		own = ent->lay->status_own;  in poll_timeout()
258 			ent->ret = 0;  in poll_timeout()
264 	ent->ret = -ETIMEDOUT;  in poll_timeout()
267 static void free_cmd(struct mlx5_cmd_work_ent *ent)  in free_cmd()  argument
269         cancel_delayed_work_sync(&ent->cb_timeout_work);  in free_cmd()
270 	kfree(ent);  in free_cmd()
274 verify_signature(struct mlx5_cmd_work_ent *ent)  in verify_signature()  argument
276 	struct mlx5_cmd_msg *msg = ent->out;  in verify_signature()
281 	sig = xor8_buf(ent->lay, sizeof(*ent->lay));  in verify_signature()
716 			 struct mlx5_cmd_work_ent *ent, int input)  in dump_command()  argument
718 	struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;  in dump_command()
719 	u16 op = MLX5_GET(mbox_in, ent->lay->in, opcode);  in dump_command()
723 	int msg_len = input ? ent->uin_size : ent->uout_size;  in dump_command()
740 			dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset);  in dump_command()
741 			offset += sizeof(ent->lay->in);  in dump_command()
743 			dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset);  in dump_command()
744 			offset += sizeof(ent->lay->out);  in dump_command()
747 		dump_buf(ent->lay, sizeof(*ent->lay), 0, offset);  in dump_command()
748 		offset += sizeof(*ent->lay);  in dump_command()
788         struct mlx5_cmd_work_ent *ent = container_of(dwork,  in cb_timeout_handler()  local
791         struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev,  in cb_timeout_handler()
794         ent->ret = -ETIMEDOUT;  in cb_timeout_handler()
796                        mlx5_command_str(msg_to_opcode(ent->in)),  in cb_timeout_handler()
797                        msg_to_opcode(ent->in));  in cb_timeout_handler()
798         mlx5_cmd_comp_handler(dev, 1UL << ent->idx, MLX5_CMD_MODE_EVENTS);  in cb_timeout_handler()
801 static void complete_command(struct mlx5_cmd_work_ent *ent)  in complete_command()  argument
803 	struct mlx5_cmd *cmd = ent->cmd;  in complete_command()
815 	if (ent->page_queue)  in complete_command()
824 		ent->ret = mlx5_internal_err_ret_value(dev, msg_to_opcode(ent->in), &drv_synd, &status);  in complete_command()
825 		MLX5_SET(mbox_out, ent->out, status, status);  in complete_command()
826 		MLX5_SET(mbox_out, ent->out, syndrome, drv_synd);  in complete_command()
829 	if (ent->callback) {  in complete_command()
830 		ds = ent->ts2 - ent->ts1;  in complete_command()
831 		if (ent->op < ARRAY_SIZE(cmd->stats)) {  in complete_command()
832 			stats = &cmd->stats[ent->op];  in complete_command()
839 		callback = ent->callback;  in complete_command()
840 		context = ent->context;  in complete_command()
841 		err = ent->ret;  in complete_command()
843 			err = mlx5_copy_from_msg(ent->uout,  in complete_command()
844 						 ent->out,  in complete_command()
845 						 ent->uout_size);  in complete_command()
847 							 ent->in->first.data,  in complete_command()
848 							 ent->uout);  in complete_command()
851 		mlx5_free_cmd_msg(dev, ent->out);  in complete_command()
852 		free_msg(dev, ent->in);  in complete_command()
854 		err = err ? err : ent->status;  in complete_command()
855 		free_cmd(ent);  in complete_command()
858 		complete(&ent->done);  in complete_command()
865 	struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);  in cmd_work_handler()  local
866 	struct mlx5_cmd *cmd = ent->cmd;  in cmd_work_handler()
871 	bool poll_cmd = ent->polling;  in cmd_work_handler()
873 	sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem;  in cmd_work_handler()
876 	if (alloc_ent(ent) < 0) {  in cmd_work_handler()
877 		complete_command(ent);  in cmd_work_handler()
881 	ent->token = alloc_token(cmd);  in cmd_work_handler()
882 	lay = get_inst(cmd, ent->idx);  in cmd_work_handler()
883 	ent->lay = lay;  in cmd_work_handler()
885 	memcpy(lay->in, ent->in->first.data, sizeof(lay->in));  in cmd_work_handler()
886 	ent->op = be32_to_cpu(lay->in[0]) >> 16;  in cmd_work_handler()
887 	if (ent->in->numpages != 0)  in cmd_work_handler()
888 		lay->in_ptr = cpu_to_be64(mlx5_fwp_get_dma(ent->in, 0));  in cmd_work_handler()
889 	if (ent->out->numpages != 0)  in cmd_work_handler()
890 		lay->out_ptr = cpu_to_be64(mlx5_fwp_get_dma(ent->out, 0));  in cmd_work_handler()
891 	lay->inlen = cpu_to_be32(ent->uin_size);  in cmd_work_handler()
892 	lay->outlen = cpu_to_be32(ent->uout_size);  in cmd_work_handler()
894 	lay->token = ent->token;  in cmd_work_handler()
896 	set_signature(ent, !cmd->checksum_disabled);  in cmd_work_handler()
897 	dump_command(dev, ent, 1);  in cmd_work_handler()
898 	ent->ts1 = ktime_get_ns();  in cmd_work_handler()
899 	ent->busy = 0;  in cmd_work_handler()
900         if (ent->callback)  in cmd_work_handler()
901                 schedule_delayed_work(&ent->cb_timeout_work, cb_timeout);  in cmd_work_handler()
904 	mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);  in cmd_work_handler()
907 	iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);  in cmd_work_handler()
912 		poll_timeout(ent);  in cmd_work_handler()
914 		mlx5_cmd_comp_handler(dev, 1U << ent->idx, MLX5_CMD_MODE_POLLING);  in cmd_work_handler()
948 static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)  in wait_func()  argument
953 	if (ent->polling) {  in wait_func()
954 		wait_for_completion(&ent->done);  in wait_func()
955 	} else if (!wait_for_completion_timeout(&ent->done, timeout)) {  in wait_func()
956                 ent->ret = -ETIMEDOUT;  in wait_func()
957                 mlx5_cmd_comp_handler(dev, 1UL << ent->idx, MLX5_CMD_MODE_EVENTS);  in wait_func()
960         err = ent->ret;  in wait_func()
964 			       mlx5_command_str(msg_to_opcode(ent->in)),  in wait_func()
965 			       msg_to_opcode(ent->in));  in wait_func()
968 		      err, deliv_status_to_str(ent->status), ent->status);  in wait_func()
985 	struct mlx5_cmd_work_ent *ent;  in mlx5_cmd_invoke()  local
994 	ent = alloc_cmd(cmd, in, uin_size, out, uout, uout_size, callback,  in mlx5_cmd_invoke()
996 	if (IS_ERR(ent))  in mlx5_cmd_invoke()
997 		return PTR_ERR(ent);  in mlx5_cmd_invoke()
999 	ent->polling = force_polling || (cmd->mode == MLX5_CMD_MODE_POLLING);  in mlx5_cmd_invoke()
1002 		init_completion(&ent->done);  in mlx5_cmd_invoke()
1004         INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler);  in mlx5_cmd_invoke()
1005 	INIT_WORK(&ent->work, cmd_work_handler);  in mlx5_cmd_invoke()
1007 		cmd_work_handler(&ent->work);  in mlx5_cmd_invoke()
1008 	} else if (!queue_work(dev->priv.health.wq_cmd, &ent->work)) {  in mlx5_cmd_invoke()
1017         err = wait_func(dev, ent);  in mlx5_cmd_invoke()
1021         ds = ent->ts2 - ent->ts1;  in mlx5_cmd_invoke()
1033         *status = ent->status;  in mlx5_cmd_invoke()
1034         free_cmd(ent);  in mlx5_cmd_invoke()
1039 	free_cmd(ent);  in mlx5_cmd_invoke()
1194 	struct mlx5_cmd_work_ent *ent;  in mlx5_cmd_comp_handler()  local
1208 		ent = cmd->ent_arr[i];  in mlx5_cmd_comp_handler()
1210 		if (ent == NULL)  in mlx5_cmd_comp_handler()
1212                 if (ent->callback)  in mlx5_cmd_comp_handler()
1213                         cancel_delayed_work(&ent->cb_timeout_work);  in mlx5_cmd_comp_handler()
1214 		ent->ts2 = ktime_get_ns();  in mlx5_cmd_comp_handler()
1215 		memcpy(ent->out->first.data, ent->lay->out,  in mlx5_cmd_comp_handler()
1216 		       sizeof(ent->lay->out));  in mlx5_cmd_comp_handler()
1218 		mlx5_fwp_invalidate(ent->out);  in mlx5_cmd_comp_handler()
1219 		dump_command(dev, ent, 0);  in mlx5_cmd_comp_handler()
1220 		if (!ent->ret) {  in mlx5_cmd_comp_handler()
1222 				ent->ret = verify_signature(ent);  in mlx5_cmd_comp_handler()
1224 				ent->ret = 0;  in mlx5_cmd_comp_handler()
1227 				ent->status = MLX5_DRIVER_STATUS_ABORTED;  in mlx5_cmd_comp_handler()
1229 				ent->status = ent->lay->status_own >> 1;  in mlx5_cmd_comp_handler()
1233 				      ent->ret,  in mlx5_cmd_comp_handler()
1234 				      deliv_status_to_str(ent->status),  in mlx5_cmd_comp_handler()
1235 				      ent->status);  in mlx5_cmd_comp_handler()
1237 		free_ent(cmd, ent->idx);  in mlx5_cmd_comp_handler()
1238 		complete_command(ent);  in mlx5_cmd_comp_handler()
1253 	struct cache_ent *ent = NULL;  in alloc_msg()  local
1256 		ent = &cmd->cache.large;  in alloc_msg()
1258 		ent = &cmd->cache.med;  in alloc_msg()
1260 	if (ent) {  in alloc_msg()
1261 		spin_lock_irq(&ent->lock);  in alloc_msg()
1262 		if (!list_empty(&ent->head)) {  in alloc_msg()
1263 			msg = list_entry(ent->head.next, struct mlx5_cmd_msg,  in alloc_msg()
1267 		spin_unlock_irq(&ent->lock);  in alloc_msg()