xref: /linux/drivers/mailbox/mtk-cmdq-mailbox.c (revision 52338415) 
1 // SPDX-License-Identifier: GPL-2.0
2 //
3 // Copyright (c) 2018 MediaTek Inc.
4 
5 #include <linux/bitops.h>
6 #include <linux/clk.h>
7 #include <linux/clk-provider.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/errno.h>
10 #include <linux/interrupt.h>
11 #include <linux/io.h>
12 #include <linux/iopoll.h>
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/platform_device.h>
16 #include <linux/mailbox_controller.h>
17 #include <linux/mailbox/mtk-cmdq-mailbox.h>
18 #include <linux/of_device.h>
19 
20 #define CMDQ_OP_CODE_MASK		(0xff << CMDQ_OP_CODE_SHIFT)
21 #define CMDQ_NUM_CMD(t)			(t->cmd_buf_size / CMDQ_INST_SIZE)
22 
23 #define CMDQ_CURR_IRQ_STATUS		0x10
24 #define CMDQ_SYNC_TOKEN_UPDATE		0x68
25 #define CMDQ_THR_SLOT_CYCLES		0x30
26 #define CMDQ_THR_BASE			0x100
27 #define CMDQ_THR_SIZE			0x80
28 #define CMDQ_THR_WARM_RESET		0x00
29 #define CMDQ_THR_ENABLE_TASK		0x04
30 #define CMDQ_THR_SUSPEND_TASK		0x08
31 #define CMDQ_THR_CURR_STATUS		0x0c
32 #define CMDQ_THR_IRQ_STATUS		0x10
33 #define CMDQ_THR_IRQ_ENABLE		0x14
34 #define CMDQ_THR_CURR_ADDR		0x20
35 #define CMDQ_THR_END_ADDR		0x24
36 #define CMDQ_THR_WAIT_TOKEN		0x30
37 #define CMDQ_THR_PRIORITY		0x40
38 
39 #define CMDQ_THR_ACTIVE_SLOT_CYCLES	0x3200
40 #define CMDQ_THR_ENABLED		0x1
41 #define CMDQ_THR_DISABLED		0x0
42 #define CMDQ_THR_SUSPEND		0x1
43 #define CMDQ_THR_RESUME			0x0
44 #define CMDQ_THR_STATUS_SUSPENDED	BIT(1)
45 #define CMDQ_THR_DO_WARM_RESET		BIT(0)
46 #define CMDQ_THR_IRQ_DONE		0x1
47 #define CMDQ_THR_IRQ_ERROR		0x12
48 #define CMDQ_THR_IRQ_EN			(CMDQ_THR_IRQ_ERROR | CMDQ_THR_IRQ_DONE)
49 #define CMDQ_THR_IS_WAITING		BIT(31)
50 
51 #define CMDQ_JUMP_BY_OFFSET		0x10000000
52 #define CMDQ_JUMP_BY_PA			0x10000001
53 
54 struct cmdq_thread {
55 	struct mbox_chan	*chan;
56 	void __iomem		*base;
57 	struct list_head	task_busy_list;
58 	u32			priority;
59 	bool			atomic_exec;
60 };
61 
62 struct cmdq_task {
63 	struct cmdq		*cmdq;
64 	struct list_head	list_entry;
65 	dma_addr_t		pa_base;
66 	struct cmdq_thread	*thread;
67 	struct cmdq_pkt		*pkt; /* the packet sent from mailbox client */
68 };
69 
70 struct cmdq {
71 	struct mbox_controller	mbox;
72 	void __iomem		*base;
73 	u32			irq;
74 	u32			thread_nr;
75 	u32			irq_mask;
76 	struct cmdq_thread	*thread;
77 	struct clk		*clock;
78 	bool			suspended;
79 };
80 
81 static int cmdq_thread_suspend(struct cmdq *cmdq, struct cmdq_thread *thread)
82 {
83 	u32 status;
84 
85 	writel(CMDQ_THR_SUSPEND, thread->base + CMDQ_THR_SUSPEND_TASK);
86 
87 	/* If already disabled, treat as suspended successful. */
88 	if (!(readl(thread->base + CMDQ_THR_ENABLE_TASK) & CMDQ_THR_ENABLED))
89 		return 0;
90 
91 	if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_CURR_STATUS,
92 			status, status & CMDQ_THR_STATUS_SUSPENDED, 0, 10)) {
93 		dev_err(cmdq->mbox.dev, "suspend GCE thread 0x%x failed\n",
94 			(u32)(thread->base - cmdq->base));
95 		return -EFAULT;
96 	}
97 
98 	return 0;
99 }
100 
101 static void cmdq_thread_resume(struct cmdq_thread *thread)
102 {
103 	writel(CMDQ_THR_RESUME, thread->base + CMDQ_THR_SUSPEND_TASK);
104 }
105 
106 static void cmdq_init(struct cmdq *cmdq)
107 {
108 	int i;
109 
110 	WARN_ON(clk_enable(cmdq->clock) < 0);
111 	writel(CMDQ_THR_ACTIVE_SLOT_CYCLES, cmdq->base + CMDQ_THR_SLOT_CYCLES);
112 	for (i = 0; i <= CMDQ_MAX_EVENT; i++)
113 		writel(i, cmdq->base + CMDQ_SYNC_TOKEN_UPDATE);
114 	clk_disable(cmdq->clock);
115 }
116 
117 static int cmdq_thread_reset(struct cmdq *cmdq, struct cmdq_thread *thread)
118 {
119 	u32 warm_reset;
120 
121 	writel(CMDQ_THR_DO_WARM_RESET, thread->base + CMDQ_THR_WARM_RESET);
122 	if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_WARM_RESET,
123 			warm_reset, !(warm_reset & CMDQ_THR_DO_WARM_RESET),
124 			0, 10)) {
125 		dev_err(cmdq->mbox.dev, "reset GCE thread 0x%x failed\n",
126 			(u32)(thread->base - cmdq->base));
127 		return -EFAULT;
128 	}
129 
130 	return 0;
131 }
132 
133 static void cmdq_thread_disable(struct cmdq *cmdq, struct cmdq_thread *thread)
134 {
135 	cmdq_thread_reset(cmdq, thread);
136 	writel(CMDQ_THR_DISABLED, thread->base + CMDQ_THR_ENABLE_TASK);
137 }
138 
139 /* notify GCE to re-fetch commands by setting GCE thread PC */
140 static void cmdq_thread_invalidate_fetched_data(struct cmdq_thread *thread)
141 {
142 	writel(readl(thread->base + CMDQ_THR_CURR_ADDR),
143 	       thread->base + CMDQ_THR_CURR_ADDR);
144 }
145 
146 static void cmdq_task_insert_into_thread(struct cmdq_task *task)
147 {
148 	struct device *dev = task->cmdq->mbox.dev;
149 	struct cmdq_thread *thread = task->thread;
150 	struct cmdq_task *prev_task = list_last_entry(
151 			&thread->task_busy_list, typeof(*task), list_entry);
152 	u64 *prev_task_base = prev_task->pkt->va_base;
153 
154 	/* let previous task jump to this task */
155 	dma_sync_single_for_cpu(dev, prev_task->pa_base,
156 				prev_task->pkt->cmd_buf_size, DMA_TO_DEVICE);
157 	prev_task_base[CMDQ_NUM_CMD(prev_task->pkt) - 1] =
158 		(u64)CMDQ_JUMP_BY_PA << 32 | task->pa_base;
159 	dma_sync_single_for_device(dev, prev_task->pa_base,
160 				   prev_task->pkt->cmd_buf_size, DMA_TO_DEVICE);
161 
162 	cmdq_thread_invalidate_fetched_data(thread);
163 }
164 
165 static bool cmdq_command_is_wfe(u64 cmd)
166 {
167 	u64 wfe_option = CMDQ_WFE_UPDATE | CMDQ_WFE_WAIT | CMDQ_WFE_WAIT_VALUE;
168 	u64 wfe_op = (u64)(CMDQ_CODE_WFE << CMDQ_OP_CODE_SHIFT) << 32;
169 	u64 wfe_mask = (u64)CMDQ_OP_CODE_MASK << 32 | 0xffffffff;
170 
171 	return ((cmd & wfe_mask) == (wfe_op | wfe_option));
172 }
173 
174 /* we assume tasks in the same display GCE thread are waiting the same event. */
175 static void cmdq_task_remove_wfe(struct cmdq_task *task)
176 {
177 	struct device *dev = task->cmdq->mbox.dev;
178 	u64 *base = task->pkt->va_base;
179 	int i;
180 
181 	dma_sync_single_for_cpu(dev, task->pa_base, task->pkt->cmd_buf_size,
182 				DMA_TO_DEVICE);
183 	for (i = 0; i < CMDQ_NUM_CMD(task->pkt); i++)
184 		if (cmdq_command_is_wfe(base[i]))
185 			base[i] = (u64)CMDQ_JUMP_BY_OFFSET << 32 |
186 				  CMDQ_JUMP_PASS;
187 	dma_sync_single_for_device(dev, task->pa_base, task->pkt->cmd_buf_size,
188 				   DMA_TO_DEVICE);
189 }
190 
191 static bool cmdq_thread_is_in_wfe(struct cmdq_thread *thread)
192 {
193 	return readl(thread->base + CMDQ_THR_WAIT_TOKEN) & CMDQ_THR_IS_WAITING;
194 }
195 
196 static void cmdq_thread_wait_end(struct cmdq_thread *thread,
197 				 unsigned long end_pa)
198 {
199 	struct device *dev = thread->chan->mbox->dev;
200 	unsigned long curr_pa;
201 
202 	if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_CURR_ADDR,
203 			curr_pa, curr_pa == end_pa, 1, 20))
204 		dev_err(dev, "GCE thread cannot run to end.\n");
205 }
206 
207 static void cmdq_task_exec_done(struct cmdq_task *task, enum cmdq_cb_status sta)
208 {
209 	struct cmdq_task_cb *cb = &task->pkt->async_cb;
210 	struct cmdq_cb_data data;
211 
212 	WARN_ON(cb->cb == (cmdq_async_flush_cb)NULL);
213 	data.sta = sta;
214 	data.data = cb->data;
215 	cb->cb(data);
216 
217 	list_del(&task->list_entry);
218 }
219 
220 static void cmdq_task_handle_error(struct cmdq_task *task)
221 {
222 	struct cmdq_thread *thread = task->thread;
223 	struct cmdq_task *next_task;
224 
225 	dev_err(task->cmdq->mbox.dev, "task 0x%p error\n", task);
226 	WARN_ON(cmdq_thread_suspend(task->cmdq, thread) < 0);
227 	next_task = list_first_entry_or_null(&thread->task_busy_list,
228 			struct cmdq_task, list_entry);
229 	if (next_task)
230 		writel(next_task->pa_base, thread->base + CMDQ_THR_CURR_ADDR);
231 	cmdq_thread_resume(thread);
232 }
233 
234 static void cmdq_thread_irq_handler(struct cmdq *cmdq,
235 				    struct cmdq_thread *thread)
236 {
237 	struct cmdq_task *task, *tmp, *curr_task = NULL;
238 	u32 curr_pa, irq_flag, task_end_pa;
239 	bool err;
240 
241 	irq_flag = readl(thread->base + CMDQ_THR_IRQ_STATUS);
242 	writel(~irq_flag, thread->base + CMDQ_THR_IRQ_STATUS);
243 
244 	/*
245 	 * When ISR call this function, another CPU core could run
246 	 * "release task" right before we acquire the spin lock, and thus
247 	 * reset / disable this GCE thread, so we need to check the enable
248 	 * bit of this GCE thread.
249 	 */
250 	if (!(readl(thread->base + CMDQ_THR_ENABLE_TASK) & CMDQ_THR_ENABLED))
251 		return;
252 
253 	if (irq_flag & CMDQ_THR_IRQ_ERROR)
254 		err = true;
255 	else if (irq_flag & CMDQ_THR_IRQ_DONE)
256 		err = false;
257 	else
258 		return;
259 
260 	curr_pa = readl(thread->base + CMDQ_THR_CURR_ADDR);
261 
262 	list_for_each_entry_safe(task, tmp, &thread->task_busy_list,
263 				 list_entry) {
264 		task_end_pa = task->pa_base + task->pkt->cmd_buf_size;
265 		if (curr_pa >= task->pa_base && curr_pa < task_end_pa)
266 			curr_task = task;
267 
268 		if (!curr_task || curr_pa == task_end_pa - CMDQ_INST_SIZE) {
269 			cmdq_task_exec_done(task, CMDQ_CB_NORMAL);
270 			kfree(task);
271 		} else if (err) {
272 			cmdq_task_exec_done(task, CMDQ_CB_ERROR);
273 			cmdq_task_handle_error(curr_task);
274 			kfree(task);
275 		}
276 
277 		if (curr_task)
278 			break;
279 	}
280 
281 	if (list_empty(&thread->task_busy_list)) {
282 		cmdq_thread_disable(cmdq, thread);
283 		clk_disable(cmdq->clock);
284 	}
285 }
286 
287 static irqreturn_t cmdq_irq_handler(int irq, void *dev)
288 {
289 	struct cmdq *cmdq = dev;
290 	unsigned long irq_status, flags = 0L;
291 	int bit;
292 
293 	irq_status = readl(cmdq->base + CMDQ_CURR_IRQ_STATUS) & cmdq->irq_mask;
294 	if (!(irq_status ^ cmdq->irq_mask))
295 		return IRQ_NONE;
296 
297 	for_each_clear_bit(bit, &irq_status, cmdq->thread_nr) {
298 		struct cmdq_thread *thread = &cmdq->thread[bit];
299 
300 		spin_lock_irqsave(&thread->chan->lock, flags);
301 		cmdq_thread_irq_handler(cmdq, thread);
302 		spin_unlock_irqrestore(&thread->chan->lock, flags);
303 	}
304 
305 	return IRQ_HANDLED;
306 }
307 
308 static int cmdq_suspend(struct device *dev)
309 {
310 	struct cmdq *cmdq = dev_get_drvdata(dev);
311 	struct cmdq_thread *thread;
312 	int i;
313 	bool task_running = false;
314 
315 	cmdq->suspended = true;
316 
317 	for (i = 0; i < cmdq->thread_nr; i++) {
318 		thread = &cmdq->thread[i];
319 		if (!list_empty(&thread->task_busy_list)) {
320 			task_running = true;
321 			break;
322 		}
323 	}
324 
325 	if (task_running)
326 		dev_warn(dev, "exist running task(s) in suspend\n");
327 
328 	clk_unprepare(cmdq->clock);
329 
330 	return 0;
331 }
332 
333 static int cmdq_resume(struct device *dev)
334 {
335 	struct cmdq *cmdq = dev_get_drvdata(dev);
336 
337 	WARN_ON(clk_prepare(cmdq->clock) < 0);
338 	cmdq->suspended = false;
339 	return 0;
340 }
341 
342 static int cmdq_remove(struct platform_device *pdev)
343 {
344 	struct cmdq *cmdq = platform_get_drvdata(pdev);
345 
346 	clk_unprepare(cmdq->clock);
347 
348 	return 0;
349 }
350 
351 static int cmdq_mbox_send_data(struct mbox_chan *chan, void *data)
352 {
353 	struct cmdq_pkt *pkt = (struct cmdq_pkt *)data;
354 	struct cmdq_thread *thread = (struct cmdq_thread *)chan->con_priv;
355 	struct cmdq *cmdq = dev_get_drvdata(chan->mbox->dev);
356 	struct cmdq_task *task;
357 	unsigned long curr_pa, end_pa;
358 
359 	/* Client should not flush new tasks if suspended. */
360 	WARN_ON(cmdq->suspended);
361 
362 	task = kzalloc(sizeof(*task), GFP_ATOMIC);
363 	if (!task)
364 		return -ENOMEM;
365 
366 	task->cmdq = cmdq;
367 	INIT_LIST_HEAD(&task->list_entry);
368 	task->pa_base = pkt->pa_base;
369 	task->thread = thread;
370 	task->pkt = pkt;
371 
372 	if (list_empty(&thread->task_busy_list)) {
373 		WARN_ON(clk_enable(cmdq->clock) < 0);
374 		WARN_ON(cmdq_thread_reset(cmdq, thread) < 0);
375 
376 		writel(task->pa_base, thread->base + CMDQ_THR_CURR_ADDR);
377 		writel(task->pa_base + pkt->cmd_buf_size,
378 		       thread->base + CMDQ_THR_END_ADDR);
379 		writel(thread->priority, thread->base + CMDQ_THR_PRIORITY);
380 		writel(CMDQ_THR_IRQ_EN, thread->base + CMDQ_THR_IRQ_ENABLE);
381 		writel(CMDQ_THR_ENABLED, thread->base + CMDQ_THR_ENABLE_TASK);
382 	} else {
383 		WARN_ON(cmdq_thread_suspend(cmdq, thread) < 0);
384 		curr_pa = readl(thread->base + CMDQ_THR_CURR_ADDR);
385 		end_pa = readl(thread->base + CMDQ_THR_END_ADDR);
386 
387 		/*
388 		 * Atomic execution should remove the following wfe, i.e. only
389 		 * wait event at first task, and prevent to pause when running.
390 		 */
391 		if (thread->atomic_exec) {
392 			/* GCE is executing if command is not WFE */
393 			if (!cmdq_thread_is_in_wfe(thread)) {
394 				cmdq_thread_resume(thread);
395 				cmdq_thread_wait_end(thread, end_pa);
396 				WARN_ON(cmdq_thread_suspend(cmdq, thread) < 0);
397 				/* set to this task directly */
398 				writel(task->pa_base,
399 				       thread->base + CMDQ_THR_CURR_ADDR);
400 			} else {
401 				cmdq_task_insert_into_thread(task);
402 				cmdq_task_remove_wfe(task);
403 				smp_mb(); /* modify jump before enable thread */
404 			}
405 		} else {
406 			/* check boundary */
407 			if (curr_pa == end_pa - CMDQ_INST_SIZE ||
408 			    curr_pa == end_pa) {
409 				/* set to this task directly */
410 				writel(task->pa_base,
411 				       thread->base + CMDQ_THR_CURR_ADDR);
412 			} else {
413 				cmdq_task_insert_into_thread(task);
414 				smp_mb(); /* modify jump before enable thread */
415 			}
416 		}
417 		writel(task->pa_base + pkt->cmd_buf_size,
418 		       thread->base + CMDQ_THR_END_ADDR);
419 		cmdq_thread_resume(thread);
420 	}
421 	list_move_tail(&task->list_entry, &thread->task_busy_list);
422 
423 	return 0;
424 }
425 
426 static int cmdq_mbox_startup(struct mbox_chan *chan)
427 {
428 	return 0;
429 }
430 
431 static void cmdq_mbox_shutdown(struct mbox_chan *chan)
432 {
433 }
434 
435 static const struct mbox_chan_ops cmdq_mbox_chan_ops = {
436 	.send_data = cmdq_mbox_send_data,
437 	.startup = cmdq_mbox_startup,
438 	.shutdown = cmdq_mbox_shutdown,
439 };
440 
441 static struct mbox_chan *cmdq_xlate(struct mbox_controller *mbox,
442 		const struct of_phandle_args *sp)
443 {
444 	int ind = sp->args[0];
445 	struct cmdq_thread *thread;
446 
447 	if (ind >= mbox->num_chans)
448 		return ERR_PTR(-EINVAL);
449 
450 	thread = (struct cmdq_thread *)mbox->chans[ind].con_priv;
451 	thread->priority = sp->args[1];
452 	thread->atomic_exec = (sp->args[2] != 0);
453 	thread->chan = &mbox->chans[ind];
454 
455 	return &mbox->chans[ind];
456 }
457 
458 static int cmdq_probe(struct platform_device *pdev)
459 {
460 	struct device *dev = &pdev->dev;
461 	struct resource *res;
462 	struct cmdq *cmdq;
463 	int err, i;
464 
465 	cmdq = devm_kzalloc(dev, sizeof(*cmdq), GFP_KERNEL);
466 	if (!cmdq)
467 		return -ENOMEM;
468 
469 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
470 	cmdq->base = devm_ioremap_resource(dev, res);
471 	if (IS_ERR(cmdq->base)) {
472 		dev_err(dev, "failed to ioremap gce\n");
473 		return PTR_ERR(cmdq->base);
474 	}
475 
476 	cmdq->irq = platform_get_irq(pdev, 0);
477 	if (!cmdq->irq) {
478 		dev_err(dev, "failed to get irq\n");
479 		return -EINVAL;
480 	}
481 
482 	cmdq->thread_nr = (u32)(unsigned long)of_device_get_match_data(dev);
483 	cmdq->irq_mask = GENMASK(cmdq->thread_nr - 1, 0);
484 	err = devm_request_irq(dev, cmdq->irq, cmdq_irq_handler, IRQF_SHARED,
485 			       "mtk_cmdq", cmdq);
486 	if (err < 0) {
487 		dev_err(dev, "failed to register ISR (%d)\n", err);
488 		return err;
489 	}
490 
491 	dev_dbg(dev, "cmdq device: addr:0x%p, va:0x%p, irq:%d\n",
492 		dev, cmdq->base, cmdq->irq);
493 
494 	cmdq->clock = devm_clk_get(dev, "gce");
495 	if (IS_ERR(cmdq->clock)) {
496 		dev_err(dev, "failed to get gce clk\n");
497 		return PTR_ERR(cmdq->clock);
498 	}
499 
500 	cmdq->mbox.dev = dev;
501 	cmdq->mbox.chans = devm_kcalloc(dev, cmdq->thread_nr,
502 					sizeof(*cmdq->mbox.chans), GFP_KERNEL);
503 	if (!cmdq->mbox.chans)
504 		return -ENOMEM;
505 
506 	cmdq->mbox.num_chans = cmdq->thread_nr;
507 	cmdq->mbox.ops = &cmdq_mbox_chan_ops;
508 	cmdq->mbox.of_xlate = cmdq_xlate;
509 
510 	/* make use of TXDONE_BY_ACK */
511 	cmdq->mbox.txdone_irq = false;
512 	cmdq->mbox.txdone_poll = false;
513 
514 	cmdq->thread = devm_kcalloc(dev, cmdq->thread_nr,
515 					sizeof(*cmdq->thread), GFP_KERNEL);
516 	if (!cmdq->thread)
517 		return -ENOMEM;
518 
519 	for (i = 0; i < cmdq->thread_nr; i++) {
520 		cmdq->thread[i].base = cmdq->base + CMDQ_THR_BASE +
521 				CMDQ_THR_SIZE * i;
522 		INIT_LIST_HEAD(&cmdq->thread[i].task_busy_list);
523 		cmdq->mbox.chans[i].con_priv = (void *)&cmdq->thread[i];
524 	}
525 
526 	err = devm_mbox_controller_register(dev, &cmdq->mbox);
527 	if (err < 0) {
528 		dev_err(dev, "failed to register mailbox: %d\n", err);
529 		return err;
530 	}
531 
532 	platform_set_drvdata(pdev, cmdq);
533 	WARN_ON(clk_prepare(cmdq->clock) < 0);
534 
535 	cmdq_init(cmdq);
536 
537 	return 0;
538 }
539 
540 static const struct dev_pm_ops cmdq_pm_ops = {
541 	.suspend = cmdq_suspend,
542 	.resume = cmdq_resume,
543 };
544 
545 static const struct of_device_id cmdq_of_ids[] = {
546 	{.compatible = "mediatek,mt8173-gce", .data = (void *)16},
547 	{.compatible = "mediatek,mt8183-gce", .data = (void *)24},
548 	{}
549 };
550 
551 static struct platform_driver cmdq_drv = {
552 	.probe = cmdq_probe,
553 	.remove = cmdq_remove,
554 	.driver = {
555 		.name = "mtk_cmdq",
556 		.pm = &cmdq_pm_ops,
557 		.of_match_table = cmdq_of_ids,
558 	}
559 };
560 
561 static int __init cmdq_drv_init(void)
562 {
563 	return platform_driver_register(&cmdq_drv);
564 }
565 
566 static void __exit cmdq_drv_exit(void)
567 {
568 	platform_driver_unregister(&cmdq_drv);
569 }
570 
571 subsys_initcall(cmdq_drv_init);
572 module_exit(cmdq_drv_exit);
573 
574 MODULE_LICENSE("GPL v2");
575