xref: /linux/drivers/hwtracing/intel_th/msu.c (revision cb787f4a)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Intel(R) Trace Hub Memory Storage Unit
4  *
5  * Copyright (C) 2014-2015 Intel Corporation.
6  */
7 
8 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
9 
10 #include <linux/types.h>
11 #include <linux/module.h>
12 #include <linux/device.h>
13 #include <linux/uaccess.h>
14 #include <linux/sizes.h>
15 #include <linux/printk.h>
16 #include <linux/slab.h>
17 #include <linux/mm.h>
18 #include <linux/fs.h>
19 #include <linux/io.h>
20 #include <linux/workqueue.h>
21 #include <linux/dma-mapping.h>
22 
23 #ifdef CONFIG_X86
24 #include <asm/set_memory.h>
25 #endif
26 
27 #include <linux/intel_th.h>
28 #include "intel_th.h"
29 #include "msu.h"
30 
31 #define msc_dev(x) (&(x)->thdev->dev)
32 
33 /*
34  * Lockout state transitions:
35  *   READY -> INUSE -+-> LOCKED -+-> READY -> etc.
36  *                   \-----------/
37  * WIN_READY:	window can be used by HW
38  * WIN_INUSE:	window is in use
39  * WIN_LOCKED:	window is filled up and is being processed by the buffer
40  * handling code
41  *
42  * All state transitions happen automatically, except for the LOCKED->READY,
43  * which needs to be signalled by the buffer code by calling
44  * intel_th_msc_window_unlock().
45  *
46  * When the interrupt handler has to switch to the next window, it checks
47  * whether it's READY, and if it is, it performs the switch and tracing
48  * continues. If it's LOCKED, it stops the trace.
49  */
50 enum lockout_state {
51 	WIN_READY = 0,
52 	WIN_INUSE,
53 	WIN_LOCKED
54 };
55 
56 /**
57  * struct msc_window - multiblock mode window descriptor
58  * @entry:	window list linkage (msc::win_list)
59  * @pgoff:	page offset into the buffer that this window starts at
60  * @lockout:	lockout state, see comment below
61  * @lo_lock:	lockout state serialization
62  * @nr_blocks:	number of blocks (pages) in this window
63  * @nr_segs:	number of segments in this window (<= @nr_blocks)
64  * @msc:	pointer to the MSC device
65  * @_sgt:	array of block descriptors
66  * @sgt:	array of block descriptors
67  */
68 struct msc_window {
69 	struct list_head	entry;
70 	unsigned long		pgoff;
71 	enum lockout_state	lockout;
72 	spinlock_t		lo_lock;
73 	unsigned int		nr_blocks;
74 	unsigned int		nr_segs;
75 	struct msc		*msc;
76 	struct sg_table		_sgt;
77 	struct sg_table		*sgt;
78 };
79 
80 /**
81  * struct msc_iter - iterator for msc buffer
82  * @entry:		msc::iter_list linkage
83  * @msc:		pointer to the MSC device
84  * @start_win:		oldest window
85  * @win:		current window
86  * @offset:		current logical offset into the buffer
87  * @start_block:	oldest block in the window
88  * @block:		block number in the window
89  * @block_off:		offset into current block
90  * @wrap_count:		block wrapping handling
91  * @eof:		end of buffer reached
92  */
93 struct msc_iter {
94 	struct list_head	entry;
95 	struct msc		*msc;
96 	struct msc_window	*start_win;
97 	struct msc_window	*win;
98 	unsigned long		offset;
99 	struct scatterlist	*start_block;
100 	struct scatterlist	*block;
101 	unsigned int		block_off;
102 	unsigned int		wrap_count;
103 	unsigned int		eof;
104 };
105 
106 /**
107  * struct msc - MSC device representation
108  * @reg_base:		register window base address
109  * @thdev:		intel_th_device pointer
110  * @mbuf:		MSU buffer, if assigned
111  * @mbuf_priv		MSU buffer's private data, if @mbuf
112  * @win_list:		list of windows in multiblock mode
113  * @single_sgt:		single mode buffer
114  * @cur_win:		current window
115  * @nr_pages:		total number of pages allocated for this buffer
116  * @single_sz:		amount of data in single mode
117  * @single_wrap:	single mode wrap occurred
118  * @base:		buffer's base pointer
119  * @base_addr:		buffer's base address
120  * @user_count:		number of users of the buffer
121  * @mmap_count:		number of mappings
122  * @buf_mutex:		mutex to serialize access to buffer-related bits
123  * @enabled:		MSC is enabled
124  * @wrap:		wrapping is enabled
125  * @mode:		MSC operating mode
126  * @burst_len:		write burst length
127  * @index:		number of this MSC in the MSU
128  */
129 struct msc {
130 	void __iomem		*reg_base;
131 	void __iomem		*msu_base;
132 	struct intel_th_device	*thdev;
133 
134 	const struct msu_buffer	*mbuf;
135 	void			*mbuf_priv;
136 
137 	struct work_struct	work;
138 	struct list_head	win_list;
139 	struct sg_table		single_sgt;
140 	struct msc_window	*cur_win;
141 	struct msc_window	*switch_on_unlock;
142 	unsigned long		nr_pages;
143 	unsigned long		single_sz;
144 	unsigned int		single_wrap : 1;
145 	void			*base;
146 	dma_addr_t		base_addr;
147 	u32			orig_addr;
148 	u32			orig_sz;
149 
150 	/* <0: no buffer, 0: no users, >0: active users */
151 	atomic_t		user_count;
152 
153 	atomic_t		mmap_count;
154 	struct mutex		buf_mutex;
155 
156 	struct list_head	iter_list;
157 
158 	bool			stop_on_full;
159 
160 	/* config */
161 	unsigned int		enabled : 1,
162 				wrap	: 1,
163 				do_irq	: 1,
164 				multi_is_broken : 1;
165 	unsigned int		mode;
166 	unsigned int		burst_len;
167 	unsigned int		index;
168 };
169 
170 static LIST_HEAD(msu_buffer_list);
171 static DEFINE_MUTEX(msu_buffer_mutex);
172 
173 /**
174  * struct msu_buffer_entry - internal MSU buffer bookkeeping
175  * @entry:	link to msu_buffer_list
176  * @mbuf:	MSU buffer object
177  * @owner:	module that provides this MSU buffer
178  */
179 struct msu_buffer_entry {
180 	struct list_head	entry;
181 	const struct msu_buffer	*mbuf;
182 	struct module		*owner;
183 };
184 
__msu_buffer_entry_find(const char * name)185 static struct msu_buffer_entry *__msu_buffer_entry_find(const char *name)
186 {
187 	struct msu_buffer_entry *mbe;
188 
189 	lockdep_assert_held(&msu_buffer_mutex);
190 
191 	list_for_each_entry(mbe, &msu_buffer_list, entry) {
192 		if (!strcmp(mbe->mbuf->name, name))
193 			return mbe;
194 	}
195 
196 	return NULL;
197 }
198 
199 static const struct msu_buffer *
msu_buffer_get(const char * name)200 msu_buffer_get(const char *name)
201 {
202 	struct msu_buffer_entry *mbe;
203 
204 	mutex_lock(&msu_buffer_mutex);
205 	mbe = __msu_buffer_entry_find(name);
206 	if (mbe && !try_module_get(mbe->owner))
207 		mbe = NULL;
208 	mutex_unlock(&msu_buffer_mutex);
209 
210 	return mbe ? mbe->mbuf : NULL;
211 }
212 
msu_buffer_put(const struct msu_buffer * mbuf)213 static void msu_buffer_put(const struct msu_buffer *mbuf)
214 {
215 	struct msu_buffer_entry *mbe;
216 
217 	mutex_lock(&msu_buffer_mutex);
218 	mbe = __msu_buffer_entry_find(mbuf->name);
219 	if (mbe)
220 		module_put(mbe->owner);
221 	mutex_unlock(&msu_buffer_mutex);
222 }
223 
intel_th_msu_buffer_register(const struct msu_buffer * mbuf,struct module * owner)224 int intel_th_msu_buffer_register(const struct msu_buffer *mbuf,
225 				 struct module *owner)
226 {
227 	struct msu_buffer_entry *mbe;
228 	int ret = 0;
229 
230 	mbe = kzalloc(sizeof(*mbe), GFP_KERNEL);
231 	if (!mbe)
232 		return -ENOMEM;
233 
234 	mutex_lock(&msu_buffer_mutex);
235 	if (__msu_buffer_entry_find(mbuf->name)) {
236 		ret = -EEXIST;
237 		kfree(mbe);
238 		goto unlock;
239 	}
240 
241 	mbe->mbuf = mbuf;
242 	mbe->owner = owner;
243 	list_add_tail(&mbe->entry, &msu_buffer_list);
244 unlock:
245 	mutex_unlock(&msu_buffer_mutex);
246 
247 	return ret;
248 }
249 EXPORT_SYMBOL_GPL(intel_th_msu_buffer_register);
250 
intel_th_msu_buffer_unregister(const struct msu_buffer * mbuf)251 void intel_th_msu_buffer_unregister(const struct msu_buffer *mbuf)
252 {
253 	struct msu_buffer_entry *mbe;
254 
255 	mutex_lock(&msu_buffer_mutex);
256 	mbe = __msu_buffer_entry_find(mbuf->name);
257 	if (mbe) {
258 		list_del(&mbe->entry);
259 		kfree(mbe);
260 	}
261 	mutex_unlock(&msu_buffer_mutex);
262 }
263 EXPORT_SYMBOL_GPL(intel_th_msu_buffer_unregister);
264 
msc_block_is_empty(struct msc_block_desc * bdesc)265 static inline bool msc_block_is_empty(struct msc_block_desc *bdesc)
266 {
267 	/* header hasn't been written */
268 	if (!bdesc->valid_dw)
269 		return true;
270 
271 	/* valid_dw includes the header */
272 	if (!msc_data_sz(bdesc))
273 		return true;
274 
275 	return false;
276 }
277 
msc_win_base_sg(struct msc_window * win)278 static inline struct scatterlist *msc_win_base_sg(struct msc_window *win)
279 {
280 	return win->sgt->sgl;
281 }
282 
msc_win_base(struct msc_window * win)283 static inline struct msc_block_desc *msc_win_base(struct msc_window *win)
284 {
285 	return sg_virt(msc_win_base_sg(win));
286 }
287 
msc_win_base_dma(struct msc_window * win)288 static inline dma_addr_t msc_win_base_dma(struct msc_window *win)
289 {
290 	return sg_dma_address(msc_win_base_sg(win));
291 }
292 
293 static inline unsigned long
msc_win_base_pfn(struct msc_window * win)294 msc_win_base_pfn(struct msc_window *win)
295 {
296 	return PFN_DOWN(msc_win_base_dma(win));
297 }
298 
299 /**
300  * msc_is_last_win() - check if a window is the last one for a given MSC
301  * @win:	window
302  * Return:	true if @win is the last window in MSC's multiblock buffer
303  */
msc_is_last_win(struct msc_window * win)304 static inline bool msc_is_last_win(struct msc_window *win)
305 {
306 	return win->entry.next == &win->msc->win_list;
307 }
308 
309 /**
310  * msc_next_window() - return next window in the multiblock buffer
311  * @win:	current window
312  *
313  * Return:	window following the current one
314  */
msc_next_window(struct msc_window * win)315 static struct msc_window *msc_next_window(struct msc_window *win)
316 {
317 	if (msc_is_last_win(win))
318 		return list_first_entry(&win->msc->win_list, struct msc_window,
319 					entry);
320 
321 	return list_next_entry(win, entry);
322 }
323 
msc_win_total_sz(struct msc_window * win)324 static size_t msc_win_total_sz(struct msc_window *win)
325 {
326 	struct scatterlist *sg;
327 	unsigned int blk;
328 	size_t size = 0;
329 
330 	for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
331 		struct msc_block_desc *bdesc = sg_virt(sg);
332 
333 		if (msc_block_wrapped(bdesc))
334 			return (size_t)win->nr_blocks << PAGE_SHIFT;
335 
336 		size += msc_total_sz(bdesc);
337 		if (msc_block_last_written(bdesc))
338 			break;
339 	}
340 
341 	return size;
342 }
343 
344 /**
345  * msc_find_window() - find a window matching a given sg_table
346  * @msc:	MSC device
347  * @sgt:	SG table of the window
348  * @nonempty:	skip over empty windows
349  *
350  * Return:	MSC window structure pointer or NULL if the window
351  *		could not be found.
352  */
353 static struct msc_window *
msc_find_window(struct msc * msc,struct sg_table * sgt,bool nonempty)354 msc_find_window(struct msc *msc, struct sg_table *sgt, bool nonempty)
355 {
356 	struct msc_window *win;
357 	unsigned int found = 0;
358 
359 	if (list_empty(&msc->win_list))
360 		return NULL;
361 
362 	/*
363 	 * we might need a radix tree for this, depending on how
364 	 * many windows a typical user would allocate; ideally it's
365 	 * something like 2, in which case we're good
366 	 */
367 	list_for_each_entry(win, &msc->win_list, entry) {
368 		if (win->sgt == sgt)
369 			found++;
370 
371 		/* skip the empty ones */
372 		if (nonempty && msc_block_is_empty(msc_win_base(win)))
373 			continue;
374 
375 		if (found)
376 			return win;
377 	}
378 
379 	return NULL;
380 }
381 
382 /**
383  * msc_oldest_window() - locate the window with oldest data
384  * @msc:	MSC device
385  *
386  * This should only be used in multiblock mode. Caller should hold the
387  * msc::user_count reference.
388  *
389  * Return:	the oldest window with valid data
390  */
msc_oldest_window(struct msc * msc)391 static struct msc_window *msc_oldest_window(struct msc *msc)
392 {
393 	struct msc_window *win;
394 
395 	if (list_empty(&msc->win_list))
396 		return NULL;
397 
398 	win = msc_find_window(msc, msc_next_window(msc->cur_win)->sgt, true);
399 	if (win)
400 		return win;
401 
402 	return list_first_entry(&msc->win_list, struct msc_window, entry);
403 }
404 
405 /**
406  * msc_win_oldest_sg() - locate the oldest block in a given window
407  * @win:	window to look at
408  *
409  * Return:	index of the block with the oldest data
410  */
msc_win_oldest_sg(struct msc_window * win)411 static struct scatterlist *msc_win_oldest_sg(struct msc_window *win)
412 {
413 	unsigned int blk;
414 	struct scatterlist *sg;
415 	struct msc_block_desc *bdesc = msc_win_base(win);
416 
417 	/* without wrapping, first block is the oldest */
418 	if (!msc_block_wrapped(bdesc))
419 		return msc_win_base_sg(win);
420 
421 	/*
422 	 * with wrapping, last written block contains both the newest and the
423 	 * oldest data for this window.
424 	 */
425 	for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
426 		struct msc_block_desc *bdesc = sg_virt(sg);
427 
428 		if (msc_block_last_written(bdesc))
429 			return sg;
430 	}
431 
432 	return msc_win_base_sg(win);
433 }
434 
msc_iter_bdesc(struct msc_iter * iter)435 static struct msc_block_desc *msc_iter_bdesc(struct msc_iter *iter)
436 {
437 	return sg_virt(iter->block);
438 }
439 
msc_iter_install(struct msc * msc)440 static struct msc_iter *msc_iter_install(struct msc *msc)
441 {
442 	struct msc_iter *iter;
443 
444 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
445 	if (!iter)
446 		return ERR_PTR(-ENOMEM);
447 
448 	mutex_lock(&msc->buf_mutex);
449 
450 	/*
451 	 * Reading and tracing are mutually exclusive; if msc is
452 	 * enabled, open() will fail; otherwise existing readers
453 	 * will prevent enabling the msc and the rest of fops don't
454 	 * need to worry about it.
455 	 */
456 	if (msc->enabled) {
457 		kfree(iter);
458 		iter = ERR_PTR(-EBUSY);
459 		goto unlock;
460 	}
461 
462 	iter->msc = msc;
463 
464 	list_add_tail(&iter->entry, &msc->iter_list);
465 unlock:
466 	mutex_unlock(&msc->buf_mutex);
467 
468 	return iter;
469 }
470 
msc_iter_remove(struct msc_iter * iter,struct msc * msc)471 static void msc_iter_remove(struct msc_iter *iter, struct msc *msc)
472 {
473 	mutex_lock(&msc->buf_mutex);
474 	list_del(&iter->entry);
475 	mutex_unlock(&msc->buf_mutex);
476 
477 	kfree(iter);
478 }
479 
msc_iter_block_start(struct msc_iter * iter)480 static void msc_iter_block_start(struct msc_iter *iter)
481 {
482 	if (iter->start_block)
483 		return;
484 
485 	iter->start_block = msc_win_oldest_sg(iter->win);
486 	iter->block = iter->start_block;
487 	iter->wrap_count = 0;
488 
489 	/*
490 	 * start with the block with oldest data; if data has wrapped
491 	 * in this window, it should be in this block
492 	 */
493 	if (msc_block_wrapped(msc_iter_bdesc(iter)))
494 		iter->wrap_count = 2;
495 
496 }
497 
msc_iter_win_start(struct msc_iter * iter,struct msc * msc)498 static int msc_iter_win_start(struct msc_iter *iter, struct msc *msc)
499 {
500 	/* already started, nothing to do */
501 	if (iter->start_win)
502 		return 0;
503 
504 	iter->start_win = msc_oldest_window(msc);
505 	if (!iter->start_win)
506 		return -EINVAL;
507 
508 	iter->win = iter->start_win;
509 	iter->start_block = NULL;
510 
511 	msc_iter_block_start(iter);
512 
513 	return 0;
514 }
515 
msc_iter_win_advance(struct msc_iter * iter)516 static int msc_iter_win_advance(struct msc_iter *iter)
517 {
518 	iter->win = msc_next_window(iter->win);
519 	iter->start_block = NULL;
520 
521 	if (iter->win == iter->start_win) {
522 		iter->eof++;
523 		return 1;
524 	}
525 
526 	msc_iter_block_start(iter);
527 
528 	return 0;
529 }
530 
msc_iter_block_advance(struct msc_iter * iter)531 static int msc_iter_block_advance(struct msc_iter *iter)
532 {
533 	iter->block_off = 0;
534 
535 	/* wrapping */
536 	if (iter->wrap_count && iter->block == iter->start_block) {
537 		iter->wrap_count--;
538 		if (!iter->wrap_count)
539 			/* copied newest data from the wrapped block */
540 			return msc_iter_win_advance(iter);
541 	}
542 
543 	/* no wrapping, check for last written block */
544 	if (!iter->wrap_count && msc_block_last_written(msc_iter_bdesc(iter)))
545 		/* copied newest data for the window */
546 		return msc_iter_win_advance(iter);
547 
548 	/* block advance */
549 	if (sg_is_last(iter->block))
550 		iter->block = msc_win_base_sg(iter->win);
551 	else
552 		iter->block = sg_next(iter->block);
553 
554 	/* no wrapping, sanity check in case there is no last written block */
555 	if (!iter->wrap_count && iter->block == iter->start_block)
556 		return msc_iter_win_advance(iter);
557 
558 	return 0;
559 }
560 
561 /**
562  * msc_buffer_iterate() - go through multiblock buffer's data
563  * @iter:	iterator structure
564  * @size:	amount of data to scan
565  * @data:	callback's private data
566  * @fn:		iterator callback
567  *
568  * This will start at the window which will be written to next (containing
569  * the oldest data) and work its way to the current window, calling @fn
570  * for each chunk of data as it goes.
571  *
572  * Caller should have msc::user_count reference to make sure the buffer
573  * doesn't disappear from under us.
574  *
575  * Return:	amount of data actually scanned.
576  */
577 static ssize_t
msc_buffer_iterate(struct msc_iter * iter,size_t size,void * data,unsigned long (* fn)(void *,void *,size_t))578 msc_buffer_iterate(struct msc_iter *iter, size_t size, void *data,
579 		   unsigned long (*fn)(void *, void *, size_t))
580 {
581 	struct msc *msc = iter->msc;
582 	size_t len = size;
583 	unsigned int advance;
584 
585 	if (iter->eof)
586 		return 0;
587 
588 	/* start with the oldest window */
589 	if (msc_iter_win_start(iter, msc))
590 		return 0;
591 
592 	do {
593 		unsigned long data_bytes = msc_data_sz(msc_iter_bdesc(iter));
594 		void *src = (void *)msc_iter_bdesc(iter) + MSC_BDESC;
595 		size_t tocopy = data_bytes, copied = 0;
596 		size_t remaining = 0;
597 
598 		advance = 1;
599 
600 		/*
601 		 * If block wrapping happened, we need to visit the last block
602 		 * twice, because it contains both the oldest and the newest
603 		 * data in this window.
604 		 *
605 		 * First time (wrap_count==2), in the very beginning, to collect
606 		 * the oldest data, which is in the range
607 		 * (data_bytes..DATA_IN_PAGE).
608 		 *
609 		 * Second time (wrap_count==1), it's just like any other block,
610 		 * containing data in the range of [MSC_BDESC..data_bytes].
611 		 */
612 		if (iter->block == iter->start_block && iter->wrap_count == 2) {
613 			tocopy = DATA_IN_PAGE - data_bytes;
614 			src += data_bytes;
615 		}
616 
617 		if (!tocopy)
618 			goto next_block;
619 
620 		tocopy -= iter->block_off;
621 		src += iter->block_off;
622 
623 		if (len < tocopy) {
624 			tocopy = len;
625 			advance = 0;
626 		}
627 
628 		remaining = fn(data, src, tocopy);
629 
630 		if (remaining)
631 			advance = 0;
632 
633 		copied = tocopy - remaining;
634 		len -= copied;
635 		iter->block_off += copied;
636 		iter->offset += copied;
637 
638 		if (!advance)
639 			break;
640 
641 next_block:
642 		if (msc_iter_block_advance(iter))
643 			break;
644 
645 	} while (len);
646 
647 	return size - len;
648 }
649 
650 /**
651  * msc_buffer_clear_hw_header() - clear hw header for multiblock
652  * @msc:	MSC device
653  */
msc_buffer_clear_hw_header(struct msc * msc)654 static void msc_buffer_clear_hw_header(struct msc *msc)
655 {
656 	struct msc_window *win;
657 	struct scatterlist *sg;
658 
659 	list_for_each_entry(win, &msc->win_list, entry) {
660 		unsigned int blk;
661 
662 		for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
663 			struct msc_block_desc *bdesc = sg_virt(sg);
664 
665 			memset_startat(bdesc, 0, hw_tag);
666 		}
667 	}
668 }
669 
intel_th_msu_init(struct msc * msc)670 static int intel_th_msu_init(struct msc *msc)
671 {
672 	u32 mintctl, msusts;
673 
674 	if (!msc->do_irq)
675 		return 0;
676 
677 	if (!msc->mbuf)
678 		return 0;
679 
680 	mintctl = ioread32(msc->msu_base + REG_MSU_MINTCTL);
681 	mintctl |= msc->index ? M1BLIE : M0BLIE;
682 	iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL);
683 	if (mintctl != ioread32(msc->msu_base + REG_MSU_MINTCTL)) {
684 		dev_info(msc_dev(msc), "MINTCTL ignores writes: no usable interrupts\n");
685 		msc->do_irq = 0;
686 		return 0;
687 	}
688 
689 	msusts = ioread32(msc->msu_base + REG_MSU_MSUSTS);
690 	iowrite32(msusts, msc->msu_base + REG_MSU_MSUSTS);
691 
692 	return 0;
693 }
694 
intel_th_msu_deinit(struct msc * msc)695 static void intel_th_msu_deinit(struct msc *msc)
696 {
697 	u32 mintctl;
698 
699 	if (!msc->do_irq)
700 		return;
701 
702 	mintctl = ioread32(msc->msu_base + REG_MSU_MINTCTL);
703 	mintctl &= msc->index ? ~M1BLIE : ~M0BLIE;
704 	iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL);
705 }
706 
msc_win_set_lockout(struct msc_window * win,enum lockout_state expect,enum lockout_state new)707 static int msc_win_set_lockout(struct msc_window *win,
708 			       enum lockout_state expect,
709 			       enum lockout_state new)
710 {
711 	enum lockout_state old;
712 	unsigned long flags;
713 	int ret = 0;
714 
715 	if (!win->msc->mbuf)
716 		return 0;
717 
718 	spin_lock_irqsave(&win->lo_lock, flags);
719 	old = win->lockout;
720 
721 	if (old != expect) {
722 		ret = -EINVAL;
723 		goto unlock;
724 	}
725 
726 	win->lockout = new;
727 
728 	if (old == expect && new == WIN_LOCKED)
729 		atomic_inc(&win->msc->user_count);
730 	else if (old == expect && old == WIN_LOCKED)
731 		atomic_dec(&win->msc->user_count);
732 
733 unlock:
734 	spin_unlock_irqrestore(&win->lo_lock, flags);
735 
736 	if (ret) {
737 		if (expect == WIN_READY && old == WIN_LOCKED)
738 			return -EBUSY;
739 
740 		/* from intel_th_msc_window_unlock(), don't warn if not locked */
741 		if (expect == WIN_LOCKED && old == new)
742 			return 0;
743 
744 		dev_warn_ratelimited(msc_dev(win->msc),
745 				     "expected lockout state %d, got %d\n",
746 				     expect, old);
747 	}
748 
749 	return ret;
750 }
751 /**
752  * msc_configure() - set up MSC hardware
753  * @msc:	the MSC device to configure
754  *
755  * Program storage mode, wrapping, burst length and trace buffer address
756  * into a given MSC. Then, enable tracing and set msc::enabled.
757  * The latter is serialized on msc::buf_mutex, so make sure to hold it.
758  *
759  * Return:	%0 for success or a negative error code otherwise.
760  */
msc_configure(struct msc * msc)761 static int msc_configure(struct msc *msc)
762 {
763 	u32 reg;
764 
765 	lockdep_assert_held(&msc->buf_mutex);
766 
767 	if (msc->mode > MSC_MODE_MULTI)
768 		return -EINVAL;
769 
770 	if (msc->mode == MSC_MODE_MULTI) {
771 		if (msc_win_set_lockout(msc->cur_win, WIN_READY, WIN_INUSE))
772 			return -EBUSY;
773 
774 		msc_buffer_clear_hw_header(msc);
775 	}
776 
777 	msc->orig_addr = ioread32(msc->reg_base + REG_MSU_MSC0BAR);
778 	msc->orig_sz   = ioread32(msc->reg_base + REG_MSU_MSC0SIZE);
779 
780 	reg = msc->base_addr >> PAGE_SHIFT;
781 	iowrite32(reg, msc->reg_base + REG_MSU_MSC0BAR);
782 
783 	if (msc->mode == MSC_MODE_SINGLE) {
784 		reg = msc->nr_pages;
785 		iowrite32(reg, msc->reg_base + REG_MSU_MSC0SIZE);
786 	}
787 
788 	reg = ioread32(msc->reg_base + REG_MSU_MSC0CTL);
789 	reg &= ~(MSC_MODE | MSC_WRAPEN | MSC_EN | MSC_RD_HDR_OVRD);
790 
791 	reg |= MSC_EN;
792 	reg |= msc->mode << __ffs(MSC_MODE);
793 	reg |= msc->burst_len << __ffs(MSC_LEN);
794 
795 	if (msc->wrap)
796 		reg |= MSC_WRAPEN;
797 
798 	iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL);
799 
800 	intel_th_msu_init(msc);
801 
802 	msc->thdev->output.multiblock = msc->mode == MSC_MODE_MULTI;
803 	intel_th_trace_enable(msc->thdev);
804 	msc->enabled = 1;
805 
806 	if (msc->mbuf && msc->mbuf->activate)
807 		msc->mbuf->activate(msc->mbuf_priv);
808 
809 	return 0;
810 }
811 
812 /**
813  * msc_disable() - disable MSC hardware
814  * @msc:	MSC device to disable
815  *
816  * If @msc is enabled, disable tracing on the switch and then disable MSC
817  * storage. Caller must hold msc::buf_mutex.
818  */
msc_disable(struct msc * msc)819 static void msc_disable(struct msc *msc)
820 {
821 	struct msc_window *win = msc->cur_win;
822 	u32 reg;
823 
824 	lockdep_assert_held(&msc->buf_mutex);
825 
826 	if (msc->mode == MSC_MODE_MULTI)
827 		msc_win_set_lockout(win, WIN_INUSE, WIN_LOCKED);
828 
829 	if (msc->mbuf && msc->mbuf->deactivate)
830 		msc->mbuf->deactivate(msc->mbuf_priv);
831 	intel_th_msu_deinit(msc);
832 	intel_th_trace_disable(msc->thdev);
833 
834 	if (msc->mode == MSC_MODE_SINGLE) {
835 		reg = ioread32(msc->reg_base + REG_MSU_MSC0STS);
836 		msc->single_wrap = !!(reg & MSCSTS_WRAPSTAT);
837 
838 		reg = ioread32(msc->reg_base + REG_MSU_MSC0MWP);
839 		msc->single_sz = reg & ((msc->nr_pages << PAGE_SHIFT) - 1);
840 		dev_dbg(msc_dev(msc), "MSCnMWP: %08x/%08lx, wrap: %d\n",
841 			reg, msc->single_sz, msc->single_wrap);
842 	}
843 
844 	reg = ioread32(msc->reg_base + REG_MSU_MSC0CTL);
845 	reg &= ~MSC_EN;
846 	iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL);
847 
848 	if (msc->mbuf && msc->mbuf->ready)
849 		msc->mbuf->ready(msc->mbuf_priv, win->sgt,
850 				 msc_win_total_sz(win));
851 
852 	msc->enabled = 0;
853 
854 	iowrite32(msc->orig_addr, msc->reg_base + REG_MSU_MSC0BAR);
855 	iowrite32(msc->orig_sz, msc->reg_base + REG_MSU_MSC0SIZE);
856 
857 	dev_dbg(msc_dev(msc), "MSCnNWSA: %08x\n",
858 		ioread32(msc->reg_base + REG_MSU_MSC0NWSA));
859 
860 	reg = ioread32(msc->reg_base + REG_MSU_MSC0STS);
861 	dev_dbg(msc_dev(msc), "MSCnSTS: %08x\n", reg);
862 
863 	reg = ioread32(msc->reg_base + REG_MSU_MSUSTS);
864 	reg &= msc->index ? MSUSTS_MSC1BLAST : MSUSTS_MSC0BLAST;
865 	iowrite32(reg, msc->reg_base + REG_MSU_MSUSTS);
866 }
867 
intel_th_msc_activate(struct intel_th_device * thdev)868 static int intel_th_msc_activate(struct intel_th_device *thdev)
869 {
870 	struct msc *msc = dev_get_drvdata(&thdev->dev);
871 	int ret = -EBUSY;
872 
873 	if (!atomic_inc_unless_negative(&msc->user_count))
874 		return -ENODEV;
875 
876 	mutex_lock(&msc->buf_mutex);
877 
878 	/* if there are readers, refuse */
879 	if (list_empty(&msc->iter_list))
880 		ret = msc_configure(msc);
881 
882 	mutex_unlock(&msc->buf_mutex);
883 
884 	if (ret)
885 		atomic_dec(&msc->user_count);
886 
887 	return ret;
888 }
889 
intel_th_msc_deactivate(struct intel_th_device * thdev)890 static void intel_th_msc_deactivate(struct intel_th_device *thdev)
891 {
892 	struct msc *msc = dev_get_drvdata(&thdev->dev);
893 
894 	mutex_lock(&msc->buf_mutex);
895 	if (msc->enabled) {
896 		msc_disable(msc);
897 		atomic_dec(&msc->user_count);
898 	}
899 	mutex_unlock(&msc->buf_mutex);
900 }
901 
902 /**
903  * msc_buffer_contig_alloc() - allocate a contiguous buffer for SINGLE mode
904  * @msc:	MSC device
905  * @size:	allocation size in bytes
906  *
907  * This modifies msc::base, which requires msc::buf_mutex to serialize, so the
908  * caller is expected to hold it.
909  *
910  * Return:	0 on success, -errno otherwise.
911  */
msc_buffer_contig_alloc(struct msc * msc,unsigned long size)912 static int msc_buffer_contig_alloc(struct msc *msc, unsigned long size)
913 {
914 	unsigned long nr_pages = size >> PAGE_SHIFT;
915 	unsigned int order = get_order(size);
916 	struct page *page;
917 	int ret;
918 
919 	if (!size)
920 		return 0;
921 
922 	ret = sg_alloc_table(&msc->single_sgt, 1, GFP_KERNEL);
923 	if (ret)
924 		goto err_out;
925 
926 	ret = -ENOMEM;
927 	page = alloc_pages(GFP_KERNEL | __GFP_ZERO | GFP_DMA32, order);
928 	if (!page)
929 		goto err_free_sgt;
930 
931 	split_page(page, order);
932 	sg_set_buf(msc->single_sgt.sgl, page_address(page), size);
933 
934 	ret = dma_map_sg(msc_dev(msc)->parent->parent, msc->single_sgt.sgl, 1,
935 			 DMA_FROM_DEVICE);
936 	if (ret < 0)
937 		goto err_free_pages;
938 
939 	msc->nr_pages = nr_pages;
940 	msc->base = page_address(page);
941 	msc->base_addr = sg_dma_address(msc->single_sgt.sgl);
942 
943 	return 0;
944 
945 err_free_pages:
946 	__free_pages(page, order);
947 
948 err_free_sgt:
949 	sg_free_table(&msc->single_sgt);
950 
951 err_out:
952 	return ret;
953 }
954 
955 /**
956  * msc_buffer_contig_free() - free a contiguous buffer
957  * @msc:	MSC configured in SINGLE mode
958  */
msc_buffer_contig_free(struct msc * msc)959 static void msc_buffer_contig_free(struct msc *msc)
960 {
961 	unsigned long off;
962 
963 	dma_unmap_sg(msc_dev(msc)->parent->parent, msc->single_sgt.sgl,
964 		     1, DMA_FROM_DEVICE);
965 	sg_free_table(&msc->single_sgt);
966 
967 	for (off = 0; off < msc->nr_pages << PAGE_SHIFT; off += PAGE_SIZE) {
968 		struct page *page = virt_to_page(msc->base + off);
969 
970 		page->mapping = NULL;
971 		__free_page(page);
972 	}
973 
974 	msc->nr_pages = 0;
975 }
976 
977 /**
978  * msc_buffer_contig_get_page() - find a page at a given offset
979  * @msc:	MSC configured in SINGLE mode
980  * @pgoff:	page offset
981  *
982  * Return:	page, if @pgoff is within the range, NULL otherwise.
983  */
msc_buffer_contig_get_page(struct msc * msc,unsigned long pgoff)984 static struct page *msc_buffer_contig_get_page(struct msc *msc,
985 					       unsigned long pgoff)
986 {
987 	if (pgoff >= msc->nr_pages)
988 		return NULL;
989 
990 	return virt_to_page(msc->base + (pgoff << PAGE_SHIFT));
991 }
992 
__msc_buffer_win_alloc(struct msc_window * win,unsigned int nr_segs)993 static int __msc_buffer_win_alloc(struct msc_window *win,
994 				  unsigned int nr_segs)
995 {
996 	struct scatterlist *sg_ptr;
997 	void *block;
998 	int i, ret;
999 
1000 	ret = sg_alloc_table(win->sgt, nr_segs, GFP_KERNEL);
1001 	if (ret)
1002 		return -ENOMEM;
1003 
1004 	for_each_sg(win->sgt->sgl, sg_ptr, nr_segs, i) {
1005 		block = dma_alloc_coherent(msc_dev(win->msc)->parent->parent,
1006 					  PAGE_SIZE, &sg_dma_address(sg_ptr),
1007 					  GFP_KERNEL);
1008 		if (!block)
1009 			goto err_nomem;
1010 
1011 		sg_set_buf(sg_ptr, block, PAGE_SIZE);
1012 	}
1013 
1014 	return nr_segs;
1015 
1016 err_nomem:
1017 	for_each_sg(win->sgt->sgl, sg_ptr, i, ret)
1018 		dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE,
1019 				  sg_virt(sg_ptr), sg_dma_address(sg_ptr));
1020 
1021 	sg_free_table(win->sgt);
1022 
1023 	return -ENOMEM;
1024 }
1025 
1026 #ifdef CONFIG_X86
msc_buffer_set_uc(struct msc * msc)1027 static void msc_buffer_set_uc(struct msc *msc)
1028 {
1029 	struct scatterlist *sg_ptr;
1030 	struct msc_window *win;
1031 	int i;
1032 
1033 	if (msc->mode == MSC_MODE_SINGLE) {
1034 		set_memory_uc((unsigned long)msc->base, msc->nr_pages);
1035 		return;
1036 	}
1037 
1038 	list_for_each_entry(win, &msc->win_list, entry) {
1039 		for_each_sg(win->sgt->sgl, sg_ptr, win->nr_segs, i) {
1040 			/* Set the page as uncached */
1041 			set_memory_uc((unsigned long)sg_virt(sg_ptr),
1042 					PFN_DOWN(sg_ptr->length));
1043 		}
1044 	}
1045 }
1046 
msc_buffer_set_wb(struct msc * msc)1047 static void msc_buffer_set_wb(struct msc *msc)
1048 {
1049 	struct scatterlist *sg_ptr;
1050 	struct msc_window *win;
1051 	int i;
1052 
1053 	if (msc->mode == MSC_MODE_SINGLE) {
1054 		set_memory_wb((unsigned long)msc->base, msc->nr_pages);
1055 		return;
1056 	}
1057 
1058 	list_for_each_entry(win, &msc->win_list, entry) {
1059 		for_each_sg(win->sgt->sgl, sg_ptr, win->nr_segs, i) {
1060 			/* Reset the page to write-back */
1061 			set_memory_wb((unsigned long)sg_virt(sg_ptr),
1062 					PFN_DOWN(sg_ptr->length));
1063 		}
1064 	}
1065 }
1066 #else /* !X86 */
1067 static inline void
msc_buffer_set_uc(struct msc * msc)1068 msc_buffer_set_uc(struct msc *msc) {}
msc_buffer_set_wb(struct msc * msc)1069 static inline void msc_buffer_set_wb(struct msc *msc) {}
1070 #endif /* CONFIG_X86 */
1071 
msc_sg_page(struct scatterlist * sg)1072 static struct page *msc_sg_page(struct scatterlist *sg)
1073 {
1074 	void *addr = sg_virt(sg);
1075 
1076 	if (is_vmalloc_addr(addr))
1077 		return vmalloc_to_page(addr);
1078 
1079 	return sg_page(sg);
1080 }
1081 
1082 /**
1083  * msc_buffer_win_alloc() - alloc a window for a multiblock mode
1084  * @msc:	MSC device
1085  * @nr_blocks:	number of pages in this window
1086  *
1087  * This modifies msc::win_list and msc::base, which requires msc::buf_mutex
1088  * to serialize, so the caller is expected to hold it.
1089  *
1090  * Return:	0 on success, -errno otherwise.
1091  */
msc_buffer_win_alloc(struct msc * msc,unsigned int nr_blocks)1092 static int msc_buffer_win_alloc(struct msc *msc, unsigned int nr_blocks)
1093 {
1094 	struct msc_window *win;
1095 	int ret = -ENOMEM;
1096 
1097 	if (!nr_blocks)
1098 		return 0;
1099 
1100 	win = kzalloc(sizeof(*win), GFP_KERNEL);
1101 	if (!win)
1102 		return -ENOMEM;
1103 
1104 	win->msc = msc;
1105 	win->sgt = &win->_sgt;
1106 	win->lockout = WIN_READY;
1107 	spin_lock_init(&win->lo_lock);
1108 
1109 	if (!list_empty(&msc->win_list)) {
1110 		struct msc_window *prev = list_last_entry(&msc->win_list,
1111 							  struct msc_window,
1112 							  entry);
1113 
1114 		win->pgoff = prev->pgoff + prev->nr_blocks;
1115 	}
1116 
1117 	if (msc->mbuf && msc->mbuf->alloc_window)
1118 		ret = msc->mbuf->alloc_window(msc->mbuf_priv, &win->sgt,
1119 					      nr_blocks << PAGE_SHIFT);
1120 	else
1121 		ret = __msc_buffer_win_alloc(win, nr_blocks);
1122 
1123 	if (ret <= 0)
1124 		goto err_nomem;
1125 
1126 	win->nr_segs = ret;
1127 	win->nr_blocks = nr_blocks;
1128 
1129 	if (list_empty(&msc->win_list)) {
1130 		msc->base = msc_win_base(win);
1131 		msc->base_addr = msc_win_base_dma(win);
1132 		msc->cur_win = win;
1133 	}
1134 
1135 	list_add_tail(&win->entry, &msc->win_list);
1136 	msc->nr_pages += nr_blocks;
1137 
1138 	return 0;
1139 
1140 err_nomem:
1141 	kfree(win);
1142 
1143 	return ret;
1144 }
1145 
__msc_buffer_win_free(struct msc * msc,struct msc_window * win)1146 static void __msc_buffer_win_free(struct msc *msc, struct msc_window *win)
1147 {
1148 	struct scatterlist *sg;
1149 	int i;
1150 
1151 	for_each_sg(win->sgt->sgl, sg, win->nr_segs, i) {
1152 		struct page *page = msc_sg_page(sg);
1153 
1154 		page->mapping = NULL;
1155 		dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE,
1156 				  sg_virt(sg), sg_dma_address(sg));
1157 	}
1158 	sg_free_table(win->sgt);
1159 }
1160 
1161 /**
1162  * msc_buffer_win_free() - free a window from MSC's window list
1163  * @msc:	MSC device
1164  * @win:	window to free
1165  *
1166  * This modifies msc::win_list and msc::base, which requires msc::buf_mutex
1167  * to serialize, so the caller is expected to hold it.
1168  */
msc_buffer_win_free(struct msc * msc,struct msc_window * win)1169 static void msc_buffer_win_free(struct msc *msc, struct msc_window *win)
1170 {
1171 	msc->nr_pages -= win->nr_blocks;
1172 
1173 	list_del(&win->entry);
1174 	if (list_empty(&msc->win_list)) {
1175 		msc->base = NULL;
1176 		msc->base_addr = 0;
1177 	}
1178 
1179 	if (msc->mbuf && msc->mbuf->free_window)
1180 		msc->mbuf->free_window(msc->mbuf_priv, win->sgt);
1181 	else
1182 		__msc_buffer_win_free(msc, win);
1183 
1184 	kfree(win);
1185 }
1186 
1187 /**
1188  * msc_buffer_relink() - set up block descriptors for multiblock mode
1189  * @msc:	MSC device
1190  *
1191  * This traverses msc::win_list, which requires msc::buf_mutex to serialize,
1192  * so the caller is expected to hold it.
1193  */
msc_buffer_relink(struct msc * msc)1194 static void msc_buffer_relink(struct msc *msc)
1195 {
1196 	struct msc_window *win, *next_win;
1197 
1198 	/* call with msc::mutex locked */
1199 	list_for_each_entry(win, &msc->win_list, entry) {
1200 		struct scatterlist *sg;
1201 		unsigned int blk;
1202 		u32 sw_tag = 0;
1203 
1204 		/*
1205 		 * Last window's next_win should point to the first window
1206 		 * and MSC_SW_TAG_LASTWIN should be set.
1207 		 */
1208 		if (msc_is_last_win(win)) {
1209 			sw_tag |= MSC_SW_TAG_LASTWIN;
1210 			next_win = list_first_entry(&msc->win_list,
1211 						    struct msc_window, entry);
1212 		} else {
1213 			next_win = list_next_entry(win, entry);
1214 		}
1215 
1216 		for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
1217 			struct msc_block_desc *bdesc = sg_virt(sg);
1218 
1219 			memset(bdesc, 0, sizeof(*bdesc));
1220 
1221 			bdesc->next_win = msc_win_base_pfn(next_win);
1222 
1223 			/*
1224 			 * Similarly to last window, last block should point
1225 			 * to the first one.
1226 			 */
1227 			if (blk == win->nr_segs - 1) {
1228 				sw_tag |= MSC_SW_TAG_LASTBLK;
1229 				bdesc->next_blk = msc_win_base_pfn(win);
1230 			} else {
1231 				dma_addr_t addr = sg_dma_address(sg_next(sg));
1232 
1233 				bdesc->next_blk = PFN_DOWN(addr);
1234 			}
1235 
1236 			bdesc->sw_tag = sw_tag;
1237 			bdesc->block_sz = sg->length / 64;
1238 		}
1239 	}
1240 
1241 	/*
1242 	 * Make the above writes globally visible before tracing is
1243 	 * enabled to make sure hardware sees them coherently.
1244 	 */
1245 	wmb();
1246 }
1247 
msc_buffer_multi_free(struct msc * msc)1248 static void msc_buffer_multi_free(struct msc *msc)
1249 {
1250 	struct msc_window *win, *iter;
1251 
1252 	list_for_each_entry_safe(win, iter, &msc->win_list, entry)
1253 		msc_buffer_win_free(msc, win);
1254 }
1255 
msc_buffer_multi_alloc(struct msc * msc,unsigned long * nr_pages,unsigned int nr_wins)1256 static int msc_buffer_multi_alloc(struct msc *msc, unsigned long *nr_pages,
1257 				  unsigned int nr_wins)
1258 {
1259 	int ret, i;
1260 
1261 	for (i = 0; i < nr_wins; i++) {
1262 		ret = msc_buffer_win_alloc(msc, nr_pages[i]);
1263 		if (ret) {
1264 			msc_buffer_multi_free(msc);
1265 			return ret;
1266 		}
1267 	}
1268 
1269 	msc_buffer_relink(msc);
1270 
1271 	return 0;
1272 }
1273 
1274 /**
1275  * msc_buffer_free() - free buffers for MSC
1276  * @msc:	MSC device
1277  *
1278  * Free MSC's storage buffers.
1279  *
1280  * This modifies msc::win_list and msc::base, which requires msc::buf_mutex to
1281  * serialize, so the caller is expected to hold it.
1282  */
msc_buffer_free(struct msc * msc)1283 static void msc_buffer_free(struct msc *msc)
1284 {
1285 	msc_buffer_set_wb(msc);
1286 
1287 	if (msc->mode == MSC_MODE_SINGLE)
1288 		msc_buffer_contig_free(msc);
1289 	else if (msc->mode == MSC_MODE_MULTI)
1290 		msc_buffer_multi_free(msc);
1291 }
1292 
1293 /**
1294  * msc_buffer_alloc() - allocate a buffer for MSC
1295  * @msc:	MSC device
1296  * @nr_pages:	number of pages for each window
1297  * @nr_wins:	number of windows
1298  *
1299  * Allocate a storage buffer for MSC, depending on the msc::mode, it will be
1300  * either done via msc_buffer_contig_alloc() for SINGLE operation mode or
1301  * msc_buffer_win_alloc() for multiblock operation. The latter allocates one
1302  * window per invocation, so in multiblock mode this can be called multiple
1303  * times for the same MSC to allocate multiple windows.
1304  *
1305  * This modifies msc::win_list and msc::base, which requires msc::buf_mutex
1306  * to serialize, so the caller is expected to hold it.
1307  *
1308  * Return:	0 on success, -errno otherwise.
1309  */
msc_buffer_alloc(struct msc * msc,unsigned long * nr_pages,unsigned int nr_wins)1310 static int msc_buffer_alloc(struct msc *msc, unsigned long *nr_pages,
1311 			    unsigned int nr_wins)
1312 {
1313 	int ret;
1314 
1315 	/* -1: buffer not allocated */
1316 	if (atomic_read(&msc->user_count) != -1)
1317 		return -EBUSY;
1318 
1319 	if (msc->mode == MSC_MODE_SINGLE) {
1320 		if (nr_wins != 1)
1321 			return -EINVAL;
1322 
1323 		ret = msc_buffer_contig_alloc(msc, nr_pages[0] << PAGE_SHIFT);
1324 	} else if (msc->mode == MSC_MODE_MULTI) {
1325 		ret = msc_buffer_multi_alloc(msc, nr_pages, nr_wins);
1326 	} else {
1327 		ret = -EINVAL;
1328 	}
1329 
1330 	if (!ret) {
1331 		msc_buffer_set_uc(msc);
1332 
1333 		/* allocation should be visible before the counter goes to 0 */
1334 		smp_mb__before_atomic();
1335 
1336 		if (WARN_ON_ONCE(atomic_cmpxchg(&msc->user_count, -1, 0) != -1))
1337 			return -EINVAL;
1338 	}
1339 
1340 	return ret;
1341 }
1342 
1343 /**
1344  * msc_buffer_unlocked_free_unless_used() - free a buffer unless it's in use
1345  * @msc:	MSC device
1346  *
1347  * This will free MSC buffer unless it is in use or there is no allocated
1348  * buffer.
1349  * Caller needs to hold msc::buf_mutex.
1350  *
1351  * Return:	0 on successful deallocation or if there was no buffer to
1352  *		deallocate, -EBUSY if there are active users.
1353  */
msc_buffer_unlocked_free_unless_used(struct msc * msc)1354 static int msc_buffer_unlocked_free_unless_used(struct msc *msc)
1355 {
1356 	int count, ret = 0;
1357 
1358 	count = atomic_cmpxchg(&msc->user_count, 0, -1);
1359 
1360 	/* > 0: buffer is allocated and has users */
1361 	if (count > 0)
1362 		ret = -EBUSY;
1363 	/* 0: buffer is allocated, no users */
1364 	else if (!count)
1365 		msc_buffer_free(msc);
1366 	/* < 0: no buffer, nothing to do */
1367 
1368 	return ret;
1369 }
1370 
1371 /**
1372  * msc_buffer_free_unless_used() - free a buffer unless it's in use
1373  * @msc:	MSC device
1374  *
1375  * This is a locked version of msc_buffer_unlocked_free_unless_used().
1376  *
1377  * Return:	0 on successful deallocation or if there was no buffer to
1378  *		deallocate, -EBUSY if there are active users.
1379  */
msc_buffer_free_unless_used(struct msc * msc)1380 static int msc_buffer_free_unless_used(struct msc *msc)
1381 {
1382 	int ret;
1383 
1384 	mutex_lock(&msc->buf_mutex);
1385 	ret = msc_buffer_unlocked_free_unless_used(msc);
1386 	mutex_unlock(&msc->buf_mutex);
1387 
1388 	return ret;
1389 }
1390 
1391 /**
1392  * msc_buffer_get_page() - get MSC buffer page at a given offset
1393  * @msc:	MSC device
1394  * @pgoff:	page offset into the storage buffer
1395  *
1396  * This traverses msc::win_list, so holding msc::buf_mutex is expected from
1397  * the caller.
1398  *
1399  * Return:	page if @pgoff corresponds to a valid buffer page or NULL.
1400  */
msc_buffer_get_page(struct msc * msc,unsigned long pgoff)1401 static struct page *msc_buffer_get_page(struct msc *msc, unsigned long pgoff)
1402 {
1403 	struct msc_window *win;
1404 	struct scatterlist *sg;
1405 	unsigned int blk;
1406 
1407 	if (msc->mode == MSC_MODE_SINGLE)
1408 		return msc_buffer_contig_get_page(msc, pgoff);
1409 
1410 	list_for_each_entry(win, &msc->win_list, entry)
1411 		if (pgoff >= win->pgoff && pgoff < win->pgoff + win->nr_blocks)
1412 			goto found;
1413 
1414 	return NULL;
1415 
1416 found:
1417 	pgoff -= win->pgoff;
1418 
1419 	for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) {
1420 		struct page *page = msc_sg_page(sg);
1421 		size_t pgsz = PFN_DOWN(sg->length);
1422 
1423 		if (pgoff < pgsz)
1424 			return page + pgoff;
1425 
1426 		pgoff -= pgsz;
1427 	}
1428 
1429 	return NULL;
1430 }
1431 
1432 /**
1433  * struct msc_win_to_user_struct - data for copy_to_user() callback
1434  * @buf:	userspace buffer to copy data to
1435  * @offset:	running offset
1436  */
1437 struct msc_win_to_user_struct {
1438 	char __user	*buf;
1439 	unsigned long	offset;
1440 };
1441 
1442 /**
1443  * msc_win_to_user() - iterator for msc_buffer_iterate() to copy data to user
1444  * @data:	callback's private data
1445  * @src:	source buffer
1446  * @len:	amount of data to copy from the source buffer
1447  *
1448  * Return:	>= %0 for success or -errno for error.
1449  */
msc_win_to_user(void * data,void * src,size_t len)1450 static unsigned long msc_win_to_user(void *data, void *src, size_t len)
1451 {
1452 	struct msc_win_to_user_struct *u = data;
1453 	unsigned long ret;
1454 
1455 	ret = copy_to_user(u->buf + u->offset, src, len);
1456 	u->offset += len - ret;
1457 
1458 	return ret;
1459 }
1460 
1461 
1462 /*
1463  * file operations' callbacks
1464  */
1465 
intel_th_msc_open(struct inode * inode,struct file * file)1466 static int intel_th_msc_open(struct inode *inode, struct file *file)
1467 {
1468 	struct intel_th_device *thdev = file->private_data;
1469 	struct msc *msc = dev_get_drvdata(&thdev->dev);
1470 	struct msc_iter *iter;
1471 
1472 	if (!capable(CAP_SYS_RAWIO))
1473 		return -EPERM;
1474 
1475 	iter = msc_iter_install(msc);
1476 	if (IS_ERR(iter))
1477 		return PTR_ERR(iter);
1478 
1479 	file->private_data = iter;
1480 
1481 	return nonseekable_open(inode, file);
1482 }
1483 
intel_th_msc_release(struct inode * inode,struct file * file)1484 static int intel_th_msc_release(struct inode *inode, struct file *file)
1485 {
1486 	struct msc_iter *iter = file->private_data;
1487 	struct msc *msc = iter->msc;
1488 
1489 	msc_iter_remove(iter, msc);
1490 
1491 	return 0;
1492 }
1493 
1494 static ssize_t
msc_single_to_user(struct msc * msc,char __user * buf,loff_t off,size_t len)1495 msc_single_to_user(struct msc *msc, char __user *buf, loff_t off, size_t len)
1496 {
1497 	unsigned long size = msc->nr_pages << PAGE_SHIFT, rem = len;
1498 	unsigned long start = off, tocopy = 0;
1499 
1500 	if (msc->single_wrap) {
1501 		start += msc->single_sz;
1502 		if (start < size) {
1503 			tocopy = min(rem, size - start);
1504 			if (copy_to_user(buf, msc->base + start, tocopy))
1505 				return -EFAULT;
1506 
1507 			buf += tocopy;
1508 			rem -= tocopy;
1509 			start += tocopy;
1510 		}
1511 
1512 		start &= size - 1;
1513 		if (rem) {
1514 			tocopy = min(rem, msc->single_sz - start);
1515 			if (copy_to_user(buf, msc->base + start, tocopy))
1516 				return -EFAULT;
1517 
1518 			rem -= tocopy;
1519 		}
1520 
1521 		return len - rem;
1522 	}
1523 
1524 	if (copy_to_user(buf, msc->base + start, rem))
1525 		return -EFAULT;
1526 
1527 	return len;
1528 }
1529 
intel_th_msc_read(struct file * file,char __user * buf,size_t len,loff_t * ppos)1530 static ssize_t intel_th_msc_read(struct file *file, char __user *buf,
1531 				 size_t len, loff_t *ppos)
1532 {
1533 	struct msc_iter *iter = file->private_data;
1534 	struct msc *msc = iter->msc;
1535 	size_t size;
1536 	loff_t off = *ppos;
1537 	ssize_t ret = 0;
1538 
1539 	if (!atomic_inc_unless_negative(&msc->user_count))
1540 		return 0;
1541 
1542 	if (msc->mode == MSC_MODE_SINGLE && !msc->single_wrap)
1543 		size = msc->single_sz;
1544 	else
1545 		size = msc->nr_pages << PAGE_SHIFT;
1546 
1547 	if (!size)
1548 		goto put_count;
1549 
1550 	if (off >= size)
1551 		goto put_count;
1552 
1553 	if (off + len >= size)
1554 		len = size - off;
1555 
1556 	if (msc->mode == MSC_MODE_SINGLE) {
1557 		ret = msc_single_to_user(msc, buf, off, len);
1558 		if (ret >= 0)
1559 			*ppos += ret;
1560 	} else if (msc->mode == MSC_MODE_MULTI) {
1561 		struct msc_win_to_user_struct u = {
1562 			.buf	= buf,
1563 			.offset	= 0,
1564 		};
1565 
1566 		ret = msc_buffer_iterate(iter, len, &u, msc_win_to_user);
1567 		if (ret >= 0)
1568 			*ppos = iter->offset;
1569 	} else {
1570 		ret = -EINVAL;
1571 	}
1572 
1573 put_count:
1574 	atomic_dec(&msc->user_count);
1575 
1576 	return ret;
1577 }
1578 
1579 /*
1580  * vm operations callbacks (vm_ops)
1581  */
1582 
msc_mmap_open(struct vm_area_struct * vma)1583 static void msc_mmap_open(struct vm_area_struct *vma)
1584 {
1585 	struct msc_iter *iter = vma->vm_file->private_data;
1586 	struct msc *msc = iter->msc;
1587 
1588 	atomic_inc(&msc->mmap_count);
1589 }
1590 
msc_mmap_close(struct vm_area_struct * vma)1591 static void msc_mmap_close(struct vm_area_struct *vma)
1592 {
1593 	struct msc_iter *iter = vma->vm_file->private_data;
1594 	struct msc *msc = iter->msc;
1595 	unsigned long pg;
1596 
1597 	if (!atomic_dec_and_mutex_lock(&msc->mmap_count, &msc->buf_mutex))
1598 		return;
1599 
1600 	/* drop page _refcounts */
1601 	for (pg = 0; pg < msc->nr_pages; pg++) {
1602 		struct page *page = msc_buffer_get_page(msc, pg);
1603 
1604 		if (WARN_ON_ONCE(!page))
1605 			continue;
1606 
1607 		if (page->mapping)
1608 			page->mapping = NULL;
1609 	}
1610 
1611 	/* last mapping -- drop user_count */
1612 	atomic_dec(&msc->user_count);
1613 	mutex_unlock(&msc->buf_mutex);
1614 }
1615 
msc_mmap_fault(struct vm_fault * vmf)1616 static vm_fault_t msc_mmap_fault(struct vm_fault *vmf)
1617 {
1618 	struct msc_iter *iter = vmf->vma->vm_file->private_data;
1619 	struct msc *msc = iter->msc;
1620 
1621 	vmf->page = msc_buffer_get_page(msc, vmf->pgoff);
1622 	if (!vmf->page)
1623 		return VM_FAULT_SIGBUS;
1624 
1625 	get_page(vmf->page);
1626 	vmf->page->mapping = vmf->vma->vm_file->f_mapping;
1627 	vmf->page->index = vmf->pgoff;
1628 
1629 	return 0;
1630 }
1631 
1632 static const struct vm_operations_struct msc_mmap_ops = {
1633 	.open	= msc_mmap_open,
1634 	.close	= msc_mmap_close,
1635 	.fault	= msc_mmap_fault,
1636 };
1637 
intel_th_msc_mmap(struct file * file,struct vm_area_struct * vma)1638 static int intel_th_msc_mmap(struct file *file, struct vm_area_struct *vma)
1639 {
1640 	unsigned long size = vma->vm_end - vma->vm_start;
1641 	struct msc_iter *iter = vma->vm_file->private_data;
1642 	struct msc *msc = iter->msc;
1643 	int ret = -EINVAL;
1644 
1645 	if (!size || offset_in_page(size))
1646 		return -EINVAL;
1647 
1648 	if (vma->vm_pgoff)
1649 		return -EINVAL;
1650 
1651 	/* grab user_count once per mmap; drop in msc_mmap_close() */
1652 	if (!atomic_inc_unless_negative(&msc->user_count))
1653 		return -EINVAL;
1654 
1655 	if (msc->mode != MSC_MODE_SINGLE &&
1656 	    msc->mode != MSC_MODE_MULTI)
1657 		goto out;
1658 
1659 	if (size >> PAGE_SHIFT != msc->nr_pages)
1660 		goto out;
1661 
1662 	atomic_set(&msc->mmap_count, 1);
1663 	ret = 0;
1664 
1665 out:
1666 	if (ret)
1667 		atomic_dec(&msc->user_count);
1668 
1669 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1670 	vm_flags_set(vma, VM_DONTEXPAND | VM_DONTCOPY);
1671 	vma->vm_ops = &msc_mmap_ops;
1672 	return ret;
1673 }
1674 
1675 static const struct file_operations intel_th_msc_fops = {
1676 	.open		= intel_th_msc_open,
1677 	.release	= intel_th_msc_release,
1678 	.read		= intel_th_msc_read,
1679 	.mmap		= intel_th_msc_mmap,
1680 	.owner		= THIS_MODULE,
1681 };
1682 
intel_th_msc_wait_empty(struct intel_th_device * thdev)1683 static void intel_th_msc_wait_empty(struct intel_th_device *thdev)
1684 {
1685 	struct msc *msc = dev_get_drvdata(&thdev->dev);
1686 	unsigned long count;
1687 	u32 reg;
1688 
1689 	for (reg = 0, count = MSC_PLE_WAITLOOP_DEPTH;
1690 	     count && !(reg & MSCSTS_PLE); count--) {
1691 		reg = __raw_readl(msc->reg_base + REG_MSU_MSC0STS);
1692 		cpu_relax();
1693 	}
1694 
1695 	if (!count)
1696 		dev_dbg(msc_dev(msc), "timeout waiting for MSC0 PLE\n");
1697 }
1698 
intel_th_msc_init(struct msc * msc)1699 static int intel_th_msc_init(struct msc *msc)
1700 {
1701 	atomic_set(&msc->user_count, -1);
1702 
1703 	msc->mode = msc->multi_is_broken ? MSC_MODE_SINGLE : MSC_MODE_MULTI;
1704 	mutex_init(&msc->buf_mutex);
1705 	INIT_LIST_HEAD(&msc->win_list);
1706 	INIT_LIST_HEAD(&msc->iter_list);
1707 
1708 	msc->burst_len =
1709 		(ioread32(msc->reg_base + REG_MSU_MSC0CTL) & MSC_LEN) >>
1710 		__ffs(MSC_LEN);
1711 
1712 	return 0;
1713 }
1714 
msc_win_switch(struct msc * msc)1715 static int msc_win_switch(struct msc *msc)
1716 {
1717 	struct msc_window *first;
1718 
1719 	if (list_empty(&msc->win_list))
1720 		return -EINVAL;
1721 
1722 	first = list_first_entry(&msc->win_list, struct msc_window, entry);
1723 
1724 	if (msc_is_last_win(msc->cur_win))
1725 		msc->cur_win = first;
1726 	else
1727 		msc->cur_win = list_next_entry(msc->cur_win, entry);
1728 
1729 	msc->base = msc_win_base(msc->cur_win);
1730 	msc->base_addr = msc_win_base_dma(msc->cur_win);
1731 
1732 	intel_th_trace_switch(msc->thdev);
1733 
1734 	return 0;
1735 }
1736 
1737 /**
1738  * intel_th_msc_window_unlock - put the window back in rotation
1739  * @dev:	MSC device to which this relates
1740  * @sgt:	buffer's sg_table for the window, does nothing if NULL
1741  */
intel_th_msc_window_unlock(struct device * dev,struct sg_table * sgt)1742 void intel_th_msc_window_unlock(struct device *dev, struct sg_table *sgt)
1743 {
1744 	struct msc *msc = dev_get_drvdata(dev);
1745 	struct msc_window *win;
1746 
1747 	if (!sgt)
1748 		return;
1749 
1750 	win = msc_find_window(msc, sgt, false);
1751 	if (!win)
1752 		return;
1753 
1754 	msc_win_set_lockout(win, WIN_LOCKED, WIN_READY);
1755 	if (msc->switch_on_unlock == win) {
1756 		msc->switch_on_unlock = NULL;
1757 		msc_win_switch(msc);
1758 	}
1759 }
1760 EXPORT_SYMBOL_GPL(intel_th_msc_window_unlock);
1761 
msc_work(struct work_struct * work)1762 static void msc_work(struct work_struct *work)
1763 {
1764 	struct msc *msc = container_of(work, struct msc, work);
1765 
1766 	intel_th_msc_deactivate(msc->thdev);
1767 }
1768 
intel_th_msc_interrupt(struct intel_th_device * thdev)1769 static irqreturn_t intel_th_msc_interrupt(struct intel_th_device *thdev)
1770 {
1771 	struct msc *msc = dev_get_drvdata(&thdev->dev);
1772 	u32 msusts = ioread32(msc->msu_base + REG_MSU_MSUSTS);
1773 	u32 mask = msc->index ? MSUSTS_MSC1BLAST : MSUSTS_MSC0BLAST;
1774 	struct msc_window *win, *next_win;
1775 
1776 	if (!msc->do_irq || !msc->mbuf)
1777 		return IRQ_NONE;
1778 
1779 	msusts &= mask;
1780 
1781 	if (!msusts)
1782 		return msc->enabled ? IRQ_HANDLED : IRQ_NONE;
1783 
1784 	iowrite32(msusts, msc->msu_base + REG_MSU_MSUSTS);
1785 
1786 	if (!msc->enabled)
1787 		return IRQ_NONE;
1788 
1789 	/* grab the window before we do the switch */
1790 	win = msc->cur_win;
1791 	if (!win)
1792 		return IRQ_HANDLED;
1793 	next_win = msc_next_window(win);
1794 	if (!next_win)
1795 		return IRQ_HANDLED;
1796 
1797 	/* next window: if READY, proceed, if LOCKED, stop the trace */
1798 	if (msc_win_set_lockout(next_win, WIN_READY, WIN_INUSE)) {
1799 		if (msc->stop_on_full)
1800 			schedule_work(&msc->work);
1801 		else
1802 			msc->switch_on_unlock = next_win;
1803 
1804 		return IRQ_HANDLED;
1805 	}
1806 
1807 	/* current window: INUSE -> LOCKED */
1808 	msc_win_set_lockout(win, WIN_INUSE, WIN_LOCKED);
1809 
1810 	msc_win_switch(msc);
1811 
1812 	if (msc->mbuf && msc->mbuf->ready)
1813 		msc->mbuf->ready(msc->mbuf_priv, win->sgt,
1814 				 msc_win_total_sz(win));
1815 
1816 	return IRQ_HANDLED;
1817 }
1818 
1819 static const char * const msc_mode[] = {
1820 	[MSC_MODE_SINGLE]	= "single",
1821 	[MSC_MODE_MULTI]	= "multi",
1822 	[MSC_MODE_EXI]		= "ExI",
1823 	[MSC_MODE_DEBUG]	= "debug",
1824 };
1825 
1826 static ssize_t
wrap_show(struct device * dev,struct device_attribute * attr,char * buf)1827 wrap_show(struct device *dev, struct device_attribute *attr, char *buf)
1828 {
1829 	struct msc *msc = dev_get_drvdata(dev);
1830 
1831 	return scnprintf(buf, PAGE_SIZE, "%d\n", msc->wrap);
1832 }
1833 
1834 static ssize_t
wrap_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)1835 wrap_store(struct device *dev, struct device_attribute *attr, const char *buf,
1836 	   size_t size)
1837 {
1838 	struct msc *msc = dev_get_drvdata(dev);
1839 	unsigned long val;
1840 	int ret;
1841 
1842 	ret = kstrtoul(buf, 10, &val);
1843 	if (ret)
1844 		return ret;
1845 
1846 	msc->wrap = !!val;
1847 
1848 	return size;
1849 }
1850 
1851 static DEVICE_ATTR_RW(wrap);
1852 
msc_buffer_unassign(struct msc * msc)1853 static void msc_buffer_unassign(struct msc *msc)
1854 {
1855 	lockdep_assert_held(&msc->buf_mutex);
1856 
1857 	if (!msc->mbuf)
1858 		return;
1859 
1860 	msc->mbuf->unassign(msc->mbuf_priv);
1861 	msu_buffer_put(msc->mbuf);
1862 	msc->mbuf_priv = NULL;
1863 	msc->mbuf = NULL;
1864 }
1865 
1866 static ssize_t
mode_show(struct device * dev,struct device_attribute * attr,char * buf)1867 mode_show(struct device *dev, struct device_attribute *attr, char *buf)
1868 {
1869 	struct msc *msc = dev_get_drvdata(dev);
1870 	const char *mode = msc_mode[msc->mode];
1871 	ssize_t ret;
1872 
1873 	mutex_lock(&msc->buf_mutex);
1874 	if (msc->mbuf)
1875 		mode = msc->mbuf->name;
1876 	ret = scnprintf(buf, PAGE_SIZE, "%s\n", mode);
1877 	mutex_unlock(&msc->buf_mutex);
1878 
1879 	return ret;
1880 }
1881 
1882 static ssize_t
mode_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)1883 mode_store(struct device *dev, struct device_attribute *attr, const char *buf,
1884 	   size_t size)
1885 {
1886 	const struct msu_buffer *mbuf = NULL;
1887 	struct msc *msc = dev_get_drvdata(dev);
1888 	size_t len = size;
1889 	char *cp, *mode;
1890 	int i, ret;
1891 
1892 	if (!capable(CAP_SYS_RAWIO))
1893 		return -EPERM;
1894 
1895 	cp = memchr(buf, '\n', len);
1896 	if (cp)
1897 		len = cp - buf;
1898 
1899 	mode = kstrndup(buf, len, GFP_KERNEL);
1900 	if (!mode)
1901 		return -ENOMEM;
1902 
1903 	i = match_string(msc_mode, ARRAY_SIZE(msc_mode), mode);
1904 	if (i >= 0) {
1905 		kfree(mode);
1906 		goto found;
1907 	}
1908 
1909 	/* Buffer sinks only work with a usable IRQ */
1910 	if (!msc->do_irq) {
1911 		kfree(mode);
1912 		return -EINVAL;
1913 	}
1914 
1915 	mbuf = msu_buffer_get(mode);
1916 	kfree(mode);
1917 	if (mbuf)
1918 		goto found;
1919 
1920 	return -EINVAL;
1921 
1922 found:
1923 	if (i == MSC_MODE_MULTI && msc->multi_is_broken)
1924 		return -EOPNOTSUPP;
1925 
1926 	mutex_lock(&msc->buf_mutex);
1927 	ret = 0;
1928 
1929 	/* Same buffer: do nothing */
1930 	if (mbuf && mbuf == msc->mbuf) {
1931 		/* put the extra reference we just got */
1932 		msu_buffer_put(mbuf);
1933 		goto unlock;
1934 	}
1935 
1936 	ret = msc_buffer_unlocked_free_unless_used(msc);
1937 	if (ret)
1938 		goto unlock;
1939 
1940 	if (mbuf) {
1941 		void *mbuf_priv = mbuf->assign(dev, &i);
1942 
1943 		if (!mbuf_priv) {
1944 			ret = -ENOMEM;
1945 			goto unlock;
1946 		}
1947 
1948 		msc_buffer_unassign(msc);
1949 		msc->mbuf_priv = mbuf_priv;
1950 		msc->mbuf = mbuf;
1951 	} else {
1952 		msc_buffer_unassign(msc);
1953 	}
1954 
1955 	msc->mode = i;
1956 
1957 unlock:
1958 	if (ret && mbuf)
1959 		msu_buffer_put(mbuf);
1960 	mutex_unlock(&msc->buf_mutex);
1961 
1962 	return ret ? ret : size;
1963 }
1964 
1965 static DEVICE_ATTR_RW(mode);
1966 
1967 static ssize_t
nr_pages_show(struct device * dev,struct device_attribute * attr,char * buf)1968 nr_pages_show(struct device *dev, struct device_attribute *attr, char *buf)
1969 {
1970 	struct msc *msc = dev_get_drvdata(dev);
1971 	struct msc_window *win;
1972 	size_t count = 0;
1973 
1974 	mutex_lock(&msc->buf_mutex);
1975 
1976 	if (msc->mode == MSC_MODE_SINGLE)
1977 		count = scnprintf(buf, PAGE_SIZE, "%ld\n", msc->nr_pages);
1978 	else if (msc->mode == MSC_MODE_MULTI) {
1979 		list_for_each_entry(win, &msc->win_list, entry) {
1980 			count += scnprintf(buf + count, PAGE_SIZE - count,
1981 					   "%d%c", win->nr_blocks,
1982 					   msc_is_last_win(win) ? '\n' : ',');
1983 		}
1984 	} else {
1985 		count = scnprintf(buf, PAGE_SIZE, "unsupported\n");
1986 	}
1987 
1988 	mutex_unlock(&msc->buf_mutex);
1989 
1990 	return count;
1991 }
1992 
1993 static ssize_t
nr_pages_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)1994 nr_pages_store(struct device *dev, struct device_attribute *attr,
1995 	       const char *buf, size_t size)
1996 {
1997 	struct msc *msc = dev_get_drvdata(dev);
1998 	unsigned long val, *win = NULL, *rewin;
1999 	size_t len = size;
2000 	const char *p = buf;
2001 	char *end, *s;
2002 	int ret, nr_wins = 0;
2003 
2004 	if (!capable(CAP_SYS_RAWIO))
2005 		return -EPERM;
2006 
2007 	ret = msc_buffer_free_unless_used(msc);
2008 	if (ret)
2009 		return ret;
2010 
2011 	/* scan the comma-separated list of allocation sizes */
2012 	end = memchr(buf, '\n', len);
2013 	if (end)
2014 		len = end - buf;
2015 
2016 	do {
2017 		end = memchr(p, ',', len);
2018 		s = kstrndup(p, end ? end - p : len, GFP_KERNEL);
2019 		if (!s) {
2020 			ret = -ENOMEM;
2021 			goto free_win;
2022 		}
2023 
2024 		ret = kstrtoul(s, 10, &val);
2025 		kfree(s);
2026 
2027 		if (ret || !val)
2028 			goto free_win;
2029 
2030 		if (nr_wins && msc->mode == MSC_MODE_SINGLE) {
2031 			ret = -EINVAL;
2032 			goto free_win;
2033 		}
2034 
2035 		nr_wins++;
2036 		rewin = krealloc_array(win, nr_wins, sizeof(*win), GFP_KERNEL);
2037 		if (!rewin) {
2038 			kfree(win);
2039 			return -ENOMEM;
2040 		}
2041 
2042 		win = rewin;
2043 		win[nr_wins - 1] = val;
2044 
2045 		if (!end)
2046 			break;
2047 
2048 		/* consume the number and the following comma, hence +1 */
2049 		len -= end - p + 1;
2050 		p = end + 1;
2051 	} while (len);
2052 
2053 	mutex_lock(&msc->buf_mutex);
2054 	ret = msc_buffer_alloc(msc, win, nr_wins);
2055 	mutex_unlock(&msc->buf_mutex);
2056 
2057 free_win:
2058 	kfree(win);
2059 
2060 	return ret ? ret : size;
2061 }
2062 
2063 static DEVICE_ATTR_RW(nr_pages);
2064 
2065 static ssize_t
win_switch_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)2066 win_switch_store(struct device *dev, struct device_attribute *attr,
2067 		 const char *buf, size_t size)
2068 {
2069 	struct msc *msc = dev_get_drvdata(dev);
2070 	unsigned long val;
2071 	int ret;
2072 
2073 	ret = kstrtoul(buf, 10, &val);
2074 	if (ret)
2075 		return ret;
2076 
2077 	if (val != 1)
2078 		return -EINVAL;
2079 
2080 	ret = -EINVAL;
2081 	mutex_lock(&msc->buf_mutex);
2082 	/*
2083 	 * Window switch can only happen in the "multi" mode.
2084 	 * If a external buffer is engaged, they have the full
2085 	 * control over window switching.
2086 	 */
2087 	if (msc->mode == MSC_MODE_MULTI && !msc->mbuf)
2088 		ret = msc_win_switch(msc);
2089 	mutex_unlock(&msc->buf_mutex);
2090 
2091 	return ret ? ret : size;
2092 }
2093 
2094 static DEVICE_ATTR_WO(win_switch);
2095 
stop_on_full_show(struct device * dev,struct device_attribute * attr,char * buf)2096 static ssize_t stop_on_full_show(struct device *dev,
2097 				 struct device_attribute *attr, char *buf)
2098 {
2099 	struct msc *msc = dev_get_drvdata(dev);
2100 
2101 	return sprintf(buf, "%d\n", msc->stop_on_full);
2102 }
2103 
stop_on_full_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)2104 static ssize_t stop_on_full_store(struct device *dev,
2105 				  struct device_attribute *attr,
2106 				  const char *buf, size_t size)
2107 {
2108 	struct msc *msc = dev_get_drvdata(dev);
2109 	int ret;
2110 
2111 	ret = kstrtobool(buf, &msc->stop_on_full);
2112 	if (ret)
2113 		return ret;
2114 
2115 	return size;
2116 }
2117 
2118 static DEVICE_ATTR_RW(stop_on_full);
2119 
2120 static struct attribute *msc_output_attrs[] = {
2121 	&dev_attr_wrap.attr,
2122 	&dev_attr_mode.attr,
2123 	&dev_attr_nr_pages.attr,
2124 	&dev_attr_win_switch.attr,
2125 	&dev_attr_stop_on_full.attr,
2126 	NULL,
2127 };
2128 
2129 static const struct attribute_group msc_output_group = {
2130 	.attrs	= msc_output_attrs,
2131 };
2132 
intel_th_msc_probe(struct intel_th_device * thdev)2133 static int intel_th_msc_probe(struct intel_th_device *thdev)
2134 {
2135 	struct device *dev = &thdev->dev;
2136 	struct resource *res;
2137 	struct msc *msc;
2138 	void __iomem *base;
2139 	int err;
2140 
2141 	res = intel_th_device_get_resource(thdev, IORESOURCE_MEM, 0);
2142 	if (!res)
2143 		return -ENODEV;
2144 
2145 	base = devm_ioremap(dev, res->start, resource_size(res));
2146 	if (!base)
2147 		return -ENOMEM;
2148 
2149 	msc = devm_kzalloc(dev, sizeof(*msc), GFP_KERNEL);
2150 	if (!msc)
2151 		return -ENOMEM;
2152 
2153 	res = intel_th_device_get_resource(thdev, IORESOURCE_IRQ, 1);
2154 	if (!res)
2155 		msc->do_irq = 1;
2156 
2157 	if (INTEL_TH_CAP(to_intel_th(thdev), multi_is_broken))
2158 		msc->multi_is_broken = 1;
2159 
2160 	msc->index = thdev->id;
2161 
2162 	msc->thdev = thdev;
2163 	msc->reg_base = base + msc->index * 0x100;
2164 	msc->msu_base = base;
2165 
2166 	INIT_WORK(&msc->work, msc_work);
2167 	err = intel_th_msc_init(msc);
2168 	if (err)
2169 		return err;
2170 
2171 	dev_set_drvdata(dev, msc);
2172 
2173 	return 0;
2174 }
2175 
intel_th_msc_remove(struct intel_th_device * thdev)2176 static void intel_th_msc_remove(struct intel_th_device *thdev)
2177 {
2178 	struct msc *msc = dev_get_drvdata(&thdev->dev);
2179 	int ret;
2180 
2181 	intel_th_msc_deactivate(thdev);
2182 
2183 	/*
2184 	 * Buffers should not be used at this point except if the
2185 	 * output character device is still open and the parent
2186 	 * device gets detached from its bus, which is a FIXME.
2187 	 */
2188 	ret = msc_buffer_free_unless_used(msc);
2189 	WARN_ON_ONCE(ret);
2190 }
2191 
2192 static struct intel_th_driver intel_th_msc_driver = {
2193 	.probe	= intel_th_msc_probe,
2194 	.remove	= intel_th_msc_remove,
2195 	.irq		= intel_th_msc_interrupt,
2196 	.wait_empty	= intel_th_msc_wait_empty,
2197 	.activate	= intel_th_msc_activate,
2198 	.deactivate	= intel_th_msc_deactivate,
2199 	.fops	= &intel_th_msc_fops,
2200 	.attr_group	= &msc_output_group,
2201 	.driver	= {
2202 		.name	= "msc",
2203 		.owner	= THIS_MODULE,
2204 	},
2205 };
2206 
2207 module_driver(intel_th_msc_driver,
2208 	      intel_th_driver_register,
2209 	      intel_th_driver_unregister);
2210 
2211 MODULE_LICENSE("GPL v2");
2212 MODULE_DESCRIPTION("Intel(R) Trace Hub Memory Storage Unit driver");
2213 MODULE_AUTHOR("Alexander Shishkin <alexander.shishkin@linux.intel.com>");
2214