11a59d1b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
2ff3a3e9bSPekka Paalanen /*
3ff3a3e9bSPekka Paalanen *
4ff3a3e9bSPekka Paalanen * Copyright (C) IBM Corporation, 2005
5ff3a3e9bSPekka Paalanen * Jeff Muizelaar, 2006, 2007
6ff3a3e9bSPekka Paalanen * Pekka Paalanen, 2008 <pq@iki.fi>
7ff3a3e9bSPekka Paalanen *
8ff3a3e9bSPekka Paalanen * Derived from the read-mod example from relay-examples by Tom Zanussi.
9ff3a3e9bSPekka Paalanen */
103a0340beSJoe Perches
11eba11d6dSJoe Perches #define pr_fmt(fmt) "mmiotrace: " fmt
123a0340beSJoe Perches
134b599fedSPaul Gortmaker #include <linux/moduleparam.h>
14ff3a3e9bSPekka Paalanen #include <linux/debugfs.h>
155a0e3ad6STejun Heo #include <linux/slab.h>
16ff3a3e9bSPekka Paalanen #include <linux/uaccess.h>
17970e6fa0SPekka Paalanen #include <linux/io.h>
18ff3a3e9bSPekka Paalanen #include <linux/mmiotrace.h>
1965fddcfcSMike Rapoport #include <linux/pgtable.h>
2066441bd3SIngo Molnar #include <asm/e820/api.h> /* for ISA_START_ADDRESS */
2160063497SArun Sharma #include <linux/atomic.h>
22ff3a3e9bSPekka Paalanen #include <linux/percpu.h>
237423d111SPekka Paalanen #include <linux/cpu.h>
24ff3a3e9bSPekka Paalanen
25ff3a3e9bSPekka Paalanen #include "pf_in.h"
26ff3a3e9bSPekka Paalanen
27ff3a3e9bSPekka Paalanen struct trap_reason {
28ff3a3e9bSPekka Paalanen unsigned long addr;
29ff3a3e9bSPekka Paalanen unsigned long ip;
30ff3a3e9bSPekka Paalanen enum reason_type type;
31ff3a3e9bSPekka Paalanen int active_traces;
32ff3a3e9bSPekka Paalanen };
33ff3a3e9bSPekka Paalanen
34ff3a3e9bSPekka Paalanen struct remap_trace {
35ff3a3e9bSPekka Paalanen struct list_head list;
36ff3a3e9bSPekka Paalanen struct kmmio_probe probe;
37dee310d0SPekka Paalanen resource_size_t phys;
38ff3a3e9bSPekka Paalanen unsigned long id;
39ff3a3e9bSPekka Paalanen };
40ff3a3e9bSPekka Paalanen
41ff3a3e9bSPekka Paalanen /* Accessed per-cpu. */
42ff3a3e9bSPekka Paalanen static DEFINE_PER_CPU(struct trap_reason, pf_reason);
43ff3a3e9bSPekka Paalanen static DEFINE_PER_CPU(struct mmiotrace_rw, cpu_trace);
44ff3a3e9bSPekka Paalanen
45ff3a3e9bSPekka Paalanen static DEFINE_MUTEX(mmiotrace_mutex);
46ff3a3e9bSPekka Paalanen static DEFINE_SPINLOCK(trace_lock);
47ff3a3e9bSPekka Paalanen static atomic_t mmiotrace_enabled;
48ff3a3e9bSPekka Paalanen static LIST_HEAD(trace_list); /* struct remap_trace */
49ff3a3e9bSPekka Paalanen
50ff3a3e9bSPekka Paalanen /*
51ff3a3e9bSPekka Paalanen * Locking in this file:
52ff3a3e9bSPekka Paalanen * - mmiotrace_mutex enforces enable/disable_mmiotrace() critical sections.
53ff3a3e9bSPekka Paalanen * - mmiotrace_enabled may be modified only when holding mmiotrace_mutex
54ff3a3e9bSPekka Paalanen * and trace_lock.
55ff3a3e9bSPekka Paalanen * - Routines depending on is_enabled() must take trace_lock.
56ff3a3e9bSPekka Paalanen * - trace_list users must hold trace_lock.
579e57fb35SPekka Paalanen * - is_enabled() guarantees that mmio_trace_{rw,mapping} are allowed.
58ff3a3e9bSPekka Paalanen * - pre/post callbacks assume the effect of is_enabled() being true.
59ff3a3e9bSPekka Paalanen */
60ff3a3e9bSPekka Paalanen
61ff3a3e9bSPekka Paalanen /* module parameters */
62ff3a3e9bSPekka Paalanen static unsigned long filter_offset;
63476bc001SRusty Russell static bool nommiotrace;
64476bc001SRusty Russell static bool trace_pc;
65ff3a3e9bSPekka Paalanen
66ff3a3e9bSPekka Paalanen module_param(filter_offset, ulong, 0);
67ff3a3e9bSPekka Paalanen module_param(nommiotrace, bool, 0);
68ff3a3e9bSPekka Paalanen module_param(trace_pc, bool, 0);
69ff3a3e9bSPekka Paalanen
70ff3a3e9bSPekka Paalanen MODULE_PARM_DESC(filter_offset, "Start address of traced mappings.");
71ff3a3e9bSPekka Paalanen MODULE_PARM_DESC(nommiotrace, "Disable actual MMIO tracing.");
72ff3a3e9bSPekka Paalanen MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions.");
73ff3a3e9bSPekka Paalanen
is_enabled(void)74ff3a3e9bSPekka Paalanen static bool is_enabled(void)
75ff3a3e9bSPekka Paalanen {
76ff3a3e9bSPekka Paalanen return atomic_read(&mmiotrace_enabled);
77ff3a3e9bSPekka Paalanen }
78ff3a3e9bSPekka Paalanen
print_pte(unsigned long address)79ff3a3e9bSPekka Paalanen static void print_pte(unsigned long address)
80ff3a3e9bSPekka Paalanen {
81790e2a29SPekka Paalanen unsigned int level;
82ff3a3e9bSPekka Paalanen pte_t *pte = lookup_address(address, &level);
83ff3a3e9bSPekka Paalanen
84ff3a3e9bSPekka Paalanen if (!pte) {
853a0340beSJoe Perches pr_err("Error in %s: no pte for page 0x%08lx\n",
86ff3a3e9bSPekka Paalanen __func__, address);
87ff3a3e9bSPekka Paalanen return;
88ff3a3e9bSPekka Paalanen }
89ff3a3e9bSPekka Paalanen
90ff3a3e9bSPekka Paalanen if (level == PG_LEVEL_2M) {
913a0340beSJoe Perches pr_emerg("4MB pages are not currently supported: 0x%08lx\n",
923a0340beSJoe Perches address);
93ff3a3e9bSPekka Paalanen BUG();
94ff3a3e9bSPekka Paalanen }
953a0340beSJoe Perches pr_info("pte for 0x%lx: 0x%llx 0x%llx\n",
963a0340beSJoe Perches address,
970663bb6cSRandy Dunlap (unsigned long long)pte_val(*pte),
980663bb6cSRandy Dunlap (unsigned long long)pte_val(*pte) & _PAGE_PRESENT);
99ff3a3e9bSPekka Paalanen }
100ff3a3e9bSPekka Paalanen
101ff3a3e9bSPekka Paalanen /*
102ff3a3e9bSPekka Paalanen * For some reason the pre/post pairs have been called in an
103ff3a3e9bSPekka Paalanen * unmatched order. Report and die.
104ff3a3e9bSPekka Paalanen */
die_kmmio_nesting_error(struct pt_regs * regs,unsigned long addr)105ff3a3e9bSPekka Paalanen static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
106ff3a3e9bSPekka Paalanen {
107ff3a3e9bSPekka Paalanen const struct trap_reason *my_reason = &get_cpu_var(pf_reason);
1083a0340beSJoe Perches pr_emerg("unexpected fault for address: 0x%08lx, last fault for address: 0x%08lx\n",
109ff3a3e9bSPekka Paalanen addr, my_reason->addr);
110ff3a3e9bSPekka Paalanen print_pte(addr);
1117b606162SSergey Senozhatsky pr_emerg("faulting IP is at %pS\n", (void *)regs->ip);
1127b606162SSergey Senozhatsky pr_emerg("last faulting IP was at %pS\n", (void *)my_reason->ip);
113ff3a3e9bSPekka Paalanen #ifdef __i386__
114ff3a3e9bSPekka Paalanen pr_emerg("eax: %08lx ebx: %08lx ecx: %08lx edx: %08lx\n",
115ff3a3e9bSPekka Paalanen regs->ax, regs->bx, regs->cx, regs->dx);
116ff3a3e9bSPekka Paalanen pr_emerg("esi: %08lx edi: %08lx ebp: %08lx esp: %08lx\n",
117ff3a3e9bSPekka Paalanen regs->si, regs->di, regs->bp, regs->sp);
118ff3a3e9bSPekka Paalanen #else
119ff3a3e9bSPekka Paalanen pr_emerg("rax: %016lx rcx: %016lx rdx: %016lx\n",
120ff3a3e9bSPekka Paalanen regs->ax, regs->cx, regs->dx);
121ff3a3e9bSPekka Paalanen pr_emerg("rsi: %016lx rdi: %016lx rbp: %016lx rsp: %016lx\n",
122ff3a3e9bSPekka Paalanen regs->si, regs->di, regs->bp, regs->sp);
123ff3a3e9bSPekka Paalanen #endif
124ff3a3e9bSPekka Paalanen put_cpu_var(pf_reason);
125ff3a3e9bSPekka Paalanen BUG();
126ff3a3e9bSPekka Paalanen }
127ff3a3e9bSPekka Paalanen
pre(struct kmmio_probe * p,struct pt_regs * regs,unsigned long addr)128ff3a3e9bSPekka Paalanen static void pre(struct kmmio_probe *p, struct pt_regs *regs,
129ff3a3e9bSPekka Paalanen unsigned long addr)
130ff3a3e9bSPekka Paalanen {
131ff3a3e9bSPekka Paalanen struct trap_reason *my_reason = &get_cpu_var(pf_reason);
132ff3a3e9bSPekka Paalanen struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace);
133ff3a3e9bSPekka Paalanen const unsigned long instptr = instruction_pointer(regs);
134ff3a3e9bSPekka Paalanen const enum reason_type type = get_ins_type(instptr);
135a50445d7SPekka Paalanen struct remap_trace *trace = p->private;
136ff3a3e9bSPekka Paalanen
137ff3a3e9bSPekka Paalanen /* it doesn't make sense to have more than one active trace per cpu */
138ff3a3e9bSPekka Paalanen if (my_reason->active_traces)
139ff3a3e9bSPekka Paalanen die_kmmio_nesting_error(regs, addr);
140ff3a3e9bSPekka Paalanen else
141ff3a3e9bSPekka Paalanen my_reason->active_traces++;
142ff3a3e9bSPekka Paalanen
143ff3a3e9bSPekka Paalanen my_reason->type = type;
144ff3a3e9bSPekka Paalanen my_reason->addr = addr;
145ff3a3e9bSPekka Paalanen my_reason->ip = instptr;
146ff3a3e9bSPekka Paalanen
147ff3a3e9bSPekka Paalanen my_trace->phys = addr - trace->probe.addr + trace->phys;
148ff3a3e9bSPekka Paalanen my_trace->map_id = trace->id;
149ff3a3e9bSPekka Paalanen
150ff3a3e9bSPekka Paalanen /*
151ff3a3e9bSPekka Paalanen * Only record the program counter when requested.
152ff3a3e9bSPekka Paalanen * It may taint clean-room reverse engineering.
153ff3a3e9bSPekka Paalanen */
154ff3a3e9bSPekka Paalanen if (trace_pc)
155ff3a3e9bSPekka Paalanen my_trace->pc = instptr;
156ff3a3e9bSPekka Paalanen else
157ff3a3e9bSPekka Paalanen my_trace->pc = 0;
158ff3a3e9bSPekka Paalanen
159ff3a3e9bSPekka Paalanen /*
160ff3a3e9bSPekka Paalanen * XXX: the timestamp recorded will be *after* the tracing has been
161ff3a3e9bSPekka Paalanen * done, not at the time we hit the instruction. SMP implications
162ff3a3e9bSPekka Paalanen * on event ordering?
163ff3a3e9bSPekka Paalanen */
164ff3a3e9bSPekka Paalanen
165ff3a3e9bSPekka Paalanen switch (type) {
166ff3a3e9bSPekka Paalanen case REG_READ:
167ff3a3e9bSPekka Paalanen my_trace->opcode = MMIO_READ;
168ff3a3e9bSPekka Paalanen my_trace->width = get_ins_mem_width(instptr);
169ff3a3e9bSPekka Paalanen break;
170ff3a3e9bSPekka Paalanen case REG_WRITE:
171ff3a3e9bSPekka Paalanen my_trace->opcode = MMIO_WRITE;
172ff3a3e9bSPekka Paalanen my_trace->width = get_ins_mem_width(instptr);
173ff3a3e9bSPekka Paalanen my_trace->value = get_ins_reg_val(instptr, regs);
174ff3a3e9bSPekka Paalanen break;
175ff3a3e9bSPekka Paalanen case IMM_WRITE:
176ff3a3e9bSPekka Paalanen my_trace->opcode = MMIO_WRITE;
177ff3a3e9bSPekka Paalanen my_trace->width = get_ins_mem_width(instptr);
178ff3a3e9bSPekka Paalanen my_trace->value = get_ins_imm_val(instptr);
179ff3a3e9bSPekka Paalanen break;
180ff3a3e9bSPekka Paalanen default:
181ff3a3e9bSPekka Paalanen {
182ff3a3e9bSPekka Paalanen unsigned char *ip = (unsigned char *)instptr;
183ff3a3e9bSPekka Paalanen my_trace->opcode = MMIO_UNKNOWN_OP;
184ff3a3e9bSPekka Paalanen my_trace->width = 0;
185ff3a3e9bSPekka Paalanen my_trace->value = (*ip) << 16 | *(ip + 1) << 8 |
186ff3a3e9bSPekka Paalanen *(ip + 2);
187ff3a3e9bSPekka Paalanen }
188ff3a3e9bSPekka Paalanen }
189ff3a3e9bSPekka Paalanen put_cpu_var(cpu_trace);
190ff3a3e9bSPekka Paalanen put_cpu_var(pf_reason);
191ff3a3e9bSPekka Paalanen }
192ff3a3e9bSPekka Paalanen
post(struct kmmio_probe * p,unsigned long condition,struct pt_regs * regs)193ff3a3e9bSPekka Paalanen static void post(struct kmmio_probe *p, unsigned long condition,
194ff3a3e9bSPekka Paalanen struct pt_regs *regs)
195ff3a3e9bSPekka Paalanen {
196ff3a3e9bSPekka Paalanen struct trap_reason *my_reason = &get_cpu_var(pf_reason);
197ff3a3e9bSPekka Paalanen struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace);
198ff3a3e9bSPekka Paalanen
199ff3a3e9bSPekka Paalanen /* this should always return the active_trace count to 0 */
200ff3a3e9bSPekka Paalanen my_reason->active_traces--;
201ff3a3e9bSPekka Paalanen if (my_reason->active_traces) {
2023a0340beSJoe Perches pr_emerg("unexpected post handler");
203ff3a3e9bSPekka Paalanen BUG();
204ff3a3e9bSPekka Paalanen }
205ff3a3e9bSPekka Paalanen
206ff3a3e9bSPekka Paalanen switch (my_reason->type) {
207ff3a3e9bSPekka Paalanen case REG_READ:
208ff3a3e9bSPekka Paalanen my_trace->value = get_ins_reg_val(my_reason->ip, regs);
209ff3a3e9bSPekka Paalanen break;
210ff3a3e9bSPekka Paalanen default:
211ff3a3e9bSPekka Paalanen break;
212ff3a3e9bSPekka Paalanen }
213ff3a3e9bSPekka Paalanen
214ff3a3e9bSPekka Paalanen mmio_trace_rw(my_trace);
215ff3a3e9bSPekka Paalanen put_cpu_var(cpu_trace);
216ff3a3e9bSPekka Paalanen put_cpu_var(pf_reason);
217ff3a3e9bSPekka Paalanen }
218ff3a3e9bSPekka Paalanen
ioremap_trace_core(resource_size_t offset,unsigned long size,void __iomem * addr)219dee310d0SPekka Paalanen static void ioremap_trace_core(resource_size_t offset, unsigned long size,
220ff3a3e9bSPekka Paalanen void __iomem *addr)
221ff3a3e9bSPekka Paalanen {
222ff3a3e9bSPekka Paalanen static atomic_t next_id;
223ff3a3e9bSPekka Paalanen struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
22487e547feSPekka Paalanen /* These are page-unaligned. */
225ff3a3e9bSPekka Paalanen struct mmiotrace_map map = {
226ff3a3e9bSPekka Paalanen .phys = offset,
227ff3a3e9bSPekka Paalanen .virt = (unsigned long)addr,
228ff3a3e9bSPekka Paalanen .len = size,
229ff3a3e9bSPekka Paalanen .opcode = MMIO_PROBE
230ff3a3e9bSPekka Paalanen };
231ff3a3e9bSPekka Paalanen
232ff3a3e9bSPekka Paalanen if (!trace) {
2333a0340beSJoe Perches pr_err("kmalloc failed in ioremap\n");
234ff3a3e9bSPekka Paalanen return;
235ff3a3e9bSPekka Paalanen }
236ff3a3e9bSPekka Paalanen
237ff3a3e9bSPekka Paalanen *trace = (struct remap_trace) {
238ff3a3e9bSPekka Paalanen .probe = {
239ff3a3e9bSPekka Paalanen .addr = (unsigned long)addr,
240ff3a3e9bSPekka Paalanen .len = size,
241ff3a3e9bSPekka Paalanen .pre_handler = pre,
242ff3a3e9bSPekka Paalanen .post_handler = post,
243a50445d7SPekka Paalanen .private = trace
244ff3a3e9bSPekka Paalanen },
245ff3a3e9bSPekka Paalanen .phys = offset,
246ff3a3e9bSPekka Paalanen .id = atomic_inc_return(&next_id)
247ff3a3e9bSPekka Paalanen };
248ff3a3e9bSPekka Paalanen map.map_id = trace->id;
249ff3a3e9bSPekka Paalanen
250ff3a3e9bSPekka Paalanen spin_lock_irq(&trace_lock);
251bbe5c783SPekka Paalanen if (!is_enabled()) {
252bbe5c783SPekka Paalanen kfree(trace);
253ff3a3e9bSPekka Paalanen goto not_enabled;
254bbe5c783SPekka Paalanen }
255ff3a3e9bSPekka Paalanen
256ff3a3e9bSPekka Paalanen mmio_trace_mapping(&map);
257ff3a3e9bSPekka Paalanen list_add_tail(&trace->list, &trace_list);
258ff3a3e9bSPekka Paalanen if (!nommiotrace)
259ff3a3e9bSPekka Paalanen register_kmmio_probe(&trace->probe);
260ff3a3e9bSPekka Paalanen
261ff3a3e9bSPekka Paalanen not_enabled:
262ff3a3e9bSPekka Paalanen spin_unlock_irq(&trace_lock);
263ff3a3e9bSPekka Paalanen }
264ff3a3e9bSPekka Paalanen
mmiotrace_ioremap(resource_size_t offset,unsigned long size,void __iomem * addr)265dee310d0SPekka Paalanen void mmiotrace_ioremap(resource_size_t offset, unsigned long size,
266dee310d0SPekka Paalanen void __iomem *addr)
267ff3a3e9bSPekka Paalanen {
268ff3a3e9bSPekka Paalanen if (!is_enabled()) /* recheck and proper locking in *_core() */
269ff3a3e9bSPekka Paalanen return;
270ff3a3e9bSPekka Paalanen
2713a0340beSJoe Perches pr_debug("ioremap_*(0x%llx, 0x%lx) = %p\n",
272dee310d0SPekka Paalanen (unsigned long long)offset, size, addr);
273ff3a3e9bSPekka Paalanen if ((filter_offset) && (offset != filter_offset))
274ff3a3e9bSPekka Paalanen return;
275ff3a3e9bSPekka Paalanen ioremap_trace_core(offset, size, addr);
276ff3a3e9bSPekka Paalanen }
277ff3a3e9bSPekka Paalanen
iounmap_trace_core(volatile void __iomem * addr)278ff3a3e9bSPekka Paalanen static void iounmap_trace_core(volatile void __iomem *addr)
279ff3a3e9bSPekka Paalanen {
280ff3a3e9bSPekka Paalanen struct mmiotrace_map map = {
281ff3a3e9bSPekka Paalanen .phys = 0,
282ff3a3e9bSPekka Paalanen .virt = (unsigned long)addr,
283ff3a3e9bSPekka Paalanen .len = 0,
284ff3a3e9bSPekka Paalanen .opcode = MMIO_UNPROBE
285ff3a3e9bSPekka Paalanen };
286ff3a3e9bSPekka Paalanen struct remap_trace *trace;
287ff3a3e9bSPekka Paalanen struct remap_trace *tmp;
288ff3a3e9bSPekka Paalanen struct remap_trace *found_trace = NULL;
289ff3a3e9bSPekka Paalanen
2903a0340beSJoe Perches pr_debug("Unmapping %p.\n", addr);
291ff3a3e9bSPekka Paalanen
292ff3a3e9bSPekka Paalanen spin_lock_irq(&trace_lock);
293ff3a3e9bSPekka Paalanen if (!is_enabled())
294ff3a3e9bSPekka Paalanen goto not_enabled;
295ff3a3e9bSPekka Paalanen
296ff3a3e9bSPekka Paalanen list_for_each_entry_safe(trace, tmp, &trace_list, list) {
297ff3a3e9bSPekka Paalanen if ((unsigned long)addr == trace->probe.addr) {
298ff3a3e9bSPekka Paalanen if (!nommiotrace)
299ff3a3e9bSPekka Paalanen unregister_kmmio_probe(&trace->probe);
300ff3a3e9bSPekka Paalanen list_del(&trace->list);
301ff3a3e9bSPekka Paalanen found_trace = trace;
302ff3a3e9bSPekka Paalanen break;
303ff3a3e9bSPekka Paalanen }
304ff3a3e9bSPekka Paalanen }
305ff3a3e9bSPekka Paalanen map.map_id = (found_trace) ? found_trace->id : -1;
306ff3a3e9bSPekka Paalanen mmio_trace_mapping(&map);
307ff3a3e9bSPekka Paalanen
308ff3a3e9bSPekka Paalanen not_enabled:
309ff3a3e9bSPekka Paalanen spin_unlock_irq(&trace_lock);
310ff3a3e9bSPekka Paalanen if (found_trace) {
311ff3a3e9bSPekka Paalanen synchronize_rcu(); /* unregister_kmmio_probe() requirement */
312ff3a3e9bSPekka Paalanen kfree(found_trace);
313ff3a3e9bSPekka Paalanen }
314ff3a3e9bSPekka Paalanen }
315ff3a3e9bSPekka Paalanen
mmiotrace_iounmap(volatile void __iomem * addr)316ff3a3e9bSPekka Paalanen void mmiotrace_iounmap(volatile void __iomem *addr)
317ff3a3e9bSPekka Paalanen {
318ff3a3e9bSPekka Paalanen might_sleep();
319ff3a3e9bSPekka Paalanen if (is_enabled()) /* recheck and proper locking in *_core() */
320ff3a3e9bSPekka Paalanen iounmap_trace_core(addr);
321ff3a3e9bSPekka Paalanen }
322ff3a3e9bSPekka Paalanen
mmiotrace_printk(const char * fmt,...)3239e57fb35SPekka Paalanen int mmiotrace_printk(const char *fmt, ...)
3249e57fb35SPekka Paalanen {
3259e57fb35SPekka Paalanen int ret = 0;
3269e57fb35SPekka Paalanen va_list args;
3279e57fb35SPekka Paalanen unsigned long flags;
3289e57fb35SPekka Paalanen va_start(args, fmt);
3299e57fb35SPekka Paalanen
3309e57fb35SPekka Paalanen spin_lock_irqsave(&trace_lock, flags);
3319e57fb35SPekka Paalanen if (is_enabled())
3329e57fb35SPekka Paalanen ret = mmio_trace_printk(fmt, args);
3339e57fb35SPekka Paalanen spin_unlock_irqrestore(&trace_lock, flags);
3349e57fb35SPekka Paalanen
3359e57fb35SPekka Paalanen va_end(args);
3369e57fb35SPekka Paalanen return ret;
3379e57fb35SPekka Paalanen }
3389e57fb35SPekka Paalanen EXPORT_SYMBOL(mmiotrace_printk);
3399e57fb35SPekka Paalanen
clear_trace_list(void)340ff3a3e9bSPekka Paalanen static void clear_trace_list(void)
341ff3a3e9bSPekka Paalanen {
342ff3a3e9bSPekka Paalanen struct remap_trace *trace;
343ff3a3e9bSPekka Paalanen struct remap_trace *tmp;
344ff3a3e9bSPekka Paalanen
345ff3a3e9bSPekka Paalanen /*
346ff3a3e9bSPekka Paalanen * No locking required, because the caller ensures we are in a
347ff3a3e9bSPekka Paalanen * critical section via mutex, and is_enabled() is false,
348ff3a3e9bSPekka Paalanen * i.e. nothing can traverse or modify this list.
349ff3a3e9bSPekka Paalanen * Caller also ensures is_enabled() cannot change.
350ff3a3e9bSPekka Paalanen */
351ff3a3e9bSPekka Paalanen list_for_each_entry(trace, &trace_list, list) {
3523a0340beSJoe Perches pr_notice("purging non-iounmapped trace @0x%08lx, size 0x%lx.\n",
353ff3a3e9bSPekka Paalanen trace->probe.addr, trace->probe.len);
354ff3a3e9bSPekka Paalanen if (!nommiotrace)
355ff3a3e9bSPekka Paalanen unregister_kmmio_probe(&trace->probe);
356ff3a3e9bSPekka Paalanen }
357ff3a3e9bSPekka Paalanen synchronize_rcu(); /* unregister_kmmio_probe() requirement */
358ff3a3e9bSPekka Paalanen
359ff3a3e9bSPekka Paalanen list_for_each_entry_safe(trace, tmp, &trace_list, list) {
360ff3a3e9bSPekka Paalanen list_del(&trace->list);
361ff3a3e9bSPekka Paalanen kfree(trace);
362ff3a3e9bSPekka Paalanen }
363ff3a3e9bSPekka Paalanen }
364ff3a3e9bSPekka Paalanen
3657423d111SPekka Paalanen #ifdef CONFIG_HOTPLUG_CPU
366c38da569SRusty Russell static cpumask_var_t downed_cpus;
3677423d111SPekka Paalanen
enter_uniprocessor(void)3687423d111SPekka Paalanen static void enter_uniprocessor(void)
3697423d111SPekka Paalanen {
3707423d111SPekka Paalanen int cpu;
3717423d111SPekka Paalanen int err;
3727423d111SPekka Paalanen
373d7110a26SNathan Chancellor if (!cpumask_available(downed_cpus) &&
374c38da569SRusty Russell !alloc_cpumask_var(&downed_cpus, GFP_KERNEL)) {
3753a0340beSJoe Perches pr_notice("Failed to allocate mask\n");
376c38da569SRusty Russell goto out;
377c38da569SRusty Russell }
378c38da569SRusty Russell
37977ad320cSSebastian Andrzej Siewior cpus_read_lock();
380c38da569SRusty Russell cpumask_copy(downed_cpus, cpu_online_mask);
381c38da569SRusty Russell cpumask_clear_cpu(cpumask_first(cpu_online_mask), downed_cpus);
3827423d111SPekka Paalanen if (num_online_cpus() > 1)
3833a0340beSJoe Perches pr_notice("Disabling non-boot CPUs...\n");
38477ad320cSSebastian Andrzej Siewior cpus_read_unlock();
3857423d111SPekka Paalanen
386c38da569SRusty Russell for_each_cpu(cpu, downed_cpus) {
387af7aa046SQais Yousef err = remove_cpu(cpu);
388970e6fa0SPekka Paalanen if (!err)
3893a0340beSJoe Perches pr_info("CPU%d is down.\n", cpu);
390970e6fa0SPekka Paalanen else
3913a0340beSJoe Perches pr_err("Error taking CPU%d down: %d\n", cpu, err);
3927423d111SPekka Paalanen }
393c38da569SRusty Russell out:
3947423d111SPekka Paalanen if (num_online_cpus() > 1)
3958d3bcc44SKefeng Wang pr_warn("multiple CPUs still online, may miss events.\n");
3967423d111SPekka Paalanen }
3977423d111SPekka Paalanen
leave_uniprocessor(void)398148f9bb8SPaul Gortmaker static void leave_uniprocessor(void)
3997423d111SPekka Paalanen {
4007423d111SPekka Paalanen int cpu;
4017423d111SPekka Paalanen int err;
4027423d111SPekka Paalanen
4033a5ff1f6SYury Norov if (!cpumask_available(downed_cpus) || cpumask_empty(downed_cpus))
4047423d111SPekka Paalanen return;
4053a0340beSJoe Perches pr_notice("Re-enabling CPUs...\n");
406c38da569SRusty Russell for_each_cpu(cpu, downed_cpus) {
407af7aa046SQais Yousef err = add_cpu(cpu);
4087423d111SPekka Paalanen if (!err)
4093a0340beSJoe Perches pr_info("enabled CPU%d.\n", cpu);
4107423d111SPekka Paalanen else
4113a0340beSJoe Perches pr_err("cannot re-enable CPU%d: %d\n", cpu, err);
4127423d111SPekka Paalanen }
4137423d111SPekka Paalanen }
4147423d111SPekka Paalanen
4157423d111SPekka Paalanen #else /* !CONFIG_HOTPLUG_CPU */
enter_uniprocessor(void)4167423d111SPekka Paalanen static void enter_uniprocessor(void)
4177423d111SPekka Paalanen {
4187423d111SPekka Paalanen if (num_online_cpus() > 1)
4198d3bcc44SKefeng Wang pr_warn("multiple CPUs are online, may miss events. "
4207423d111SPekka Paalanen "Suggest booting with maxcpus=1 kernel argument.\n");
4217423d111SPekka Paalanen }
4227423d111SPekka Paalanen
leave_uniprocessor(void)4237423d111SPekka Paalanen static void leave_uniprocessor(void)
4247423d111SPekka Paalanen {
4257423d111SPekka Paalanen }
4267423d111SPekka Paalanen #endif
4277423d111SPekka Paalanen
enable_mmiotrace(void)428ff3a3e9bSPekka Paalanen void enable_mmiotrace(void)
429ff3a3e9bSPekka Paalanen {
430ff3a3e9bSPekka Paalanen mutex_lock(&mmiotrace_mutex);
431ff3a3e9bSPekka Paalanen if (is_enabled())
432ff3a3e9bSPekka Paalanen goto out;
433ff3a3e9bSPekka Paalanen
434ff3a3e9bSPekka Paalanen if (nommiotrace)
4353a0340beSJoe Perches pr_info("MMIO tracing disabled.\n");
4360f9a623dSStuart Bennett kmmio_init();
4377423d111SPekka Paalanen enter_uniprocessor();
438ff3a3e9bSPekka Paalanen spin_lock_irq(&trace_lock);
439ff3a3e9bSPekka Paalanen atomic_inc(&mmiotrace_enabled);
440ff3a3e9bSPekka Paalanen spin_unlock_irq(&trace_lock);
4413a0340beSJoe Perches pr_info("enabled.\n");
442ff3a3e9bSPekka Paalanen out:
443ff3a3e9bSPekka Paalanen mutex_unlock(&mmiotrace_mutex);
444ff3a3e9bSPekka Paalanen }
445ff3a3e9bSPekka Paalanen
disable_mmiotrace(void)446ff3a3e9bSPekka Paalanen void disable_mmiotrace(void)
447ff3a3e9bSPekka Paalanen {
448ff3a3e9bSPekka Paalanen mutex_lock(&mmiotrace_mutex);
449ff3a3e9bSPekka Paalanen if (!is_enabled())
450ff3a3e9bSPekka Paalanen goto out;
451ff3a3e9bSPekka Paalanen
452ff3a3e9bSPekka Paalanen spin_lock_irq(&trace_lock);
453ff3a3e9bSPekka Paalanen atomic_dec(&mmiotrace_enabled);
454ff3a3e9bSPekka Paalanen BUG_ON(is_enabled());
455ff3a3e9bSPekka Paalanen spin_unlock_irq(&trace_lock);
456ff3a3e9bSPekka Paalanen
457ff3a3e9bSPekka Paalanen clear_trace_list(); /* guarantees: no more kmmio callbacks */
4587423d111SPekka Paalanen leave_uniprocessor();
4590f9a623dSStuart Bennett kmmio_cleanup();
4603a0340beSJoe Perches pr_info("disabled.\n");
461ff3a3e9bSPekka Paalanen out:
462ff3a3e9bSPekka Paalanen mutex_unlock(&mmiotrace_mutex);
463ff3a3e9bSPekka Paalanen }
464