1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * VAS Fault handling.
4  * Copyright 2019, IBM Corporation
5  */
6 
7 #define pr_fmt(fmt) "vas: " fmt
8 
9 #include <linux/kernel.h>
10 #include <linux/types.h>
11 #include <linux/slab.h>
12 #include <linux/uaccess.h>
13 #include <linux/kthread.h>
14 #include <linux/sched/signal.h>
15 #include <linux/mmu_context.h>
16 #include <asm/icswx.h>
17 
18 #include "vas.h"
19 
20 /*
21  * The maximum FIFO size for fault window can be 8MB
22  * (VAS_RX_FIFO_SIZE_MAX). Using 4MB FIFO since each VAS
23  * instance will be having fault window.
24  * 8MB FIFO can be used if expects more faults for each VAS
25  * instance.
26  */
27 #define VAS_FAULT_WIN_FIFO_SIZE	(4 << 20)
28 
29 static void dump_fifo(struct vas_instance *vinst, void *entry)
30 {
31 	unsigned long *end = vinst->fault_fifo + vinst->fault_fifo_size;
32 	unsigned long *fifo = entry;
33 	int i;
34 
35 	pr_err("Fault fifo size %d, Max crbs %d\n", vinst->fault_fifo_size,
36 			vinst->fault_fifo_size / CRB_SIZE);
37 
38 	/* Dump 10 CRB entries or until end of FIFO */
39 	pr_err("Fault FIFO Dump:\n");
40 	for (i = 0; i < 10*(CRB_SIZE/8) && fifo < end; i += 4, fifo += 4) {
41 		pr_err("[%.3d, %p]: 0x%.16lx 0x%.16lx 0x%.16lx 0x%.16lx\n",
42 			i, fifo, *fifo, *(fifo+1), *(fifo+2), *(fifo+3));
43 	}
44 }
45 
46 /*
47  * Process valid CRBs in fault FIFO.
48  * NX process user space requests, return credit and update the status
49  * in CRB. If it encounters transalation error when accessing CRB or
50  * request buffers, raises interrupt on the CPU to handle the fault.
51  * It takes credit on fault window, updates nx_fault_stamp in CRB with
52  * the following information and pastes CRB in fault FIFO.
53  *
54  * pswid - window ID of the window on which the request is sent.
55  * fault_storage_addr - fault address
56  *
57  * It can raise a single interrupt for multiple faults. Expects OS to
58  * process all valid faults and return credit for each fault on user
59  * space and fault windows. This fault FIFO control will be done with
60  * credit mechanism. NX can continuously paste CRBs until credits are not
61  * available on fault window. Otherwise, returns with RMA_reject.
62  *
63  * Total credits available on fault window: FIFO_SIZE(4MB)/CRBS_SIZE(128)
64  *
65  */
66 irqreturn_t vas_fault_thread_fn(int irq, void *data)
67 {
68 	struct vas_instance *vinst = data;
69 	struct coprocessor_request_block *crb, *entry;
70 	struct coprocessor_request_block buf;
71 	struct pnv_vas_window *window;
72 	unsigned long flags;
73 	void *fifo;
74 
75 	crb = &buf;
76 
77 	/*
78 	 * VAS can interrupt with multiple page faults. So process all
79 	 * valid CRBs within fault FIFO until reaches invalid CRB.
80 	 * We use CCW[0] and pswid to validate CRBs:
81 	 *
82 	 * CCW[0]	Reserved bit. When NX pastes CRB, CCW[0]=0
83 	 *		OS sets this bit to 1 after reading CRB.
84 	 * pswid	NX assigns window ID. Set pswid to -1 after
85 	 *		reading CRB from fault FIFO.
86 	 *
87 	 * We exit this function if no valid CRBs are available to process.
88 	 * So acquire fault_lock and reset fifo_in_progress to 0 before
89 	 * exit.
90 	 * In case kernel receives another interrupt with different page
91 	 * fault, interrupt handler returns with IRQ_HANDLED if
92 	 * fifo_in_progress is set. Means these new faults will be
93 	 * handled by the current thread. Otherwise set fifo_in_progress
94 	 * and return IRQ_WAKE_THREAD to wake up thread.
95 	 */
96 	while (true) {
97 		spin_lock_irqsave(&vinst->fault_lock, flags);
98 		/*
99 		 * Advance the fault fifo pointer to next CRB.
100 		 * Use CRB_SIZE rather than sizeof(*crb) since the latter is
101 		 * aligned to CRB_ALIGN (256) but the CRB written to by VAS is
102 		 * only CRB_SIZE in len.
103 		 */
104 		fifo = vinst->fault_fifo + (vinst->fault_crbs * CRB_SIZE);
105 		entry = fifo;
106 
107 		if ((entry->stamp.nx.pswid == cpu_to_be32(FIFO_INVALID_ENTRY))
108 			|| (entry->ccw & cpu_to_be32(CCW0_INVALID))) {
109 			vinst->fifo_in_progress = 0;
110 			spin_unlock_irqrestore(&vinst->fault_lock, flags);
111 			return IRQ_HANDLED;
112 		}
113 
114 		spin_unlock_irqrestore(&vinst->fault_lock, flags);
115 		vinst->fault_crbs++;
116 		if (vinst->fault_crbs == (vinst->fault_fifo_size / CRB_SIZE))
117 			vinst->fault_crbs = 0;
118 
119 		memcpy(crb, fifo, CRB_SIZE);
120 		entry->stamp.nx.pswid = cpu_to_be32(FIFO_INVALID_ENTRY);
121 		entry->ccw |= cpu_to_be32(CCW0_INVALID);
122 		/*
123 		 * Return credit for the fault window.
124 		 */
125 		vas_return_credit(vinst->fault_win, false);
126 
127 		pr_devel("VAS[%d] fault_fifo %p, fifo %p, fault_crbs %d\n",
128 				vinst->vas_id, vinst->fault_fifo, fifo,
129 				vinst->fault_crbs);
130 
131 		vas_dump_crb(crb);
132 		window = vas_pswid_to_window(vinst,
133 				be32_to_cpu(crb->stamp.nx.pswid));
134 
135 		if (IS_ERR(window)) {
136 			/*
137 			 * We got an interrupt about a specific send
138 			 * window but we can't find that window and we can't
139 			 * even clean it up (return credit on user space
140 			 * window).
141 			 * But we should not get here.
142 			 * TODO: Disable IRQ.
143 			 */
144 			dump_fifo(vinst, (void *)entry);
145 			pr_err("VAS[%d] fault_fifo %p, fifo %p, pswid 0x%x, fault_crbs %d bad CRB?\n",
146 				vinst->vas_id, vinst->fault_fifo, fifo,
147 				be32_to_cpu(crb->stamp.nx.pswid),
148 				vinst->fault_crbs);
149 
150 			WARN_ON_ONCE(1);
151 		} else {
152 			/*
153 			 * NX sees faults only with user space windows.
154 			 */
155 			if (window->user_win)
156 				vas_update_csb(crb, &window->vas_win.task_ref);
157 			else
158 				WARN_ON_ONCE(!window->user_win);
159 
160 			/*
161 			 * Return credit for send window after processing
162 			 * fault CRB.
163 			 */
164 			vas_return_credit(window, true);
165 		}
166 	}
167 }
168 
169 irqreturn_t vas_fault_handler(int irq, void *dev_id)
170 {
171 	struct vas_instance *vinst = dev_id;
172 	irqreturn_t ret = IRQ_WAKE_THREAD;
173 	unsigned long flags;
174 
175 	/*
176 	 * NX can generate an interrupt for multiple faults. So the
177 	 * fault handler thread process all CRBs until finds invalid
178 	 * entry. In case if NX sees continuous faults, it is possible
179 	 * that the thread function entered with the first interrupt
180 	 * can execute and process all valid CRBs.
181 	 * So wake up thread only if the fault thread is not in progress.
182 	 */
183 	spin_lock_irqsave(&vinst->fault_lock, flags);
184 
185 	if (vinst->fifo_in_progress)
186 		ret = IRQ_HANDLED;
187 	else
188 		vinst->fifo_in_progress = 1;
189 
190 	spin_unlock_irqrestore(&vinst->fault_lock, flags);
191 
192 	return ret;
193 }
194 
195 /*
196  * Fault window is opened per VAS instance. NX pastes fault CRB in fault
197  * FIFO upon page faults.
198  */
199 int vas_setup_fault_window(struct vas_instance *vinst)
200 {
201 	struct vas_rx_win_attr attr;
202 	struct vas_window *win;
203 
204 	vinst->fault_fifo_size = VAS_FAULT_WIN_FIFO_SIZE;
205 	vinst->fault_fifo = kzalloc(vinst->fault_fifo_size, GFP_KERNEL);
206 	if (!vinst->fault_fifo) {
207 		pr_err("Unable to alloc %d bytes for fault_fifo\n",
208 				vinst->fault_fifo_size);
209 		return -ENOMEM;
210 	}
211 
212 	/*
213 	 * Invalidate all CRB entries. NX pastes valid entry for each fault.
214 	 */
215 	memset(vinst->fault_fifo, FIFO_INVALID_ENTRY, vinst->fault_fifo_size);
216 	vas_init_rx_win_attr(&attr, VAS_COP_TYPE_FAULT);
217 
218 	attr.rx_fifo_size = vinst->fault_fifo_size;
219 	attr.rx_fifo = __pa(vinst->fault_fifo);
220 
221 	/*
222 	 * Max creds is based on number of CRBs can fit in the FIFO.
223 	 * (fault_fifo_size/CRB_SIZE). If 8MB FIFO is used, max creds
224 	 * will be 0xffff since the receive creds field is 16bits wide.
225 	 */
226 	attr.wcreds_max = vinst->fault_fifo_size / CRB_SIZE;
227 	attr.lnotify_lpid = 0;
228 	attr.lnotify_pid = mfspr(SPRN_PID);
229 	attr.lnotify_tid = mfspr(SPRN_PID);
230 
231 	win = vas_rx_win_open(vinst->vas_id, VAS_COP_TYPE_FAULT, &attr);
232 	if (IS_ERR(win)) {
233 		pr_err("VAS: Error %ld opening FaultWin\n", PTR_ERR(win));
234 		kfree(vinst->fault_fifo);
235 		return PTR_ERR(win);
236 	}
237 
238 	vinst->fault_win = container_of(win, struct pnv_vas_window, vas_win);
239 
240 	pr_devel("VAS: Created FaultWin %d, LPID/PID/TID [%d/%d/%d]\n",
241 			vinst->fault_win->vas_win.winid, attr.lnotify_lpid,
242 			attr.lnotify_pid, attr.lnotify_tid);
243 
244 	return 0;
245 }
246