1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2020-2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include <linux/poll.h>
25 #include <linux/wait.h>
26 #include <linux/anon_inodes.h>
27 #include <uapi/linux/kfd_ioctl.h>
28 #include "amdgpu.h"
29 #include "amdgpu_vm.h"
30 #include "kfd_priv.h"
31 #include "kfd_smi_events.h"
32 
33 struct kfd_smi_client {
34 	struct list_head list;
35 	struct kfifo fifo;
36 	wait_queue_head_t wait_queue;
37 	/* events enabled */
38 	uint64_t events;
39 	struct kfd_dev *dev;
40 	spinlock_t lock;
41 	struct rcu_head rcu;
42 	pid_t pid;
43 	bool suser;
44 };
45 
46 #define MAX_KFIFO_SIZE	1024
47 
48 static __poll_t kfd_smi_ev_poll(struct file *, struct poll_table_struct *);
49 static ssize_t kfd_smi_ev_read(struct file *, char __user *, size_t, loff_t *);
50 static ssize_t kfd_smi_ev_write(struct file *, const char __user *, size_t,
51 				loff_t *);
52 static int kfd_smi_ev_release(struct inode *, struct file *);
53 
54 static const char kfd_smi_name[] = "kfd_smi_ev";
55 
56 static const struct file_operations kfd_smi_ev_fops = {
57 	.owner = THIS_MODULE,
58 	.poll = kfd_smi_ev_poll,
59 	.read = kfd_smi_ev_read,
60 	.write = kfd_smi_ev_write,
61 	.release = kfd_smi_ev_release
62 };
63 
64 static __poll_t kfd_smi_ev_poll(struct file *filep,
65 				struct poll_table_struct *wait)
66 {
67 	struct kfd_smi_client *client = filep->private_data;
68 	__poll_t mask = 0;
69 
70 	poll_wait(filep, &client->wait_queue, wait);
71 
72 	spin_lock(&client->lock);
73 	if (!kfifo_is_empty(&client->fifo))
74 		mask = EPOLLIN | EPOLLRDNORM;
75 	spin_unlock(&client->lock);
76 
77 	return mask;
78 }
79 
80 static ssize_t kfd_smi_ev_read(struct file *filep, char __user *user,
81 			       size_t size, loff_t *offset)
82 {
83 	int ret;
84 	size_t to_copy;
85 	struct kfd_smi_client *client = filep->private_data;
86 	unsigned char *buf;
87 
88 	size = min_t(size_t, size, MAX_KFIFO_SIZE);
89 	buf = kmalloc(size, GFP_KERNEL);
90 	if (!buf)
91 		return -ENOMEM;
92 
93 	/* kfifo_to_user can sleep so we can't use spinlock protection around
94 	 * it. Instead, we kfifo out as spinlocked then copy them to the user.
95 	 */
96 	spin_lock(&client->lock);
97 	to_copy = kfifo_len(&client->fifo);
98 	if (!to_copy) {
99 		spin_unlock(&client->lock);
100 		ret = -EAGAIN;
101 		goto ret_err;
102 	}
103 	to_copy = min(size, to_copy);
104 	ret = kfifo_out(&client->fifo, buf, to_copy);
105 	spin_unlock(&client->lock);
106 	if (ret <= 0) {
107 		ret = -EAGAIN;
108 		goto ret_err;
109 	}
110 
111 	ret = copy_to_user(user, buf, to_copy);
112 	if (ret) {
113 		ret = -EFAULT;
114 		goto ret_err;
115 	}
116 
117 	kfree(buf);
118 	return to_copy;
119 
120 ret_err:
121 	kfree(buf);
122 	return ret;
123 }
124 
125 static ssize_t kfd_smi_ev_write(struct file *filep, const char __user *user,
126 				size_t size, loff_t *offset)
127 {
128 	struct kfd_smi_client *client = filep->private_data;
129 	uint64_t events;
130 
131 	if (!access_ok(user, size) || size < sizeof(events))
132 		return -EFAULT;
133 	if (copy_from_user(&events, user, sizeof(events)))
134 		return -EFAULT;
135 
136 	WRITE_ONCE(client->events, events);
137 
138 	return sizeof(events);
139 }
140 
141 static void kfd_smi_ev_client_free(struct rcu_head *p)
142 {
143 	struct kfd_smi_client *ev = container_of(p, struct kfd_smi_client, rcu);
144 
145 	kfifo_free(&ev->fifo);
146 	kfree(ev);
147 }
148 
149 static int kfd_smi_ev_release(struct inode *inode, struct file *filep)
150 {
151 	struct kfd_smi_client *client = filep->private_data;
152 	struct kfd_dev *dev = client->dev;
153 
154 	spin_lock(&dev->smi_lock);
155 	list_del_rcu(&client->list);
156 	spin_unlock(&dev->smi_lock);
157 
158 	call_rcu(&client->rcu, kfd_smi_ev_client_free);
159 	return 0;
160 }
161 
162 static bool kfd_smi_ev_enabled(pid_t pid, struct kfd_smi_client *client,
163 			       unsigned int event)
164 {
165 	uint64_t all = KFD_SMI_EVENT_MASK_FROM_INDEX(KFD_SMI_EVENT_ALL_PROCESS);
166 	uint64_t events = READ_ONCE(client->events);
167 
168 	if (pid && client->pid != pid && !(client->suser && (events & all)))
169 		return false;
170 
171 	return events & KFD_SMI_EVENT_MASK_FROM_INDEX(event);
172 }
173 
174 static void add_event_to_kfifo(pid_t pid, struct kfd_dev *dev,
175 			       unsigned int smi_event, char *event_msg, int len)
176 {
177 	struct kfd_smi_client *client;
178 
179 	rcu_read_lock();
180 
181 	list_for_each_entry_rcu(client, &dev->smi_clients, list) {
182 		if (!kfd_smi_ev_enabled(pid, client, smi_event))
183 			continue;
184 		spin_lock(&client->lock);
185 		if (kfifo_avail(&client->fifo) >= len) {
186 			kfifo_in(&client->fifo, event_msg, len);
187 			wake_up_all(&client->wait_queue);
188 		} else {
189 			pr_debug("smi_event(EventID: %u): no space left\n",
190 					smi_event);
191 		}
192 		spin_unlock(&client->lock);
193 	}
194 
195 	rcu_read_unlock();
196 }
197 
198 __printf(4, 5)
199 static void kfd_smi_event_add(pid_t pid, struct kfd_dev *dev,
200 			      unsigned int event, char *fmt, ...)
201 {
202 	char fifo_in[KFD_SMI_EVENT_MSG_SIZE];
203 	int len;
204 	va_list args;
205 
206 	if (list_empty(&dev->smi_clients))
207 		return;
208 
209 	len = snprintf(fifo_in, sizeof(fifo_in), "%x ", event);
210 
211 	va_start(args, fmt);
212 	len += vsnprintf(fifo_in + len, sizeof(fifo_in) - len, fmt, args);
213 	va_end(args);
214 
215 	add_event_to_kfifo(pid, dev, event, fifo_in, len);
216 }
217 
218 void kfd_smi_event_update_gpu_reset(struct kfd_dev *dev, bool post_reset)
219 {
220 	unsigned int event;
221 
222 	if (post_reset) {
223 		event = KFD_SMI_EVENT_GPU_POST_RESET;
224 	} else {
225 		event = KFD_SMI_EVENT_GPU_PRE_RESET;
226 		++(dev->reset_seq_num);
227 	}
228 	kfd_smi_event_add(0, dev, event, "%x\n", dev->reset_seq_num);
229 }
230 
231 void kfd_smi_event_update_thermal_throttling(struct kfd_dev *dev,
232 					     uint64_t throttle_bitmask)
233 {
234 	kfd_smi_event_add(0, dev, KFD_SMI_EVENT_THERMAL_THROTTLE, "%llx:%llx\n",
235 			  throttle_bitmask,
236 			  amdgpu_dpm_get_thermal_throttling_counter(dev->adev));
237 }
238 
239 void kfd_smi_event_update_vmfault(struct kfd_dev *dev, uint16_t pasid)
240 {
241 	struct amdgpu_task_info task_info;
242 
243 	memset(&task_info, 0, sizeof(struct amdgpu_task_info));
244 	amdgpu_vm_get_task_info(dev->adev, pasid, &task_info);
245 	/* Report VM faults from user applications, not retry from kernel */
246 	if (!task_info.pid)
247 		return;
248 
249 	kfd_smi_event_add(0, dev, KFD_SMI_EVENT_VMFAULT, "%x:%s\n",
250 			  task_info.pid, task_info.task_name);
251 }
252 
253 void kfd_smi_event_page_fault_start(struct kfd_dev *dev, pid_t pid,
254 				    unsigned long address, bool write_fault,
255 				    ktime_t ts)
256 {
257 	kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_PAGE_FAULT_START,
258 			  "%lld -%d @%lx(%x) %c\n", ktime_to_ns(ts), pid,
259 			  address, dev->id, write_fault ? 'W' : 'R');
260 }
261 
262 void kfd_smi_event_page_fault_end(struct kfd_dev *dev, pid_t pid,
263 				  unsigned long address, bool migration)
264 {
265 	kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_PAGE_FAULT_END,
266 			  "%lld -%d @%lx(%x) %c\n", ktime_get_boottime_ns(),
267 			  pid, address, dev->id, migration ? 'M' : 'U');
268 }
269 
270 void kfd_smi_event_migration_start(struct kfd_dev *dev, pid_t pid,
271 				   unsigned long start, unsigned long end,
272 				   uint32_t from, uint32_t to,
273 				   uint32_t prefetch_loc, uint32_t preferred_loc,
274 				   uint32_t trigger)
275 {
276 	kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_MIGRATE_START,
277 			  "%lld -%d @%lx(%lx) %x->%x %x:%x %d\n",
278 			  ktime_get_boottime_ns(), pid, start, end - start,
279 			  from, to, prefetch_loc, preferred_loc, trigger);
280 }
281 
282 void kfd_smi_event_migration_end(struct kfd_dev *dev, pid_t pid,
283 				 unsigned long start, unsigned long end,
284 				 uint32_t from, uint32_t to, uint32_t trigger)
285 {
286 	kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_MIGRATE_END,
287 			  "%lld -%d @%lx(%lx) %x->%x %d\n",
288 			  ktime_get_boottime_ns(), pid, start, end - start,
289 			  from, to, trigger);
290 }
291 
292 void kfd_smi_event_queue_eviction(struct kfd_dev *dev, pid_t pid,
293 				  uint32_t trigger)
294 {
295 	kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_QUEUE_EVICTION,
296 			  "%lld -%d %x %d\n", ktime_get_boottime_ns(), pid,
297 			  dev->id, trigger);
298 }
299 
300 void kfd_smi_event_queue_restore(struct kfd_dev *dev, pid_t pid)
301 {
302 	kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_QUEUE_RESTORE,
303 			  "%lld -%d %x\n", ktime_get_boottime_ns(), pid,
304 			  dev->id);
305 }
306 
307 void kfd_smi_event_queue_restore_rescheduled(struct mm_struct *mm)
308 {
309 	struct kfd_process *p;
310 	int i;
311 
312 	p = kfd_lookup_process_by_mm(mm);
313 	if (!p)
314 		return;
315 
316 	for (i = 0; i < p->n_pdds; i++) {
317 		struct kfd_process_device *pdd = p->pdds[i];
318 
319 		kfd_smi_event_add(p->lead_thread->pid, pdd->dev,
320 				  KFD_SMI_EVENT_QUEUE_RESTORE,
321 				  "%lld -%d %x %c\n", ktime_get_boottime_ns(),
322 				  p->lead_thread->pid, pdd->dev->id, 'R');
323 	}
324 	kfd_unref_process(p);
325 }
326 
327 void kfd_smi_event_unmap_from_gpu(struct kfd_dev *dev, pid_t pid,
328 				  unsigned long address, unsigned long last,
329 				  uint32_t trigger)
330 {
331 	kfd_smi_event_add(pid, dev, KFD_SMI_EVENT_UNMAP_FROM_GPU,
332 			  "%lld -%d @%lx(%lx) %x %d\n", ktime_get_boottime_ns(),
333 			  pid, address, last - address + 1, dev->id, trigger);
334 }
335 
336 int kfd_smi_event_open(struct kfd_dev *dev, uint32_t *fd)
337 {
338 	struct kfd_smi_client *client;
339 	int ret;
340 
341 	client = kzalloc(sizeof(struct kfd_smi_client), GFP_KERNEL);
342 	if (!client)
343 		return -ENOMEM;
344 	INIT_LIST_HEAD(&client->list);
345 
346 	ret = kfifo_alloc(&client->fifo, MAX_KFIFO_SIZE, GFP_KERNEL);
347 	if (ret) {
348 		kfree(client);
349 		return ret;
350 	}
351 
352 	init_waitqueue_head(&client->wait_queue);
353 	spin_lock_init(&client->lock);
354 	client->events = 0;
355 	client->dev = dev;
356 	client->pid = current->tgid;
357 	client->suser = capable(CAP_SYS_ADMIN);
358 
359 	spin_lock(&dev->smi_lock);
360 	list_add_rcu(&client->list, &dev->smi_clients);
361 	spin_unlock(&dev->smi_lock);
362 
363 	ret = anon_inode_getfd(kfd_smi_name, &kfd_smi_ev_fops, (void *)client,
364 			       O_RDWR);
365 	if (ret < 0) {
366 		spin_lock(&dev->smi_lock);
367 		list_del_rcu(&client->list);
368 		spin_unlock(&dev->smi_lock);
369 
370 		synchronize_rcu();
371 
372 		kfifo_free(&client->fifo);
373 		kfree(client);
374 		return ret;
375 	}
376 	*fd = ret;
377 
378 	return 0;
379 }
380