xref: /linux/io_uring/register.c (revision 021bc4b9) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Code related to the io_uring_register() syscall
4  *
5  * Copyright (C) 2023 Jens Axboe
6  */
7 #include <linux/kernel.h>
8 #include <linux/errno.h>
9 #include <linux/syscalls.h>
10 #include <linux/refcount.h>
11 #include <linux/bits.h>
12 #include <linux/fs.h>
13 #include <linux/file.h>
14 #include <linux/slab.h>
15 #include <linux/uaccess.h>
16 #include <linux/nospec.h>
17 #include <linux/compat.h>
18 #include <linux/io_uring.h>
19 #include <linux/io_uring_types.h>
20 
21 #include "io_uring.h"
22 #include "opdef.h"
23 #include "tctx.h"
24 #include "rsrc.h"
25 #include "sqpoll.h"
26 #include "register.h"
27 #include "cancel.h"
28 #include "kbuf.h"
29 
30 #define IORING_MAX_RESTRICTIONS	(IORING_RESTRICTION_LAST + \
31 				 IORING_REGISTER_LAST + IORING_OP_LAST)
32 
33 static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
34 			       unsigned int eventfd_async)
35 {
36 	struct io_ev_fd *ev_fd;
37 	__s32 __user *fds = arg;
38 	int fd;
39 
40 	ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
41 					lockdep_is_held(&ctx->uring_lock));
42 	if (ev_fd)
43 		return -EBUSY;
44 
45 	if (copy_from_user(&fd, fds, sizeof(*fds)))
46 		return -EFAULT;
47 
48 	ev_fd = kmalloc(sizeof(*ev_fd), GFP_KERNEL);
49 	if (!ev_fd)
50 		return -ENOMEM;
51 
52 	ev_fd->cq_ev_fd = eventfd_ctx_fdget(fd);
53 	if (IS_ERR(ev_fd->cq_ev_fd)) {
54 		int ret = PTR_ERR(ev_fd->cq_ev_fd);
55 		kfree(ev_fd);
56 		return ret;
57 	}
58 
59 	spin_lock(&ctx->completion_lock);
60 	ctx->evfd_last_cq_tail = ctx->cached_cq_tail;
61 	spin_unlock(&ctx->completion_lock);
62 
63 	ev_fd->eventfd_async = eventfd_async;
64 	ctx->has_evfd = true;
65 	rcu_assign_pointer(ctx->io_ev_fd, ev_fd);
66 	atomic_set(&ev_fd->refs, 1);
67 	atomic_set(&ev_fd->ops, 0);
68 	return 0;
69 }
70 
71 int io_eventfd_unregister(struct io_ring_ctx *ctx)
72 {
73 	struct io_ev_fd *ev_fd;
74 
75 	ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
76 					lockdep_is_held(&ctx->uring_lock));
77 	if (ev_fd) {
78 		ctx->has_evfd = false;
79 		rcu_assign_pointer(ctx->io_ev_fd, NULL);
80 		if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_FREE_BIT), &ev_fd->ops))
81 			call_rcu(&ev_fd->rcu, io_eventfd_ops);
82 		return 0;
83 	}
84 
85 	return -ENXIO;
86 }
87 
88 static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg,
89 			   unsigned nr_args)
90 {
91 	struct io_uring_probe *p;
92 	size_t size;
93 	int i, ret;
94 
95 	size = struct_size(p, ops, nr_args);
96 	if (size == SIZE_MAX)
97 		return -EOVERFLOW;
98 	p = kzalloc(size, GFP_KERNEL);
99 	if (!p)
100 		return -ENOMEM;
101 
102 	ret = -EFAULT;
103 	if (copy_from_user(p, arg, size))
104 		goto out;
105 	ret = -EINVAL;
106 	if (memchr_inv(p, 0, size))
107 		goto out;
108 
109 	p->last_op = IORING_OP_LAST - 1;
110 	if (nr_args > IORING_OP_LAST)
111 		nr_args = IORING_OP_LAST;
112 
113 	for (i = 0; i < nr_args; i++) {
114 		p->ops[i].op = i;
115 		if (!io_issue_defs[i].not_supported)
116 			p->ops[i].flags = IO_URING_OP_SUPPORTED;
117 	}
118 	p->ops_len = i;
119 
120 	ret = 0;
121 	if (copy_to_user(arg, p, size))
122 		ret = -EFAULT;
123 out:
124 	kfree(p);
125 	return ret;
126 }
127 
128 int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
129 {
130 	const struct cred *creds;
131 
132 	creds = xa_erase(&ctx->personalities, id);
133 	if (creds) {
134 		put_cred(creds);
135 		return 0;
136 	}
137 
138 	return -EINVAL;
139 }
140 
141 
142 static int io_register_personality(struct io_ring_ctx *ctx)
143 {
144 	const struct cred *creds;
145 	u32 id;
146 	int ret;
147 
148 	creds = get_current_cred();
149 
150 	ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
151 			XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
152 	if (ret < 0) {
153 		put_cred(creds);
154 		return ret;
155 	}
156 	return id;
157 }
158 
159 static __cold int io_register_restrictions(struct io_ring_ctx *ctx,
160 					   void __user *arg, unsigned int nr_args)
161 {
162 	struct io_uring_restriction *res;
163 	size_t size;
164 	int i, ret;
165 
166 	/* Restrictions allowed only if rings started disabled */
167 	if (!(ctx->flags & IORING_SETUP_R_DISABLED))
168 		return -EBADFD;
169 
170 	/* We allow only a single restrictions registration */
171 	if (ctx->restrictions.registered)
172 		return -EBUSY;
173 
174 	if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
175 		return -EINVAL;
176 
177 	size = array_size(nr_args, sizeof(*res));
178 	if (size == SIZE_MAX)
179 		return -EOVERFLOW;
180 
181 	res = memdup_user(arg, size);
182 	if (IS_ERR(res))
183 		return PTR_ERR(res);
184 
185 	ret = 0;
186 
187 	for (i = 0; i < nr_args; i++) {
188 		switch (res[i].opcode) {
189 		case IORING_RESTRICTION_REGISTER_OP:
190 			if (res[i].register_op >= IORING_REGISTER_LAST) {
191 				ret = -EINVAL;
192 				goto out;
193 			}
194 
195 			__set_bit(res[i].register_op,
196 				  ctx->restrictions.register_op);
197 			break;
198 		case IORING_RESTRICTION_SQE_OP:
199 			if (res[i].sqe_op >= IORING_OP_LAST) {
200 				ret = -EINVAL;
201 				goto out;
202 			}
203 
204 			__set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
205 			break;
206 		case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
207 			ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
208 			break;
209 		case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
210 			ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
211 			break;
212 		default:
213 			ret = -EINVAL;
214 			goto out;
215 		}
216 	}
217 
218 out:
219 	/* Reset all restrictions if an error happened */
220 	if (ret != 0)
221 		memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
222 	else
223 		ctx->restrictions.registered = true;
224 
225 	kfree(res);
226 	return ret;
227 }
228 
229 static int io_register_enable_rings(struct io_ring_ctx *ctx)
230 {
231 	if (!(ctx->flags & IORING_SETUP_R_DISABLED))
232 		return -EBADFD;
233 
234 	if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task) {
235 		WRITE_ONCE(ctx->submitter_task, get_task_struct(current));
236 		/*
237 		 * Lazy activation attempts would fail if it was polled before
238 		 * submitter_task is set.
239 		 */
240 		if (wq_has_sleeper(&ctx->poll_wq))
241 			io_activate_pollwq(ctx);
242 	}
243 
244 	if (ctx->restrictions.registered)
245 		ctx->restricted = 1;
246 
247 	ctx->flags &= ~IORING_SETUP_R_DISABLED;
248 	if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
249 		wake_up(&ctx->sq_data->wait);
250 	return 0;
251 }
252 
253 static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx,
254 					 cpumask_var_t new_mask)
255 {
256 	int ret;
257 
258 	if (!(ctx->flags & IORING_SETUP_SQPOLL)) {
259 		ret = io_wq_cpu_affinity(current->io_uring, new_mask);
260 	} else {
261 		mutex_unlock(&ctx->uring_lock);
262 		ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask);
263 		mutex_lock(&ctx->uring_lock);
264 	}
265 
266 	return ret;
267 }
268 
269 static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx,
270 				       void __user *arg, unsigned len)
271 {
272 	cpumask_var_t new_mask;
273 	int ret;
274 
275 	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
276 		return -ENOMEM;
277 
278 	cpumask_clear(new_mask);
279 	if (len > cpumask_size())
280 		len = cpumask_size();
281 
282 #ifdef CONFIG_COMPAT
283 	if (in_compat_syscall())
284 		ret = compat_get_bitmap(cpumask_bits(new_mask),
285 					(const compat_ulong_t __user *)arg,
286 					len * 8 /* CHAR_BIT */);
287 	else
288 #endif
289 		ret = copy_from_user(new_mask, arg, len);
290 
291 	if (ret) {
292 		free_cpumask_var(new_mask);
293 		return -EFAULT;
294 	}
295 
296 	ret = __io_register_iowq_aff(ctx, new_mask);
297 	free_cpumask_var(new_mask);
298 	return ret;
299 }
300 
301 static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
302 {
303 	return __io_register_iowq_aff(ctx, NULL);
304 }
305 
306 static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
307 					       void __user *arg)
308 	__must_hold(&ctx->uring_lock)
309 {
310 	struct io_tctx_node *node;
311 	struct io_uring_task *tctx = NULL;
312 	struct io_sq_data *sqd = NULL;
313 	__u32 new_count[2];
314 	int i, ret;
315 
316 	if (copy_from_user(new_count, arg, sizeof(new_count)))
317 		return -EFAULT;
318 	for (i = 0; i < ARRAY_SIZE(new_count); i++)
319 		if (new_count[i] > INT_MAX)
320 			return -EINVAL;
321 
322 	if (ctx->flags & IORING_SETUP_SQPOLL) {
323 		sqd = ctx->sq_data;
324 		if (sqd) {
325 			/*
326 			 * Observe the correct sqd->lock -> ctx->uring_lock
327 			 * ordering. Fine to drop uring_lock here, we hold
328 			 * a ref to the ctx.
329 			 */
330 			refcount_inc(&sqd->refs);
331 			mutex_unlock(&ctx->uring_lock);
332 			mutex_lock(&sqd->lock);
333 			mutex_lock(&ctx->uring_lock);
334 			if (sqd->thread)
335 				tctx = sqd->thread->io_uring;
336 		}
337 	} else {
338 		tctx = current->io_uring;
339 	}
340 
341 	BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits));
342 
343 	for (i = 0; i < ARRAY_SIZE(new_count); i++)
344 		if (new_count[i])
345 			ctx->iowq_limits[i] = new_count[i];
346 	ctx->iowq_limits_set = true;
347 
348 	if (tctx && tctx->io_wq) {
349 		ret = io_wq_max_workers(tctx->io_wq, new_count);
350 		if (ret)
351 			goto err;
352 	} else {
353 		memset(new_count, 0, sizeof(new_count));
354 	}
355 
356 	if (sqd) {
357 		mutex_unlock(&sqd->lock);
358 		io_put_sq_data(sqd);
359 	}
360 
361 	if (copy_to_user(arg, new_count, sizeof(new_count)))
362 		return -EFAULT;
363 
364 	/* that's it for SQPOLL, only the SQPOLL task creates requests */
365 	if (sqd)
366 		return 0;
367 
368 	/* now propagate the restriction to all registered users */
369 	list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
370 		struct io_uring_task *tctx = node->task->io_uring;
371 
372 		if (WARN_ON_ONCE(!tctx->io_wq))
373 			continue;
374 
375 		for (i = 0; i < ARRAY_SIZE(new_count); i++)
376 			new_count[i] = ctx->iowq_limits[i];
377 		/* ignore errors, it always returns zero anyway */
378 		(void)io_wq_max_workers(tctx->io_wq, new_count);
379 	}
380 	return 0;
381 err:
382 	if (sqd) {
383 		mutex_unlock(&sqd->lock);
384 		io_put_sq_data(sqd);
385 	}
386 	return ret;
387 }
388 
389 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
390 			       void __user *arg, unsigned nr_args)
391 	__releases(ctx->uring_lock)
392 	__acquires(ctx->uring_lock)
393 {
394 	int ret;
395 
396 	/*
397 	 * We don't quiesce the refs for register anymore and so it can't be
398 	 * dying as we're holding a file ref here.
399 	 */
400 	if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs)))
401 		return -ENXIO;
402 
403 	if (ctx->submitter_task && ctx->submitter_task != current)
404 		return -EEXIST;
405 
406 	if (ctx->restricted) {
407 		opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
408 		if (!test_bit(opcode, ctx->restrictions.register_op))
409 			return -EACCES;
410 	}
411 
412 	switch (opcode) {
413 	case IORING_REGISTER_BUFFERS:
414 		ret = -EFAULT;
415 		if (!arg)
416 			break;
417 		ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
418 		break;
419 	case IORING_UNREGISTER_BUFFERS:
420 		ret = -EINVAL;
421 		if (arg || nr_args)
422 			break;
423 		ret = io_sqe_buffers_unregister(ctx);
424 		break;
425 	case IORING_REGISTER_FILES:
426 		ret = -EFAULT;
427 		if (!arg)
428 			break;
429 		ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
430 		break;
431 	case IORING_UNREGISTER_FILES:
432 		ret = -EINVAL;
433 		if (arg || nr_args)
434 			break;
435 		ret = io_sqe_files_unregister(ctx);
436 		break;
437 	case IORING_REGISTER_FILES_UPDATE:
438 		ret = io_register_files_update(ctx, arg, nr_args);
439 		break;
440 	case IORING_REGISTER_EVENTFD:
441 		ret = -EINVAL;
442 		if (nr_args != 1)
443 			break;
444 		ret = io_eventfd_register(ctx, arg, 0);
445 		break;
446 	case IORING_REGISTER_EVENTFD_ASYNC:
447 		ret = -EINVAL;
448 		if (nr_args != 1)
449 			break;
450 		ret = io_eventfd_register(ctx, arg, 1);
451 		break;
452 	case IORING_UNREGISTER_EVENTFD:
453 		ret = -EINVAL;
454 		if (arg || nr_args)
455 			break;
456 		ret = io_eventfd_unregister(ctx);
457 		break;
458 	case IORING_REGISTER_PROBE:
459 		ret = -EINVAL;
460 		if (!arg || nr_args > 256)
461 			break;
462 		ret = io_probe(ctx, arg, nr_args);
463 		break;
464 	case IORING_REGISTER_PERSONALITY:
465 		ret = -EINVAL;
466 		if (arg || nr_args)
467 			break;
468 		ret = io_register_personality(ctx);
469 		break;
470 	case IORING_UNREGISTER_PERSONALITY:
471 		ret = -EINVAL;
472 		if (arg)
473 			break;
474 		ret = io_unregister_personality(ctx, nr_args);
475 		break;
476 	case IORING_REGISTER_ENABLE_RINGS:
477 		ret = -EINVAL;
478 		if (arg || nr_args)
479 			break;
480 		ret = io_register_enable_rings(ctx);
481 		break;
482 	case IORING_REGISTER_RESTRICTIONS:
483 		ret = io_register_restrictions(ctx, arg, nr_args);
484 		break;
485 	case IORING_REGISTER_FILES2:
486 		ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
487 		break;
488 	case IORING_REGISTER_FILES_UPDATE2:
489 		ret = io_register_rsrc_update(ctx, arg, nr_args,
490 					      IORING_RSRC_FILE);
491 		break;
492 	case IORING_REGISTER_BUFFERS2:
493 		ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
494 		break;
495 	case IORING_REGISTER_BUFFERS_UPDATE:
496 		ret = io_register_rsrc_update(ctx, arg, nr_args,
497 					      IORING_RSRC_BUFFER);
498 		break;
499 	case IORING_REGISTER_IOWQ_AFF:
500 		ret = -EINVAL;
501 		if (!arg || !nr_args)
502 			break;
503 		ret = io_register_iowq_aff(ctx, arg, nr_args);
504 		break;
505 	case IORING_UNREGISTER_IOWQ_AFF:
506 		ret = -EINVAL;
507 		if (arg || nr_args)
508 			break;
509 		ret = io_unregister_iowq_aff(ctx);
510 		break;
511 	case IORING_REGISTER_IOWQ_MAX_WORKERS:
512 		ret = -EINVAL;
513 		if (!arg || nr_args != 2)
514 			break;
515 		ret = io_register_iowq_max_workers(ctx, arg);
516 		break;
517 	case IORING_REGISTER_RING_FDS:
518 		ret = io_ringfd_register(ctx, arg, nr_args);
519 		break;
520 	case IORING_UNREGISTER_RING_FDS:
521 		ret = io_ringfd_unregister(ctx, arg, nr_args);
522 		break;
523 	case IORING_REGISTER_PBUF_RING:
524 		ret = -EINVAL;
525 		if (!arg || nr_args != 1)
526 			break;
527 		ret = io_register_pbuf_ring(ctx, arg);
528 		break;
529 	case IORING_UNREGISTER_PBUF_RING:
530 		ret = -EINVAL;
531 		if (!arg || nr_args != 1)
532 			break;
533 		ret = io_unregister_pbuf_ring(ctx, arg);
534 		break;
535 	case IORING_REGISTER_SYNC_CANCEL:
536 		ret = -EINVAL;
537 		if (!arg || nr_args != 1)
538 			break;
539 		ret = io_sync_cancel(ctx, arg);
540 		break;
541 	case IORING_REGISTER_FILE_ALLOC_RANGE:
542 		ret = -EINVAL;
543 		if (!arg || nr_args)
544 			break;
545 		ret = io_register_file_alloc_range(ctx, arg);
546 		break;
547 	case IORING_REGISTER_PBUF_STATUS:
548 		ret = -EINVAL;
549 		if (!arg || nr_args != 1)
550 			break;
551 		ret = io_register_pbuf_status(ctx, arg);
552 		break;
553 	default:
554 		ret = -EINVAL;
555 		break;
556 	}
557 
558 	return ret;
559 }
560 
561 SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
562 		void __user *, arg, unsigned int, nr_args)
563 {
564 	struct io_ring_ctx *ctx;
565 	long ret = -EBADF;
566 	struct file *file;
567 	bool use_registered_ring;
568 
569 	use_registered_ring = !!(opcode & IORING_REGISTER_USE_REGISTERED_RING);
570 	opcode &= ~IORING_REGISTER_USE_REGISTERED_RING;
571 
572 	if (opcode >= IORING_REGISTER_LAST)
573 		return -EINVAL;
574 
575 	if (use_registered_ring) {
576 		/*
577 		 * Ring fd has been registered via IORING_REGISTER_RING_FDS, we
578 		 * need only dereference our task private array to find it.
579 		 */
580 		struct io_uring_task *tctx = current->io_uring;
581 
582 		if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX))
583 			return -EINVAL;
584 		fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);
585 		file = tctx->registered_rings[fd];
586 		if (unlikely(!file))
587 			return -EBADF;
588 	} else {
589 		file = fget(fd);
590 		if (unlikely(!file))
591 			return -EBADF;
592 		ret = -EOPNOTSUPP;
593 		if (!io_is_uring_fops(file))
594 			goto out_fput;
595 	}
596 
597 	ctx = file->private_data;
598 
599 	mutex_lock(&ctx->uring_lock);
600 	ret = __io_uring_register(ctx, opcode, arg, nr_args);
601 	mutex_unlock(&ctx->uring_lock);
602 	trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ret);
603 out_fput:
604 	if (!use_registered_ring)
605 		fput(file);
606 	return ret;
607 }
608