165 static int expand_fdtable(struct files_struct *files, unsigned int nr)  in expand_fdtable()  argument
166 	__releases(files->file_lock)  in expand_fdtable()
167 	__acquires(files->file_lock)  in expand_fdtable()
171 	spin_unlock(&files->file_lock);  in expand_fdtable()
177 	if (atomic_read(&files->count) > 1)  in expand_fdtable()
180 	spin_lock(&files->file_lock);  in expand_fdtable()
191 	cur_fdt = files_fdtable(files);  in expand_fdtable()
194 	rcu_assign_pointer(files->fdt, new_fdt);  in expand_fdtable()
195 	if (cur_fdt != &files->fdtab)  in expand_fdtable()
210 static int expand_files(struct files_struct *files, unsigned int nr)  in expand_files()  argument
211 	__releases(files->file_lock)  in expand_files()
212 	__acquires(files->file_lock)  in expand_files()
218 	fdt = files_fdtable(files);  in expand_files()
228 	if (unlikely(files->resize_in_progress)) {  in expand_files()
229 		spin_unlock(&files->file_lock);  in expand_files()
231 		wait_event(files->resize_wait, !files->resize_in_progress);  in expand_files()
232 		spin_lock(&files->file_lock);  in expand_files()
237 	files->resize_in_progress = true;  in expand_files()
238 	expanded = expand_fdtable(files, nr);  in expand_files()
239 	files->resize_in_progress = false;  in expand_files()
241 	wake_up_all(&files->resize_wait);  in expand_files()
398 static struct fdtable *close_files(struct files_struct * files)  in close_files()  argument
405 	struct fdtable *fdt = rcu_dereference_raw(files->fdt);  in close_files()
418 					filp_close(file, files);  in close_files()
430 void put_files_struct(struct files_struct *files)  in put_files_struct()  argument
432 	if (atomic_dec_and_test(&files->count)) {  in put_files_struct()
433 		struct fdtable *fdt = close_files(files);  in put_files_struct()
436 		if (fdt != &files->fdtab)  in put_files_struct()
438 		kmem_cache_free(files_cachep, files);  in put_files_struct()
444 	struct files_struct * files = tsk->files;  in exit_files()  local
446 	if (files) {  in exit_files()
448 		tsk->files = NULL;  in exit_files()
450 		put_files_struct(files);  in exit_files()
487 	struct files_struct *files = current->files;  in alloc_fd()  local
492 	spin_lock(&files->file_lock);  in alloc_fd()
494 	fdt = files_fdtable(files);  in alloc_fd()
496 	if (fd < files->next_fd)  in alloc_fd()
497 		fd = files->next_fd;  in alloc_fd()
510 	error = expand_files(files, fd);  in alloc_fd()
521 	if (start <= files->next_fd)  in alloc_fd()
522 		files->next_fd = fd + 1;  in alloc_fd()
539 	spin_unlock(&files->file_lock);  in alloc_fd()
554 static void __put_unused_fd(struct files_struct *files, unsigned int fd)  in __put_unused_fd()  argument
556 	struct fdtable *fdt = files_fdtable(files);  in __put_unused_fd()
558 	if (fd < files->next_fd)  in __put_unused_fd()
559 		files->next_fd = fd;  in __put_unused_fd()
564 	struct files_struct *files = current->files;  in put_unused_fd()  local
565 	spin_lock(&files->file_lock);  in put_unused_fd()
566 	__put_unused_fd(files, fd);  in put_unused_fd()
567 	spin_unlock(&files->file_lock);  in put_unused_fd()
590 	struct files_struct *files = current->files;  in fd_install()  local
598 	if (unlikely(files->resize_in_progress)) {  in fd_install()
600 		spin_lock(&files->file_lock);  in fd_install()
601 		fdt = files_fdtable(files);  in fd_install()
604 		spin_unlock(&files->file_lock);  in fd_install()
609 	fdt = rcu_dereference_sched(files->fdt);  in fd_install()
628 struct file *file_close_fd_locked(struct files_struct *files, unsigned fd)  in file_close_fd_locked()  argument
630 	struct fdtable *fdt = files_fdtable(files);  in file_close_fd_locked()
633 	lockdep_assert_held(&files->file_lock);  in file_close_fd_locked()
642 		__put_unused_fd(files, fd);  in file_close_fd_locked()
649 	struct files_struct *files = current->files;  in close_fd()  local
652 	spin_lock(&files->file_lock);  in close_fd()
653 	file = file_close_fd_locked(files, fd);  in close_fd()
654 	spin_unlock(&files->file_lock);  in close_fd()
658 	return filp_close(file, files);  in close_fd()
689 static inline void __range_close(struct files_struct *files, unsigned int fd,  in __range_close()  argument
695 	spin_lock(&files->file_lock);  in __range_close()
696 	n = last_fd(files_fdtable(files));  in __range_close()
700 		file = file_close_fd_locked(files, fd);  in __range_close()
702 			spin_unlock(&files->file_lock);  in __range_close()
703 			filp_close(file, files);  in __range_close()
705 			spin_lock(&files->file_lock);  in __range_close()
707 			spin_unlock(&files->file_lock);  in __range_close()
709 			spin_lock(&files->file_lock);  in __range_close()
712 	spin_unlock(&files->file_lock);  in __range_close()
728 	struct files_struct *cur_fds = me->files, *fds = NULL;  in __close_range()
768 		me->files = cur_fds;  in __close_range()
786 	struct files_struct *files = current->files;  in file_close_fd()  local
789 	spin_lock(&files->file_lock);  in file_close_fd()
790 	file = file_close_fd_locked(files, fd);  in file_close_fd()
791 	spin_unlock(&files->file_lock);  in file_close_fd()
796 void do_close_on_exec(struct files_struct *files)  in do_close_on_exec()  argument
802 	spin_lock(&files->file_lock);  in do_close_on_exec()
806 		fdt = files_fdtable(files);  in do_close_on_exec()
821 			__put_unused_fd(files, fd);  in do_close_on_exec()
822 			spin_unlock(&files->file_lock);  in do_close_on_exec()
823 			filp_close(file, files);  in do_close_on_exec()
825 			spin_lock(&files->file_lock);  in do_close_on_exec()
829 	spin_unlock(&files->file_lock);  in do_close_on_exec()
923 static inline struct file *__fget_files_rcu(struct files_struct *files,  in __fget_files_rcu()  argument
928 		struct fdtable *fdt = rcu_dereference_raw(files->fdt);  in __fget_files_rcu()
977 		    unlikely(rcu_dereference_raw(files->fdt) != fdt)) {  in __fget_files_rcu()
999 static struct file *__fget_files(struct files_struct *files, unsigned int fd,  in __fget_files()  argument
1005 	file = __fget_files_rcu(files, fd, mask);  in __fget_files()
1013 	return __fget_files(current->files, fd, mask);  in __fget()
1033 	if (task->files)  in fget_task()
1034 		file = __fget_files(task->files, fd, 0);  in fget_task()
1042 	return __fget_files_rcu(current->files, fd, 0);  in lookup_fdget_rcu()
1050 	struct files_struct *files;  in task_lookup_fdget_rcu()  local
1054 	files = task->files;  in task_lookup_fdget_rcu()
1055 	if (files)  in task_lookup_fdget_rcu()
1056 		file = __fget_files_rcu(files, fd, 0);  in task_lookup_fdget_rcu()
1065 	struct files_struct *files;  in task_lookup_next_fdget_rcu()  local
1070 	files = task->files;  in task_lookup_next_fdget_rcu()
1071 	if (files) {  in task_lookup_next_fdget_rcu()
1072 		for (; fd < files_fdtable(files)->max_fds; fd++) {  in task_lookup_next_fdget_rcu()
1073 			file = __fget_files_rcu(files, fd, 0);  in task_lookup_next_fdget_rcu()
1102 	struct files_struct *files = current->files;  in __fget_light()  local
1114 	if (likely(atomic_read_acquire(&files->count) == 1)) {  in __fget_light()
1115 		file = files_lookup_fd_raw(files, fd);  in __fget_light()
1120 		file = __fget_files(files, fd, mask);  in __fget_light()
1178 	struct files_struct *files = current->files;  in set_close_on_exec()  local
1180 	spin_lock(&files->file_lock);  in set_close_on_exec()
1181 	fdt = files_fdtable(files);  in set_close_on_exec()
1186 	spin_unlock(&files->file_lock);  in set_close_on_exec()
1193 	res = close_on_exec(fd, current->files);  in get_close_on_exec()
1198 static int do_dup2(struct files_struct *files,  in do_dup2()  argument
1200 __releases(&files->file_lock)  in do_dup2()
1219 	fdt = files_fdtable(files);  in do_dup2()
1231 	spin_unlock(&files->file_lock);  in do_dup2()
1234 		filp_close(tofree, files);  in do_dup2()
1239 	spin_unlock(&files->file_lock);  in do_dup2()
1246 	struct files_struct *files = current->files;  in replace_fd()  local
1254 	spin_lock(&files->file_lock);  in replace_fd()
1255 	err = expand_files(files, fd);  in replace_fd()
1258 	return do_dup2(files, file, fd, flags);  in replace_fd()
1261 	spin_unlock(&files->file_lock);  in replace_fd()
1325 	struct files_struct *files = current->files;  in ksys_dup3()  local
1336 	spin_lock(&files->file_lock);  in ksys_dup3()
1337 	err = expand_files(files, newfd);  in ksys_dup3()
1338 	file = files_lookup_fd_locked(files, oldfd);  in ksys_dup3()
1346 	return do_dup2(files, file, newfd, flags);  in ksys_dup3()
1351 	spin_unlock(&files->file_lock);  in ksys_dup3()
1363 		struct files_struct *files = current->files;  in SYSCALL_DEFINE2()  local
1368 		f = __fget_files_rcu(files, oldfd, 0);  in SYSCALL_DEFINE2()
1408 int iterate_fd(struct files_struct *files, unsigned n,  in iterate_fd()  argument
1414 	if (!files)  in iterate_fd()
1416 	spin_lock(&files->file_lock);  in iterate_fd()
1417 	for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {  in iterate_fd()
1419 		file = rcu_dereference_check_fdtable(files, fdt->fd[n]);  in iterate_fd()
1426 	spin_unlock(&files->file_lock);  in iterate_fd()