11e3db1deSHans Petter Selasky /*- 21e3db1deSHans Petter Selasky * Copyright (c) 2017 Hans Petter Selasky 31e3db1deSHans Petter Selasky * All rights reserved. 41e3db1deSHans Petter Selasky * 51e3db1deSHans Petter Selasky * Redistribution and use in source and binary forms, with or without 61e3db1deSHans Petter Selasky * modification, are permitted provided that the following conditions 71e3db1deSHans Petter Selasky * are met: 81e3db1deSHans Petter Selasky * 1. Redistributions of source code must retain the above copyright 91e3db1deSHans Petter Selasky * notice unmodified, this list of conditions, and the following 101e3db1deSHans Petter Selasky * disclaimer. 111e3db1deSHans Petter Selasky * 2. Redistributions in binary form must reproduce the above copyright 121e3db1deSHans Petter Selasky * notice, this list of conditions and the following disclaimer in the 131e3db1deSHans Petter Selasky * documentation and/or other materials provided with the distribution. 141e3db1deSHans Petter Selasky * 151e3db1deSHans Petter Selasky * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 161e3db1deSHans Petter Selasky * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 171e3db1deSHans Petter Selasky * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 181e3db1deSHans Petter Selasky * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 191e3db1deSHans Petter Selasky * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 201e3db1deSHans Petter Selasky * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 211e3db1deSHans Petter Selasky * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 221e3db1deSHans Petter Selasky * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 231e3db1deSHans Petter Selasky * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 241e3db1deSHans Petter Selasky * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 251e3db1deSHans Petter Selasky */ 261e3db1deSHans Petter Selasky 271e3db1deSHans Petter Selasky #include <sys/cdefs.h> 281e3db1deSHans Petter Selasky __FBSDID("$FreeBSD$"); 291e3db1deSHans Petter Selasky 301e3db1deSHans Petter Selasky #include <linux/compat.h> 318504aa98SMark Johnston #include <linux/completion.h> 321e3db1deSHans Petter Selasky #include <linux/mm.h> 331e3db1deSHans Petter Selasky #include <linux/kthread.h> 34165ba13fSKonstantin Belousov #include <linux/moduleparam.h> 351e3db1deSHans Petter Selasky 361e3db1deSHans Petter Selasky #include <sys/kernel.h> 371e3db1deSHans Petter Selasky #include <sys/eventhandler.h> 381e3db1deSHans Petter Selasky #include <sys/malloc.h> 39165ba13fSKonstantin Belousov #include <sys/sysctl.h> 40165ba13fSKonstantin Belousov #include <vm/uma.h> 41165ba13fSKonstantin Belousov 42165ba13fSKonstantin Belousov #if defined(__i386__) || defined(__amd64__) 43165ba13fSKonstantin Belousov extern u_int first_msi_irq, num_msi_irqs; 44165ba13fSKonstantin Belousov #endif 451e3db1deSHans Petter Selasky 461e3db1deSHans Petter Selasky static eventhandler_tag linuxkpi_thread_dtor_tag; 471e3db1deSHans Petter Selasky 48165ba13fSKonstantin Belousov static uma_zone_t linux_current_zone; 49165ba13fSKonstantin Belousov static uma_zone_t linux_mm_zone; 501e3db1deSHans Petter Selasky 51fad437baSKonstantin Belousov /* check if another thread already has a mm_struct */ 52fad437baSKonstantin Belousov static struct mm_struct * 53fad437baSKonstantin Belousov find_other_mm(struct proc *p) 54fad437baSKonstantin Belousov { 55fad437baSKonstantin Belousov struct thread *td; 56fad437baSKonstantin Belousov struct task_struct *ts; 57fad437baSKonstantin Belousov struct mm_struct *mm; 58fad437baSKonstantin Belousov 59fad437baSKonstantin Belousov PROC_LOCK_ASSERT(p, MA_OWNED); 60fad437baSKonstantin Belousov FOREACH_THREAD_IN_PROC(p, td) { 61fad437baSKonstantin Belousov ts = td->td_lkpi_task; 62fad437baSKonstantin Belousov if (ts == NULL) 63fad437baSKonstantin Belousov continue; 64fad437baSKonstantin Belousov mm = ts->mm; 65fad437baSKonstantin Belousov if (mm == NULL) 66fad437baSKonstantin Belousov continue; 67fad437baSKonstantin Belousov /* try to share other mm_struct */ 68fad437baSKonstantin Belousov if (atomic_inc_not_zero(&mm->mm_users)) 69fad437baSKonstantin Belousov return (mm); 70fad437baSKonstantin Belousov } 71fad437baSKonstantin Belousov return (NULL); 72fad437baSKonstantin Belousov } 73fad437baSKonstantin Belousov 741e3db1deSHans Petter Selasky int 751e3db1deSHans Petter Selasky linux_alloc_current(struct thread *td, int flags) 761e3db1deSHans Petter Selasky { 77e54b103eSHans Petter Selasky struct proc *proc; 781e3db1deSHans Petter Selasky struct task_struct *ts; 79fad437baSKonstantin Belousov struct mm_struct *mm, *mm_other; 801e3db1deSHans Petter Selasky 811e3db1deSHans Petter Selasky MPASS(td->td_lkpi_task == NULL); 821e3db1deSHans Petter Selasky 83165ba13fSKonstantin Belousov if ((td->td_pflags & TDP_ITHREAD) != 0 || !THREAD_CAN_SLEEP()) { 84165ba13fSKonstantin Belousov flags &= ~M_WAITOK; 85165ba13fSKonstantin Belousov flags |= M_NOWAIT | M_USE_RESERVE; 86165ba13fSKonstantin Belousov } 871e3db1deSHans Petter Selasky 88165ba13fSKonstantin Belousov ts = uma_zalloc(linux_current_zone, flags | M_ZERO); 89165ba13fSKonstantin Belousov if (ts == NULL) { 90165ba13fSKonstantin Belousov if ((flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK) 91165ba13fSKonstantin Belousov panic("linux_alloc_current: failed to allocate task"); 92165ba13fSKonstantin Belousov return (ENOMEM); 93165ba13fSKonstantin Belousov } 94fad437baSKonstantin Belousov mm = NULL; 9505d4f501SHans Petter Selasky 96e54b103eSHans Petter Selasky /* setup new task structure */ 971e3db1deSHans Petter Selasky atomic_set(&ts->kthread_flags, 0); 981e3db1deSHans Petter Selasky ts->task_thread = td; 991e3db1deSHans Petter Selasky ts->comm = td->td_name; 1001e3db1deSHans Petter Selasky ts->pid = td->td_tid; 10107e0a3caSJohannes Lundberg ts->group_leader = ts; 102a0699ebfSHans Petter Selasky atomic_set(&ts->usage, 1); 103ef925749SHans Petter Selasky atomic_set(&ts->state, TASK_RUNNING); 1048504aa98SMark Johnston init_completion(&ts->parked); 1058504aa98SMark Johnston init_completion(&ts->exited); 10605d4f501SHans Petter Selasky 107e54b103eSHans Petter Selasky proc = td->td_proc; 108e54b103eSHans Petter Selasky 109e54b103eSHans Petter Selasky PROC_LOCK(proc); 110fad437baSKonstantin Belousov mm_other = find_other_mm(proc); 111e54b103eSHans Petter Selasky 112e54b103eSHans Petter Selasky /* use allocated mm_struct as a fallback */ 113fad437baSKonstantin Belousov if (mm_other == NULL) { 114fad437baSKonstantin Belousov PROC_UNLOCK(proc); 115fad437baSKonstantin Belousov mm = uma_zalloc(linux_mm_zone, flags | M_ZERO); 116fad437baSKonstantin Belousov if (mm == NULL) { 117fad437baSKonstantin Belousov if ((flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK) 118fad437baSKonstantin Belousov panic( 119fad437baSKonstantin Belousov "linux_alloc_current: failed to allocate mm"); 120fad437baSKonstantin Belousov uma_zfree(linux_current_zone, mm); 121fad437baSKonstantin Belousov return (ENOMEM); 122fad437baSKonstantin Belousov } 123fad437baSKonstantin Belousov 124fad437baSKonstantin Belousov PROC_LOCK(proc); 125fad437baSKonstantin Belousov mm_other = find_other_mm(proc); 126fad437baSKonstantin Belousov if (mm_other == NULL) { 127e54b103eSHans Petter Selasky /* setup new mm_struct */ 12805d4f501SHans Petter Selasky init_rwsem(&mm->mmap_sem); 12905d4f501SHans Petter Selasky atomic_set(&mm->mm_count, 1); 13005d4f501SHans Petter Selasky atomic_set(&mm->mm_users, 1); 131e54b103eSHans Petter Selasky /* set mm_struct pointer */ 132e54b103eSHans Petter Selasky ts->mm = mm; 133e54b103eSHans Petter Selasky /* clear pointer to not free memory */ 134e54b103eSHans Petter Selasky mm = NULL; 135fad437baSKonstantin Belousov } else { 136fad437baSKonstantin Belousov ts->mm = mm_other; 137fad437baSKonstantin Belousov } 138fad437baSKonstantin Belousov } else { 139fad437baSKonstantin Belousov ts->mm = mm_other; 140e54b103eSHans Petter Selasky } 14105d4f501SHans Petter Selasky 14205d4f501SHans Petter Selasky /* store pointer to task struct */ 1431e3db1deSHans Petter Selasky td->td_lkpi_task = ts; 144e54b103eSHans Petter Selasky PROC_UNLOCK(proc); 145e54b103eSHans Petter Selasky 146e54b103eSHans Petter Selasky /* free mm_struct pointer, if any */ 147165ba13fSKonstantin Belousov uma_zfree(linux_mm_zone, mm); 148e54b103eSHans Petter Selasky 1491e3db1deSHans Petter Selasky return (0); 1501e3db1deSHans Petter Selasky } 1511e3db1deSHans Petter Selasky 15205d4f501SHans Petter Selasky struct mm_struct * 15305d4f501SHans Petter Selasky linux_get_task_mm(struct task_struct *task) 15405d4f501SHans Petter Selasky { 15505d4f501SHans Petter Selasky struct mm_struct *mm; 15605d4f501SHans Petter Selasky 15705d4f501SHans Petter Selasky mm = task->mm; 15868b9f2f0SHans Petter Selasky if (mm != NULL) { 15905d4f501SHans Petter Selasky atomic_inc(&mm->mm_users); 16005d4f501SHans Petter Selasky return (mm); 16105d4f501SHans Petter Selasky } 16205d4f501SHans Petter Selasky return (NULL); 16305d4f501SHans Petter Selasky } 16405d4f501SHans Petter Selasky 16505d4f501SHans Petter Selasky void 16605d4f501SHans Petter Selasky linux_mm_dtor(struct mm_struct *mm) 16705d4f501SHans Petter Selasky { 168165ba13fSKonstantin Belousov uma_zfree(linux_mm_zone, mm); 16905d4f501SHans Petter Selasky } 17005d4f501SHans Petter Selasky 1711e3db1deSHans Petter Selasky void 1721e3db1deSHans Petter Selasky linux_free_current(struct task_struct *ts) 1731e3db1deSHans Petter Selasky { 17405d4f501SHans Petter Selasky mmput(ts->mm); 175165ba13fSKonstantin Belousov uma_zfree(linux_current_zone, ts); 1761e3db1deSHans Petter Selasky } 1771e3db1deSHans Petter Selasky 1781e3db1deSHans Petter Selasky static void 1791e3db1deSHans Petter Selasky linuxkpi_thread_dtor(void *arg __unused, struct thread *td) 1801e3db1deSHans Petter Selasky { 1811e3db1deSHans Petter Selasky struct task_struct *ts; 1821e3db1deSHans Petter Selasky 1831e3db1deSHans Petter Selasky ts = td->td_lkpi_task; 1841e3db1deSHans Petter Selasky if (ts == NULL) 1851e3db1deSHans Petter Selasky return; 1861e3db1deSHans Petter Selasky 1871e3db1deSHans Petter Selasky td->td_lkpi_task = NULL; 188a0699ebfSHans Petter Selasky put_task_struct(ts); 189a0699ebfSHans Petter Selasky } 190a0699ebfSHans Petter Selasky 191f334f212SKonstantin Belousov static struct task_struct * 192f334f212SKonstantin Belousov linux_get_pid_task_int(pid_t pid, const bool do_get) 193a0699ebfSHans Petter Selasky { 194a0699ebfSHans Petter Selasky struct thread *td; 1958402f058SHans Petter Selasky struct proc *p; 196f334f212SKonstantin Belousov struct task_struct *ts; 197a0699ebfSHans Petter Selasky 198f334f212SKonstantin Belousov if (pid > PID_MAX) { 1998402f058SHans Petter Selasky /* try to find corresponding thread */ 200a0699ebfSHans Petter Selasky td = tdfind(pid, -1); 201a0699ebfSHans Petter Selasky if (td != NULL) { 202f334f212SKonstantin Belousov ts = td->td_lkpi_task; 203f334f212SKonstantin Belousov if (do_get && ts != NULL) 204f334f212SKonstantin Belousov get_task_struct(ts); 205a0699ebfSHans Petter Selasky PROC_UNLOCK(td->td_proc); 206a0699ebfSHans Petter Selasky return (ts); 207a0699ebfSHans Petter Selasky } 208f334f212SKonstantin Belousov } else { 2098402f058SHans Petter Selasky /* try to find corresponding procedure */ 2108402f058SHans Petter Selasky p = pfind(pid); 2118402f058SHans Petter Selasky if (p != NULL) { 2128402f058SHans Petter Selasky FOREACH_THREAD_IN_PROC(p, td) { 213f334f212SKonstantin Belousov ts = td->td_lkpi_task; 2148402f058SHans Petter Selasky if (ts != NULL) { 215f334f212SKonstantin Belousov if (do_get) 216f334f212SKonstantin Belousov get_task_struct(ts); 2178402f058SHans Petter Selasky PROC_UNLOCK(p); 2188402f058SHans Petter Selasky return (ts); 2198402f058SHans Petter Selasky } 2208402f058SHans Petter Selasky } 2218402f058SHans Petter Selasky PROC_UNLOCK(p); 2228402f058SHans Petter Selasky } 223f334f212SKonstantin Belousov } 224a0699ebfSHans Petter Selasky return (NULL); 225a0699ebfSHans Petter Selasky } 226a0699ebfSHans Petter Selasky 227a0699ebfSHans Petter Selasky struct task_struct * 228f334f212SKonstantin Belousov linux_pid_task(pid_t pid) 229f334f212SKonstantin Belousov { 230f334f212SKonstantin Belousov return (linux_get_pid_task_int(pid, false)); 231f334f212SKonstantin Belousov } 232f334f212SKonstantin Belousov 233f334f212SKonstantin Belousov struct task_struct * 234a0699ebfSHans Petter Selasky linux_get_pid_task(pid_t pid) 235a0699ebfSHans Petter Selasky { 236f334f212SKonstantin Belousov return (linux_get_pid_task_int(pid, true)); 2371e3db1deSHans Petter Selasky } 2381e3db1deSHans Petter Selasky 239638fa5a3SHans Petter Selasky bool 240638fa5a3SHans Petter Selasky linux_task_exiting(struct task_struct *task) 241638fa5a3SHans Petter Selasky { 2429a4e535bSHans Petter Selasky struct thread *td; 243638fa5a3SHans Petter Selasky struct proc *p; 244638fa5a3SHans Petter Selasky bool ret; 245638fa5a3SHans Petter Selasky 246638fa5a3SHans Petter Selasky ret = false; 2479a4e535bSHans Petter Selasky 2489a4e535bSHans Petter Selasky /* try to find corresponding thread */ 2499a4e535bSHans Petter Selasky td = tdfind(task->pid, -1); 2509a4e535bSHans Petter Selasky if (td != NULL) { 2519a4e535bSHans Petter Selasky p = td->td_proc; 2529a4e535bSHans Petter Selasky } else { 2539a4e535bSHans Petter Selasky /* try to find corresponding procedure */ 254638fa5a3SHans Petter Selasky p = pfind(task->pid); 2559a4e535bSHans Petter Selasky } 2569a4e535bSHans Petter Selasky 257638fa5a3SHans Petter Selasky if (p != NULL) { 258638fa5a3SHans Petter Selasky if ((p->p_flag & P_WEXIT) != 0) 259638fa5a3SHans Petter Selasky ret = true; 260638fa5a3SHans Petter Selasky PROC_UNLOCK(p); 261638fa5a3SHans Petter Selasky } 262638fa5a3SHans Petter Selasky return (ret); 263638fa5a3SHans Petter Selasky } 264638fa5a3SHans Petter Selasky 265165ba13fSKonstantin Belousov static int lkpi_task_resrv; 266165ba13fSKonstantin Belousov SYSCTL_INT(_compat_linuxkpi, OID_AUTO, task_struct_reserve, 267165ba13fSKonstantin Belousov CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &lkpi_task_resrv, 0, 268165ba13fSKonstantin Belousov "Number of struct task and struct mm to reserve for non-sleepable " 269165ba13fSKonstantin Belousov "allocations"); 270165ba13fSKonstantin Belousov 2711e3db1deSHans Petter Selasky static void 2721e3db1deSHans Petter Selasky linux_current_init(void *arg __unused) 2731e3db1deSHans Petter Selasky { 274983ed4f9SMatt Macy lkpi_alloc_current = linux_alloc_current; 2751e3db1deSHans Petter Selasky linuxkpi_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 2761e3db1deSHans Petter Selasky linuxkpi_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); 277165ba13fSKonstantin Belousov 278165ba13fSKonstantin Belousov TUNABLE_INT_FETCH("compat.linuxkpi.task_struct_reserve", 279165ba13fSKonstantin Belousov &lkpi_task_resrv); 280165ba13fSKonstantin Belousov if (lkpi_task_resrv == 0) { 281165ba13fSKonstantin Belousov #if defined(__i386__) || defined(__amd64__) 282165ba13fSKonstantin Belousov /* 283165ba13fSKonstantin Belousov * Number of interrupt threads plus per-cpu callout 284165ba13fSKonstantin Belousov * SWI threads. 285165ba13fSKonstantin Belousov */ 286165ba13fSKonstantin Belousov lkpi_task_resrv = first_msi_irq + num_msi_irqs + MAXCPU; 287165ba13fSKonstantin Belousov #else 288165ba13fSKonstantin Belousov lkpi_task_resrv = 1024; /* XXXKIB arbitrary */ 289165ba13fSKonstantin Belousov #endif 290165ba13fSKonstantin Belousov } 291165ba13fSKonstantin Belousov linux_current_zone = uma_zcreate("lkpicurr", 292165ba13fSKonstantin Belousov sizeof(struct task_struct), NULL, NULL, NULL, NULL, 293165ba13fSKonstantin Belousov UMA_ALIGN_PTR, 0); 294165ba13fSKonstantin Belousov uma_zone_reserve(linux_current_zone, lkpi_task_resrv); 295165ba13fSKonstantin Belousov uma_prealloc(linux_current_zone, lkpi_task_resrv); 296165ba13fSKonstantin Belousov linux_mm_zone = uma_zcreate("lkpimm", 297165ba13fSKonstantin Belousov sizeof(struct task_struct), NULL, NULL, NULL, NULL, 298165ba13fSKonstantin Belousov UMA_ALIGN_PTR, 0); 299165ba13fSKonstantin Belousov uma_zone_reserve(linux_mm_zone, lkpi_task_resrv); 300165ba13fSKonstantin Belousov uma_prealloc(linux_mm_zone, lkpi_task_resrv); 3011e3db1deSHans Petter Selasky } 3024ce1f616SKonstantin Belousov SYSINIT(linux_current, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, 3034ce1f616SKonstantin Belousov linux_current_init, NULL); 3041e3db1deSHans Petter Selasky 3051e3db1deSHans Petter Selasky static void 3061e3db1deSHans Petter Selasky linux_current_uninit(void *arg __unused) 3071e3db1deSHans Petter Selasky { 308abf5c031SMark Johnston struct proc *p; 309abf5c031SMark Johnston struct task_struct *ts; 310abf5c031SMark Johnston struct thread *td; 311abf5c031SMark Johnston 312abf5c031SMark Johnston sx_slock(&allproc_lock); 313abf5c031SMark Johnston FOREACH_PROC_IN_SYSTEM(p) { 314abf5c031SMark Johnston PROC_LOCK(p); 315abf5c031SMark Johnston FOREACH_THREAD_IN_PROC(p, td) { 316abf5c031SMark Johnston if ((ts = td->td_lkpi_task) != NULL) { 317abf5c031SMark Johnston td->td_lkpi_task = NULL; 318abf5c031SMark Johnston put_task_struct(ts); 319abf5c031SMark Johnston } 320abf5c031SMark Johnston } 321abf5c031SMark Johnston PROC_UNLOCK(p); 322abf5c031SMark Johnston } 323abf5c031SMark Johnston sx_sunlock(&allproc_lock); 3241e3db1deSHans Petter Selasky EVENTHANDLER_DEREGISTER(thread_dtor, linuxkpi_thread_dtor_tag); 325983ed4f9SMatt Macy lkpi_alloc_current = linux_alloc_current_noop; 326165ba13fSKonstantin Belousov uma_zdestroy(linux_current_zone); 327165ba13fSKonstantin Belousov uma_zdestroy(linux_mm_zone); 3281e3db1deSHans Petter Selasky } 3294ce1f616SKonstantin Belousov SYSUNINIT(linux_current, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, 3304ce1f616SKonstantin Belousov linux_current_uninit, NULL); 331