11e3db1deSHans Petter Selasky /*-
21e3db1deSHans Petter Selasky * Copyright (c) 2017 Hans Petter Selasky
31e3db1deSHans Petter Selasky * All rights reserved.
41e3db1deSHans Petter Selasky *
51e3db1deSHans Petter Selasky * Redistribution and use in source and binary forms, with or without
61e3db1deSHans Petter Selasky * modification, are permitted provided that the following conditions
71e3db1deSHans Petter Selasky * are met:
81e3db1deSHans Petter Selasky * 1. Redistributions of source code must retain the above copyright
91e3db1deSHans Petter Selasky * notice unmodified, this list of conditions, and the following
101e3db1deSHans Petter Selasky * disclaimer.
111e3db1deSHans Petter Selasky * 2. Redistributions in binary form must reproduce the above copyright
121e3db1deSHans Petter Selasky * notice, this list of conditions and the following disclaimer in the
131e3db1deSHans Petter Selasky * documentation and/or other materials provided with the distribution.
141e3db1deSHans Petter Selasky *
151e3db1deSHans Petter Selasky * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
161e3db1deSHans Petter Selasky * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
171e3db1deSHans Petter Selasky * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
181e3db1deSHans Petter Selasky * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
191e3db1deSHans Petter Selasky * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
201e3db1deSHans Petter Selasky * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
211e3db1deSHans Petter Selasky * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
221e3db1deSHans Petter Selasky * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
231e3db1deSHans Petter Selasky * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
241e3db1deSHans Petter Selasky * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
251e3db1deSHans Petter Selasky */
261e3db1deSHans Petter Selasky
271e3db1deSHans Petter Selasky #include <sys/cdefs.h>
28fdc18947STijl Coosemans #ifdef __amd64__
29fdc18947STijl Coosemans #define DEV_APIC
30fdc18947STijl Coosemans #elif defined(__i386__)
31fdc18947STijl Coosemans #include "opt_apic.h"
32fdc18947STijl Coosemans #endif
33fdc18947STijl Coosemans
341e3db1deSHans Petter Selasky #include <linux/compat.h>
358504aa98SMark Johnston #include <linux/completion.h>
361e3db1deSHans Petter Selasky #include <linux/mm.h>
371e3db1deSHans Petter Selasky #include <linux/kthread.h>
38165ba13fSKonstantin Belousov #include <linux/moduleparam.h>
391e3db1deSHans Petter Selasky
401e3db1deSHans Petter Selasky #include <sys/kernel.h>
411e3db1deSHans Petter Selasky #include <sys/eventhandler.h>
421e3db1deSHans Petter Selasky #include <sys/malloc.h>
43165ba13fSKonstantin Belousov #include <sys/sysctl.h>
44165ba13fSKonstantin Belousov #include <vm/uma.h>
45165ba13fSKonstantin Belousov
46fdc18947STijl Coosemans #ifdef DEV_APIC
47165ba13fSKonstantin Belousov extern u_int first_msi_irq, num_msi_irqs;
48165ba13fSKonstantin Belousov #endif
491e3db1deSHans Petter Selasky
501e3db1deSHans Petter Selasky static eventhandler_tag linuxkpi_thread_dtor_tag;
511e3db1deSHans Petter Selasky
52165ba13fSKonstantin Belousov static uma_zone_t linux_current_zone;
53165ba13fSKonstantin Belousov static uma_zone_t linux_mm_zone;
541e3db1deSHans Petter Selasky
55fad437baSKonstantin Belousov /* check if another thread already has a mm_struct */
56fad437baSKonstantin Belousov static struct mm_struct *
find_other_mm(struct proc * p)57fad437baSKonstantin Belousov find_other_mm(struct proc *p)
58fad437baSKonstantin Belousov {
59fad437baSKonstantin Belousov struct thread *td;
60fad437baSKonstantin Belousov struct task_struct *ts;
61fad437baSKonstantin Belousov struct mm_struct *mm;
62fad437baSKonstantin Belousov
63fad437baSKonstantin Belousov PROC_LOCK_ASSERT(p, MA_OWNED);
64fad437baSKonstantin Belousov FOREACH_THREAD_IN_PROC(p, td) {
65fad437baSKonstantin Belousov ts = td->td_lkpi_task;
66fad437baSKonstantin Belousov if (ts == NULL)
67fad437baSKonstantin Belousov continue;
68fad437baSKonstantin Belousov mm = ts->mm;
69fad437baSKonstantin Belousov if (mm == NULL)
70fad437baSKonstantin Belousov continue;
71fad437baSKonstantin Belousov /* try to share other mm_struct */
72fad437baSKonstantin Belousov if (atomic_inc_not_zero(&mm->mm_users))
73fad437baSKonstantin Belousov return (mm);
74fad437baSKonstantin Belousov }
75fad437baSKonstantin Belousov return (NULL);
76fad437baSKonstantin Belousov }
77fad437baSKonstantin Belousov
781e3db1deSHans Petter Selasky int
linux_alloc_current(struct thread * td,int flags)791e3db1deSHans Petter Selasky linux_alloc_current(struct thread *td, int flags)
801e3db1deSHans Petter Selasky {
81e54b103eSHans Petter Selasky struct proc *proc;
821e3db1deSHans Petter Selasky struct task_struct *ts;
83fad437baSKonstantin Belousov struct mm_struct *mm, *mm_other;
841e3db1deSHans Petter Selasky
851e3db1deSHans Petter Selasky MPASS(td->td_lkpi_task == NULL);
861e3db1deSHans Petter Selasky
87165ba13fSKonstantin Belousov if ((td->td_pflags & TDP_ITHREAD) != 0 || !THREAD_CAN_SLEEP()) {
88165ba13fSKonstantin Belousov flags &= ~M_WAITOK;
89165ba13fSKonstantin Belousov flags |= M_NOWAIT | M_USE_RESERVE;
90165ba13fSKonstantin Belousov }
911e3db1deSHans Petter Selasky
92165ba13fSKonstantin Belousov ts = uma_zalloc(linux_current_zone, flags | M_ZERO);
93165ba13fSKonstantin Belousov if (ts == NULL) {
94165ba13fSKonstantin Belousov if ((flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK)
95165ba13fSKonstantin Belousov panic("linux_alloc_current: failed to allocate task");
96165ba13fSKonstantin Belousov return (ENOMEM);
97165ba13fSKonstantin Belousov }
98fad437baSKonstantin Belousov mm = NULL;
9905d4f501SHans Petter Selasky
100e54b103eSHans Petter Selasky /* setup new task structure */
1011e3db1deSHans Petter Selasky atomic_set(&ts->kthread_flags, 0);
1021e3db1deSHans Petter Selasky ts->task_thread = td;
1031e3db1deSHans Petter Selasky ts->comm = td->td_name;
1041e3db1deSHans Petter Selasky ts->pid = td->td_tid;
10507e0a3caSJohannes Lundberg ts->group_leader = ts;
106a0699ebfSHans Petter Selasky atomic_set(&ts->usage, 1);
107ef925749SHans Petter Selasky atomic_set(&ts->state, TASK_RUNNING);
1088504aa98SMark Johnston init_completion(&ts->parked);
1098504aa98SMark Johnston init_completion(&ts->exited);
11005d4f501SHans Petter Selasky
111e54b103eSHans Petter Selasky proc = td->td_proc;
112e54b103eSHans Petter Selasky
113e54b103eSHans Petter Selasky PROC_LOCK(proc);
114fad437baSKonstantin Belousov mm_other = find_other_mm(proc);
115e54b103eSHans Petter Selasky
116e54b103eSHans Petter Selasky /* use allocated mm_struct as a fallback */
117fad437baSKonstantin Belousov if (mm_other == NULL) {
118fad437baSKonstantin Belousov PROC_UNLOCK(proc);
119fad437baSKonstantin Belousov mm = uma_zalloc(linux_mm_zone, flags | M_ZERO);
120fad437baSKonstantin Belousov if (mm == NULL) {
121fad437baSKonstantin Belousov if ((flags & (M_WAITOK | M_NOWAIT)) == M_WAITOK)
122fad437baSKonstantin Belousov panic(
123fad437baSKonstantin Belousov "linux_alloc_current: failed to allocate mm");
124fad437baSKonstantin Belousov uma_zfree(linux_current_zone, mm);
125fad437baSKonstantin Belousov return (ENOMEM);
126fad437baSKonstantin Belousov }
127fad437baSKonstantin Belousov
128fad437baSKonstantin Belousov PROC_LOCK(proc);
129fad437baSKonstantin Belousov mm_other = find_other_mm(proc);
130fad437baSKonstantin Belousov if (mm_other == NULL) {
131e54b103eSHans Petter Selasky /* setup new mm_struct */
13205d4f501SHans Petter Selasky init_rwsem(&mm->mmap_sem);
13305d4f501SHans Petter Selasky atomic_set(&mm->mm_count, 1);
13405d4f501SHans Petter Selasky atomic_set(&mm->mm_users, 1);
135e54b103eSHans Petter Selasky /* set mm_struct pointer */
136e54b103eSHans Petter Selasky ts->mm = mm;
137e54b103eSHans Petter Selasky /* clear pointer to not free memory */
138e54b103eSHans Petter Selasky mm = NULL;
139fad437baSKonstantin Belousov } else {
140fad437baSKonstantin Belousov ts->mm = mm_other;
141fad437baSKonstantin Belousov }
142fad437baSKonstantin Belousov } else {
143fad437baSKonstantin Belousov ts->mm = mm_other;
144e54b103eSHans Petter Selasky }
14505d4f501SHans Petter Selasky
14605d4f501SHans Petter Selasky /* store pointer to task struct */
1471e3db1deSHans Petter Selasky td->td_lkpi_task = ts;
148e54b103eSHans Petter Selasky PROC_UNLOCK(proc);
149e54b103eSHans Petter Selasky
150e54b103eSHans Petter Selasky /* free mm_struct pointer, if any */
151165ba13fSKonstantin Belousov uma_zfree(linux_mm_zone, mm);
152e54b103eSHans Petter Selasky
1531e3db1deSHans Petter Selasky return (0);
1541e3db1deSHans Petter Selasky }
1551e3db1deSHans Petter Selasky
15605d4f501SHans Petter Selasky struct mm_struct *
linux_get_task_mm(struct task_struct * task)15705d4f501SHans Petter Selasky linux_get_task_mm(struct task_struct *task)
15805d4f501SHans Petter Selasky {
15905d4f501SHans Petter Selasky struct mm_struct *mm;
16005d4f501SHans Petter Selasky
16105d4f501SHans Petter Selasky mm = task->mm;
16268b9f2f0SHans Petter Selasky if (mm != NULL) {
16305d4f501SHans Petter Selasky atomic_inc(&mm->mm_users);
16405d4f501SHans Petter Selasky return (mm);
16505d4f501SHans Petter Selasky }
16605d4f501SHans Petter Selasky return (NULL);
16705d4f501SHans Petter Selasky }
16805d4f501SHans Petter Selasky
16905d4f501SHans Petter Selasky void
linux_mm_dtor(struct mm_struct * mm)17005d4f501SHans Petter Selasky linux_mm_dtor(struct mm_struct *mm)
17105d4f501SHans Petter Selasky {
172165ba13fSKonstantin Belousov uma_zfree(linux_mm_zone, mm);
17305d4f501SHans Petter Selasky }
17405d4f501SHans Petter Selasky
1751e3db1deSHans Petter Selasky void
linux_free_current(struct task_struct * ts)1761e3db1deSHans Petter Selasky linux_free_current(struct task_struct *ts)
1771e3db1deSHans Petter Selasky {
17805d4f501SHans Petter Selasky mmput(ts->mm);
179165ba13fSKonstantin Belousov uma_zfree(linux_current_zone, ts);
1801e3db1deSHans Petter Selasky }
1811e3db1deSHans Petter Selasky
1821e3db1deSHans Petter Selasky static void
linuxkpi_thread_dtor(void * arg __unused,struct thread * td)1831e3db1deSHans Petter Selasky linuxkpi_thread_dtor(void *arg __unused, struct thread *td)
1841e3db1deSHans Petter Selasky {
1851e3db1deSHans Petter Selasky struct task_struct *ts;
1861e3db1deSHans Petter Selasky
1871e3db1deSHans Petter Selasky ts = td->td_lkpi_task;
1881e3db1deSHans Petter Selasky if (ts == NULL)
1891e3db1deSHans Petter Selasky return;
1901e3db1deSHans Petter Selasky
1911e3db1deSHans Petter Selasky td->td_lkpi_task = NULL;
192a0699ebfSHans Petter Selasky put_task_struct(ts);
193a0699ebfSHans Petter Selasky }
194a0699ebfSHans Petter Selasky
195f334f212SKonstantin Belousov static struct task_struct *
linux_get_pid_task_int(pid_t pid,const bool do_get)196f334f212SKonstantin Belousov linux_get_pid_task_int(pid_t pid, const bool do_get)
197a0699ebfSHans Petter Selasky {
198a0699ebfSHans Petter Selasky struct thread *td;
1998402f058SHans Petter Selasky struct proc *p;
200f334f212SKonstantin Belousov struct task_struct *ts;
201a0699ebfSHans Petter Selasky
202f334f212SKonstantin Belousov if (pid > PID_MAX) {
2038402f058SHans Petter Selasky /* try to find corresponding thread */
204a0699ebfSHans Petter Selasky td = tdfind(pid, -1);
205a0699ebfSHans Petter Selasky if (td != NULL) {
206f334f212SKonstantin Belousov ts = td->td_lkpi_task;
207f334f212SKonstantin Belousov if (do_get && ts != NULL)
208f334f212SKonstantin Belousov get_task_struct(ts);
209a0699ebfSHans Petter Selasky PROC_UNLOCK(td->td_proc);
210a0699ebfSHans Petter Selasky return (ts);
211a0699ebfSHans Petter Selasky }
212f334f212SKonstantin Belousov } else {
2138402f058SHans Petter Selasky /* try to find corresponding procedure */
2148402f058SHans Petter Selasky p = pfind(pid);
2158402f058SHans Petter Selasky if (p != NULL) {
2168402f058SHans Petter Selasky FOREACH_THREAD_IN_PROC(p, td) {
217f334f212SKonstantin Belousov ts = td->td_lkpi_task;
2188402f058SHans Petter Selasky if (ts != NULL) {
219f334f212SKonstantin Belousov if (do_get)
220f334f212SKonstantin Belousov get_task_struct(ts);
2218402f058SHans Petter Selasky PROC_UNLOCK(p);
2228402f058SHans Petter Selasky return (ts);
2238402f058SHans Petter Selasky }
2248402f058SHans Petter Selasky }
2258402f058SHans Petter Selasky PROC_UNLOCK(p);
2268402f058SHans Petter Selasky }
227f334f212SKonstantin Belousov }
228a0699ebfSHans Petter Selasky return (NULL);
229a0699ebfSHans Petter Selasky }
230a0699ebfSHans Petter Selasky
231a0699ebfSHans Petter Selasky struct task_struct *
linux_pid_task(pid_t pid)232f334f212SKonstantin Belousov linux_pid_task(pid_t pid)
233f334f212SKonstantin Belousov {
234f334f212SKonstantin Belousov return (linux_get_pid_task_int(pid, false));
235f334f212SKonstantin Belousov }
236f334f212SKonstantin Belousov
237f334f212SKonstantin Belousov struct task_struct *
linux_get_pid_task(pid_t pid)238a0699ebfSHans Petter Selasky linux_get_pid_task(pid_t pid)
239a0699ebfSHans Petter Selasky {
240f334f212SKonstantin Belousov return (linux_get_pid_task_int(pid, true));
2411e3db1deSHans Petter Selasky }
2421e3db1deSHans Petter Selasky
243638fa5a3SHans Petter Selasky bool
linux_task_exiting(struct task_struct * task)244638fa5a3SHans Petter Selasky linux_task_exiting(struct task_struct *task)
245638fa5a3SHans Petter Selasky {
2469a4e535bSHans Petter Selasky struct thread *td;
247638fa5a3SHans Petter Selasky struct proc *p;
248638fa5a3SHans Petter Selasky bool ret;
249638fa5a3SHans Petter Selasky
250638fa5a3SHans Petter Selasky ret = false;
2519a4e535bSHans Petter Selasky
2529a4e535bSHans Petter Selasky /* try to find corresponding thread */
2539a4e535bSHans Petter Selasky td = tdfind(task->pid, -1);
2549a4e535bSHans Petter Selasky if (td != NULL) {
2559a4e535bSHans Petter Selasky p = td->td_proc;
2569a4e535bSHans Petter Selasky } else {
2579a4e535bSHans Petter Selasky /* try to find corresponding procedure */
258638fa5a3SHans Petter Selasky p = pfind(task->pid);
2599a4e535bSHans Petter Selasky }
2609a4e535bSHans Petter Selasky
261638fa5a3SHans Petter Selasky if (p != NULL) {
262638fa5a3SHans Petter Selasky if ((p->p_flag & P_WEXIT) != 0)
263638fa5a3SHans Petter Selasky ret = true;
264638fa5a3SHans Petter Selasky PROC_UNLOCK(p);
265638fa5a3SHans Petter Selasky }
266638fa5a3SHans Petter Selasky return (ret);
267638fa5a3SHans Petter Selasky }
268638fa5a3SHans Petter Selasky
269165ba13fSKonstantin Belousov static int lkpi_task_resrv;
270165ba13fSKonstantin Belousov SYSCTL_INT(_compat_linuxkpi, OID_AUTO, task_struct_reserve,
271165ba13fSKonstantin Belousov CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &lkpi_task_resrv, 0,
272165ba13fSKonstantin Belousov "Number of struct task and struct mm to reserve for non-sleepable "
273165ba13fSKonstantin Belousov "allocations");
274165ba13fSKonstantin Belousov
2751e3db1deSHans Petter Selasky static void
linux_current_init(void * arg __unused)2761e3db1deSHans Petter Selasky linux_current_init(void *arg __unused)
2771e3db1deSHans Petter Selasky {
278165ba13fSKonstantin Belousov TUNABLE_INT_FETCH("compat.linuxkpi.task_struct_reserve",
279165ba13fSKonstantin Belousov &lkpi_task_resrv);
280165ba13fSKonstantin Belousov if (lkpi_task_resrv == 0) {
281fdc18947STijl Coosemans #ifdef DEV_APIC
282165ba13fSKonstantin Belousov /*
283165ba13fSKonstantin Belousov * Number of interrupt threads plus per-cpu callout
284165ba13fSKonstantin Belousov * SWI threads.
285165ba13fSKonstantin Belousov */
286165ba13fSKonstantin Belousov lkpi_task_resrv = first_msi_irq + num_msi_irqs + MAXCPU;
287165ba13fSKonstantin Belousov #else
288165ba13fSKonstantin Belousov lkpi_task_resrv = 1024; /* XXXKIB arbitrary */
289165ba13fSKonstantin Belousov #endif
290165ba13fSKonstantin Belousov }
291165ba13fSKonstantin Belousov linux_current_zone = uma_zcreate("lkpicurr",
292165ba13fSKonstantin Belousov sizeof(struct task_struct), NULL, NULL, NULL, NULL,
293165ba13fSKonstantin Belousov UMA_ALIGN_PTR, 0);
294165ba13fSKonstantin Belousov uma_zone_reserve(linux_current_zone, lkpi_task_resrv);
295165ba13fSKonstantin Belousov uma_prealloc(linux_current_zone, lkpi_task_resrv);
296165ba13fSKonstantin Belousov linux_mm_zone = uma_zcreate("lkpimm",
2974958df5dSKonstantin Belousov sizeof(struct mm_struct), NULL, NULL, NULL, NULL,
298165ba13fSKonstantin Belousov UMA_ALIGN_PTR, 0);
299165ba13fSKonstantin Belousov uma_zone_reserve(linux_mm_zone, lkpi_task_resrv);
300165ba13fSKonstantin Belousov uma_prealloc(linux_mm_zone, lkpi_task_resrv);
301b764a426SHans Petter Selasky
302b764a426SHans Petter Selasky atomic_thread_fence_seq_cst();
303b764a426SHans Petter Selasky
304b764a426SHans Petter Selasky linuxkpi_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor,
305b764a426SHans Petter Selasky linuxkpi_thread_dtor, NULL, EVENTHANDLER_PRI_ANY);
306d96f55bcSKonstantin Belousov lkpi_alloc_current = linux_alloc_current;
3071e3db1deSHans Petter Selasky }
308c3707bd3SMateusz Guzik SYSINIT(linux_current, SI_SUB_EVENTHANDLER + 1, SI_ORDER_SECOND,
3094ce1f616SKonstantin Belousov linux_current_init, NULL);
3101e3db1deSHans Petter Selasky
3111e3db1deSHans Petter Selasky static void
linux_current_uninit(void * arg __unused)3121e3db1deSHans Petter Selasky linux_current_uninit(void *arg __unused)
3131e3db1deSHans Petter Selasky {
314abf5c031SMark Johnston struct proc *p;
315abf5c031SMark Johnston struct task_struct *ts;
316abf5c031SMark Johnston struct thread *td;
317abf5c031SMark Johnston
318b764a426SHans Petter Selasky lkpi_alloc_current = linux_alloc_current_noop;
319b764a426SHans Petter Selasky
320b764a426SHans Petter Selasky atomic_thread_fence_seq_cst();
321b764a426SHans Petter Selasky
322abf5c031SMark Johnston sx_slock(&allproc_lock);
323abf5c031SMark Johnston FOREACH_PROC_IN_SYSTEM(p) {
324abf5c031SMark Johnston PROC_LOCK(p);
325abf5c031SMark Johnston FOREACH_THREAD_IN_PROC(p, td) {
326abf5c031SMark Johnston if ((ts = td->td_lkpi_task) != NULL) {
327abf5c031SMark Johnston td->td_lkpi_task = NULL;
328abf5c031SMark Johnston put_task_struct(ts);
329abf5c031SMark Johnston }
330abf5c031SMark Johnston }
331abf5c031SMark Johnston PROC_UNLOCK(p);
332abf5c031SMark Johnston }
333abf5c031SMark Johnston sx_sunlock(&allproc_lock);
334b764a426SHans Petter Selasky
335d96f55bcSKonstantin Belousov thread_reap_barrier();
336b764a426SHans Petter Selasky
3371e3db1deSHans Petter Selasky EVENTHANDLER_DEREGISTER(thread_dtor, linuxkpi_thread_dtor_tag);
338b764a426SHans Petter Selasky
339165ba13fSKonstantin Belousov uma_zdestroy(linux_current_zone);
340165ba13fSKonstantin Belousov uma_zdestroy(linux_mm_zone);
3411e3db1deSHans Petter Selasky }
342c3707bd3SMateusz Guzik SYSUNINIT(linux_current, SI_SUB_EVENTHANDLER + 1, SI_ORDER_SECOND,
3434ce1f616SKonstantin Belousov linux_current_uninit, NULL);
344