1d7f687fcSJeff Roberson /*- 28a36da99SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 38a36da99SPedro F. Giffuni * 4d7f687fcSJeff Roberson * Copyright (c) 2008, Jeffrey Roberson <jeff@freebsd.org> 5d7f687fcSJeff Roberson * All rights reserved. 6d7f687fcSJeff Roberson * 73bc8c68dSJeff Roberson * Copyright (c) 2008 Nokia Corporation 83bc8c68dSJeff Roberson * All rights reserved. 93bc8c68dSJeff Roberson * 10d7f687fcSJeff Roberson * Redistribution and use in source and binary forms, with or without 11d7f687fcSJeff Roberson * modification, are permitted provided that the following conditions 12d7f687fcSJeff Roberson * are met: 13d7f687fcSJeff Roberson * 1. Redistributions of source code must retain the above copyright 14d7f687fcSJeff Roberson * notice unmodified, this list of conditions, and the following 15d7f687fcSJeff Roberson * disclaimer. 16d7f687fcSJeff Roberson * 2. Redistributions in binary form must reproduce the above copyright 17d7f687fcSJeff Roberson * notice, this list of conditions and the following disclaimer in the 18d7f687fcSJeff Roberson * documentation and/or other materials provided with the distribution. 19d7f687fcSJeff Roberson * 20d7f687fcSJeff Roberson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21d7f687fcSJeff Roberson * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22d7f687fcSJeff Roberson * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23d7f687fcSJeff Roberson * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24d7f687fcSJeff Roberson * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25d7f687fcSJeff Roberson * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26d7f687fcSJeff Roberson * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27d7f687fcSJeff Roberson * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28d7f687fcSJeff Roberson * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29d7f687fcSJeff Roberson * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30d7f687fcSJeff Roberson * 31d7f687fcSJeff Roberson */ 32d7f687fcSJeff Roberson 33d7f687fcSJeff Roberson #include <sys/cdefs.h> 34d7f687fcSJeff Roberson __FBSDID("$FreeBSD$"); 35d7f687fcSJeff Roberson 36dea0ed66SBjoern A. Zeeb #include "opt_ddb.h" 37586ed321SDmitry Chagin #include "opt_ktrace.h" 38dea0ed66SBjoern A. Zeeb 39d7f687fcSJeff Roberson #include <sys/param.h> 40d7f687fcSJeff Roberson #include <sys/systm.h> 41e5818a53SJeff Roberson #include <sys/sysctl.h> 42e5818a53SJeff Roberson #include <sys/ctype.h> 43d7f687fcSJeff Roberson #include <sys/sysproto.h> 440304c731SJamie Gritton #include <sys/jail.h> 45d7f687fcSJeff Roberson #include <sys/kernel.h> 46d7f687fcSJeff Roberson #include <sys/lock.h> 47d7f687fcSJeff Roberson #include <sys/malloc.h> 48d7f687fcSJeff Roberson #include <sys/mutex.h> 49d7f687fcSJeff Roberson #include <sys/priv.h> 50d7f687fcSJeff Roberson #include <sys/proc.h> 51d7f687fcSJeff Roberson #include <sys/refcount.h> 52d7f687fcSJeff Roberson #include <sys/sched.h> 53d7f687fcSJeff Roberson #include <sys/smp.h> 54d7f687fcSJeff Roberson #include <sys/syscallsubr.h> 5547a57144SJustin Hibbits #include <sys/sysent.h> 56f299c47bSAllan Jude #include <sys/capsicum.h> 57d7f687fcSJeff Roberson #include <sys/cpuset.h> 583f289c3fSJeff Roberson #include <sys/domainset.h> 59d7f687fcSJeff Roberson #include <sys/sx.h> 60d7f687fcSJeff Roberson #include <sys/queue.h> 61e3709597SAttilio Rao #include <sys/libkern.h> 62d7f687fcSJeff Roberson #include <sys/limits.h> 63a03ee000SJeff Roberson #include <sys/bus.h> 64a03ee000SJeff Roberson #include <sys/interrupt.h> 653f289c3fSJeff Roberson #include <sys/vmmeter.h> 66586ed321SDmitry Chagin #include <sys/ktrace.h> 67d7f687fcSJeff Roberson 68d7f687fcSJeff Roberson #include <vm/uma.h> 69c0ae6688SJohn Baldwin #include <vm/vm.h> 703f289c3fSJeff Roberson #include <vm/vm_object.h> 7130c5525bSAndrew Gallatin #include <vm/vm_page.h> 7230c5525bSAndrew Gallatin #include <vm/vm_pageout.h> 73e5818a53SJeff Roberson #include <vm/vm_extern.h> 7430c5525bSAndrew Gallatin #include <vm/vm_param.h> 7530c5525bSAndrew Gallatin #include <vm/vm_phys.h> 7630c5525bSAndrew Gallatin #include <vm/vm_pagequeue.h> 77d7f687fcSJeff Roberson 78dea0ed66SBjoern A. Zeeb #ifdef DDB 79dea0ed66SBjoern A. Zeeb #include <ddb/ddb.h> 80dea0ed66SBjoern A. Zeeb #endif /* DDB */ 81dea0ed66SBjoern A. Zeeb 82d7f687fcSJeff Roberson /* 83d7f687fcSJeff Roberson * cpusets provide a mechanism for creating and manipulating sets of 84d7f687fcSJeff Roberson * processors for the purpose of constraining the scheduling of threads to 85d7f687fcSJeff Roberson * specific processors. 86d7f687fcSJeff Roberson * 87d7f687fcSJeff Roberson * Each process belongs to an identified set, by default this is set 1. Each 88d7f687fcSJeff Roberson * thread may further restrict the cpus it may run on to a subset of this 89d7f687fcSJeff Roberson * named set. This creates an anonymous set which other threads and processes 90d7f687fcSJeff Roberson * may not join by number. 91d7f687fcSJeff Roberson * 92d7f687fcSJeff Roberson * The named set is referred to herein as the 'base' set to avoid ambiguity. 93d7f687fcSJeff Roberson * This set is usually a child of a 'root' set while the anonymous set may 94d7f687fcSJeff Roberson * simply be referred to as a mask. In the syscall api these are referred to 95d7f687fcSJeff Roberson * as the ROOT, CPUSET, and MASK levels where CPUSET is called 'base' here. 96d7f687fcSJeff Roberson * 97d7f687fcSJeff Roberson * Threads inherit their set from their creator whether it be anonymous or 98d7f687fcSJeff Roberson * not. This means that anonymous sets are immutable because they may be 99d7f687fcSJeff Roberson * shared. To modify an anonymous set a new set is created with the desired 100d7f687fcSJeff Roberson * mask and the same parent as the existing anonymous set. This gives the 10193902625SJohn Baldwin * illusion of each thread having a private mask. 102d7f687fcSJeff Roberson * 103d7f687fcSJeff Roberson * Via the syscall apis a user may ask to retrieve or modify the root, base, 104d7f687fcSJeff Roberson * or mask that is discovered via a pid, tid, or setid. Modifying a set 105d7f687fcSJeff Roberson * modifies all numbered and anonymous child sets to comply with the new mask. 106d7f687fcSJeff Roberson * Modifying a pid or tid's mask applies only to that tid but must still 107d7f687fcSJeff Roberson * exist within the assigned parent set. 108d7f687fcSJeff Roberson * 10986855bf5SJohn Baldwin * A thread may not be assigned to a group separate from other threads in 110d7f687fcSJeff Roberson * the process. This is to remove ambiguity when the setid is queried with 111d7f687fcSJeff Roberson * a pid argument. There is no other technical limitation. 112d7f687fcSJeff Roberson * 113d7f687fcSJeff Roberson * This somewhat complex arrangement is intended to make it easy for 114d7f687fcSJeff Roberson * applications to query available processors and bind their threads to 115d7f687fcSJeff Roberson * specific processors while also allowing administrators to dynamically 116d7f687fcSJeff Roberson * reprovision by changing sets which apply to groups of processes. 117d7f687fcSJeff Roberson * 118d7f687fcSJeff Roberson * A simple application should not concern itself with sets at all and 119d7f687fcSJeff Roberson * rather apply masks to its own threads via CPU_WHICH_TID and a -1 id 12093902625SJohn Baldwin * meaning 'curthread'. It may query available cpus for that tid with a 121d7f687fcSJeff Roberson * getaffinity call using (CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, ...). 122d7f687fcSJeff Roberson */ 123e5818a53SJeff Roberson 124e5818a53SJeff Roberson LIST_HEAD(domainlist, domainset); 12529bb6c19SMark Johnston struct domainset __read_mostly domainset_firsttouch; 1269978bd99SMark Johnston struct domainset __read_mostly domainset_fixed[MAXMEMDOM]; 12729bb6c19SMark Johnston struct domainset __read_mostly domainset_interleave; 128662e7fa8SMark Johnston struct domainset __read_mostly domainset_prefer[MAXMEMDOM]; 129662e7fa8SMark Johnston struct domainset __read_mostly domainset_roundrobin; 130e5818a53SJeff Roberson 131d7f687fcSJeff Roberson static uma_zone_t cpuset_zone; 1323f289c3fSJeff Roberson static uma_zone_t domainset_zone; 133d7f687fcSJeff Roberson static struct mtx cpuset_lock; 134d7f687fcSJeff Roberson static struct setlist cpuset_ids; 1353f289c3fSJeff Roberson static struct domainlist cpuset_domains; 136d7f687fcSJeff Roberson static struct unrhdr *cpuset_unr; 137e5818a53SJeff Roberson static struct cpuset *cpuset_zero, *cpuset_default, *cpuset_kernel; 13829bb6c19SMark Johnston static struct domainset *domainset0, *domainset2; 139a03ee000SJeff Roberson 140444528c0SDavid Xu /* Return the size of cpuset_t at the kernel level */ 14160aa2c85SJonathan Anderson SYSCTL_INT(_kern_sched, OID_AUTO, cpusetsize, CTLFLAG_RD | CTLFLAG_CAPRD, 142f0188618SHans Petter Selasky SYSCTL_NULL_INT_PTR, sizeof(cpuset_t), "sizeof(cpuset_t)"); 143444528c0SDavid Xu 144a03ee000SJeff Roberson cpuset_t *cpuset_root; 145c0ae6688SJohn Baldwin cpuset_t cpuset_domain[MAXMEMDOM]; 146d7f687fcSJeff Roberson 1473f289c3fSJeff Roberson static int domainset_valid(const struct domainset *, const struct domainset *); 1483f289c3fSJeff Roberson 1493f289c3fSJeff Roberson /* 1503f289c3fSJeff Roberson * Find the first non-anonymous set starting from 'set'. 1513f289c3fSJeff Roberson */ 1523f289c3fSJeff Roberson static struct cpuset * 1533f289c3fSJeff Roberson cpuset_getbase(struct cpuset *set) 1543f289c3fSJeff Roberson { 1553f289c3fSJeff Roberson 1563f289c3fSJeff Roberson if (set->cs_id == CPUSET_INVALID) 1573f289c3fSJeff Roberson set = set->cs_parent; 1583f289c3fSJeff Roberson return (set); 1593f289c3fSJeff Roberson } 1603f289c3fSJeff Roberson 1613f289c3fSJeff Roberson /* 1623f289c3fSJeff Roberson * Walks up the tree from 'set' to find the root. 1633f289c3fSJeff Roberson */ 1643f289c3fSJeff Roberson static struct cpuset * 1653f289c3fSJeff Roberson cpuset_getroot(struct cpuset *set) 1663f289c3fSJeff Roberson { 1673f289c3fSJeff Roberson 1683f289c3fSJeff Roberson while ((set->cs_flags & CPU_SET_ROOT) == 0 && set->cs_parent != NULL) 1693f289c3fSJeff Roberson set = set->cs_parent; 1703f289c3fSJeff Roberson return (set); 1713f289c3fSJeff Roberson } 1723f289c3fSJeff Roberson 173d7f687fcSJeff Roberson /* 174d7f687fcSJeff Roberson * Acquire a reference to a cpuset, all pointers must be tracked with refs. 175d7f687fcSJeff Roberson */ 176d7f687fcSJeff Roberson struct cpuset * 177d7f687fcSJeff Roberson cpuset_ref(struct cpuset *set) 178d7f687fcSJeff Roberson { 179d7f687fcSJeff Roberson 180d7f687fcSJeff Roberson refcount_acquire(&set->cs_ref); 181d7f687fcSJeff Roberson return (set); 182d7f687fcSJeff Roberson } 183d7f687fcSJeff Roberson 184d7f687fcSJeff Roberson /* 1857a8f695aSBjoern A. Zeeb * Walks up the tree from 'set' to find the root. Returns the root 1867a8f695aSBjoern A. Zeeb * referenced. 1877a8f695aSBjoern A. Zeeb */ 1887a8f695aSBjoern A. Zeeb static struct cpuset * 1897a8f695aSBjoern A. Zeeb cpuset_refroot(struct cpuset *set) 1907a8f695aSBjoern A. Zeeb { 1917a8f695aSBjoern A. Zeeb 1923f289c3fSJeff Roberson return (cpuset_ref(cpuset_getroot(set))); 1937a8f695aSBjoern A. Zeeb } 1947a8f695aSBjoern A. Zeeb 1957a8f695aSBjoern A. Zeeb /* 1967a8f695aSBjoern A. Zeeb * Find the first non-anonymous set starting from 'set'. Returns this set 1977a8f695aSBjoern A. Zeeb * referenced. May return the passed in set with an extra ref if it is 1987a8f695aSBjoern A. Zeeb * not anonymous. 1997a8f695aSBjoern A. Zeeb */ 2007a8f695aSBjoern A. Zeeb static struct cpuset * 2017a8f695aSBjoern A. Zeeb cpuset_refbase(struct cpuset *set) 2027a8f695aSBjoern A. Zeeb { 2037a8f695aSBjoern A. Zeeb 2043f289c3fSJeff Roberson return (cpuset_ref(cpuset_getbase(set))); 2057a8f695aSBjoern A. Zeeb } 2067a8f695aSBjoern A. Zeeb 2077a8f695aSBjoern A. Zeeb /* 20893902625SJohn Baldwin * Release a reference in a context where it is safe to allocate. 209d7f687fcSJeff Roberson */ 210d7f687fcSJeff Roberson void 211d7f687fcSJeff Roberson cpuset_rel(struct cpuset *set) 212d7f687fcSJeff Roberson { 213d7f687fcSJeff Roberson cpusetid_t id; 214d7f687fcSJeff Roberson 215b2780e85SKyle Evans if (refcount_release_if_not_last(&set->cs_ref)) 216d7f687fcSJeff Roberson return; 217d7f687fcSJeff Roberson mtx_lock_spin(&cpuset_lock); 218b2780e85SKyle Evans if (!refcount_release(&set->cs_ref)) { 219b2780e85SKyle Evans mtx_unlock_spin(&cpuset_lock); 220b2780e85SKyle Evans return; 221b2780e85SKyle Evans } 222d7f687fcSJeff Roberson LIST_REMOVE(set, cs_siblings); 223d7f687fcSJeff Roberson id = set->cs_id; 224d7f687fcSJeff Roberson if (id != CPUSET_INVALID) 225d7f687fcSJeff Roberson LIST_REMOVE(set, cs_link); 226d7f687fcSJeff Roberson mtx_unlock_spin(&cpuset_lock); 227d7f687fcSJeff Roberson cpuset_rel(set->cs_parent); 228d7f687fcSJeff Roberson uma_zfree(cpuset_zone, set); 229d7f687fcSJeff Roberson if (id != CPUSET_INVALID) 230d7f687fcSJeff Roberson free_unr(cpuset_unr, id); 231d7f687fcSJeff Roberson } 232d7f687fcSJeff Roberson 233d7f687fcSJeff Roberson /* 234d7f687fcSJeff Roberson * Deferred release must be used when in a context that is not safe to 235d7f687fcSJeff Roberson * allocate/free. This places any unreferenced sets on the list 'head'. 236d7f687fcSJeff Roberson */ 237d7f687fcSJeff Roberson static void 238d7f687fcSJeff Roberson cpuset_rel_defer(struct setlist *head, struct cpuset *set) 239d7f687fcSJeff Roberson { 240d7f687fcSJeff Roberson 241b2780e85SKyle Evans if (refcount_release_if_not_last(&set->cs_ref)) 242d7f687fcSJeff Roberson return; 243d7f687fcSJeff Roberson mtx_lock_spin(&cpuset_lock); 244b2780e85SKyle Evans if (!refcount_release(&set->cs_ref)) { 245b2780e85SKyle Evans mtx_unlock_spin(&cpuset_lock); 246b2780e85SKyle Evans return; 247b2780e85SKyle Evans } 248d7f687fcSJeff Roberson LIST_REMOVE(set, cs_siblings); 249d7f687fcSJeff Roberson if (set->cs_id != CPUSET_INVALID) 250d7f687fcSJeff Roberson LIST_REMOVE(set, cs_link); 251d7f687fcSJeff Roberson LIST_INSERT_HEAD(head, set, cs_link); 252d7f687fcSJeff Roberson mtx_unlock_spin(&cpuset_lock); 253d7f687fcSJeff Roberson } 254d7f687fcSJeff Roberson 255d7f687fcSJeff Roberson /* 256d7f687fcSJeff Roberson * Complete a deferred release. Removes the set from the list provided to 257d7f687fcSJeff Roberson * cpuset_rel_defer. 258d7f687fcSJeff Roberson */ 259d7f687fcSJeff Roberson static void 260d7f687fcSJeff Roberson cpuset_rel_complete(struct cpuset *set) 261d7f687fcSJeff Roberson { 2629c83dab9SKyle Evans cpusetid_t id; 2639c83dab9SKyle Evans 2649c83dab9SKyle Evans id = set->cs_id; 265d7f687fcSJeff Roberson LIST_REMOVE(set, cs_link); 266d7f687fcSJeff Roberson cpuset_rel(set->cs_parent); 267d7f687fcSJeff Roberson uma_zfree(cpuset_zone, set); 2689c83dab9SKyle Evans if (id != CPUSET_INVALID) 2699c83dab9SKyle Evans free_unr(cpuset_unr, id); 270d7f687fcSJeff Roberson } 271d7f687fcSJeff Roberson 272d7f687fcSJeff Roberson /* 273d7f687fcSJeff Roberson * Find a set based on an id. Returns it with a ref. 274d7f687fcSJeff Roberson */ 275d7f687fcSJeff Roberson static struct cpuset * 276413628a7SBjoern A. Zeeb cpuset_lookup(cpusetid_t setid, struct thread *td) 277d7f687fcSJeff Roberson { 278d7f687fcSJeff Roberson struct cpuset *set; 279d7f687fcSJeff Roberson 280d7f687fcSJeff Roberson if (setid == CPUSET_INVALID) 281d7f687fcSJeff Roberson return (NULL); 282d7f687fcSJeff Roberson mtx_lock_spin(&cpuset_lock); 283d7f687fcSJeff Roberson LIST_FOREACH(set, &cpuset_ids, cs_link) 284d7f687fcSJeff Roberson if (set->cs_id == setid) 285d7f687fcSJeff Roberson break; 286d7f687fcSJeff Roberson if (set) 287d7f687fcSJeff Roberson cpuset_ref(set); 288d7f687fcSJeff Roberson mtx_unlock_spin(&cpuset_lock); 289413628a7SBjoern A. Zeeb 290413628a7SBjoern A. Zeeb KASSERT(td != NULL, ("[%s:%d] td is NULL", __func__, __LINE__)); 291413628a7SBjoern A. Zeeb if (set != NULL && jailed(td->td_ucred)) { 2920304c731SJamie Gritton struct cpuset *jset, *tset; 293413628a7SBjoern A. Zeeb 2940304c731SJamie Gritton jset = td->td_ucred->cr_prison->pr_cpuset; 2950304c731SJamie Gritton for (tset = set; tset != NULL; tset = tset->cs_parent) 2960304c731SJamie Gritton if (tset == jset) 2970304c731SJamie Gritton break; 2980304c731SJamie Gritton if (tset == NULL) { 299413628a7SBjoern A. Zeeb cpuset_rel(set); 300413628a7SBjoern A. Zeeb set = NULL; 301413628a7SBjoern A. Zeeb } 302413628a7SBjoern A. Zeeb } 303413628a7SBjoern A. Zeeb 304d7f687fcSJeff Roberson return (set); 305d7f687fcSJeff Roberson } 306d7f687fcSJeff Roberson 307d7f687fcSJeff Roberson /* 30830b7c6f9SKyle Evans * Initialize a set in the space provided in 'set' with the provided parameters. 309d7f687fcSJeff Roberson * The set is returned with a single ref. May return EDEADLK if the set 310d7f687fcSJeff Roberson * will have no valid cpu based on restrictions from the parent. 311d7f687fcSJeff Roberson */ 312d7f687fcSJeff Roberson static int 31330b7c6f9SKyle Evans cpuset_init(struct cpuset *set, struct cpuset *parent, 3143f289c3fSJeff Roberson const cpuset_t *mask, struct domainset *domain, cpusetid_t id) 315d7f687fcSJeff Roberson { 316d7f687fcSJeff Roberson 3173f289c3fSJeff Roberson if (domain == NULL) 3183f289c3fSJeff Roberson domain = parent->cs_domain; 3193f289c3fSJeff Roberson if (mask == NULL) 3203f289c3fSJeff Roberson mask = &parent->cs_mask; 32173c40187SJeff Roberson if (!CPU_OVERLAP(&parent->cs_mask, mask)) 32273c40187SJeff Roberson return (EDEADLK); 3233f289c3fSJeff Roberson /* The domain must be prepared ahead of time. */ 3243f289c3fSJeff Roberson if (!domainset_valid(parent->cs_domain, domain)) 3253f289c3fSJeff Roberson return (EDEADLK); 326d7f687fcSJeff Roberson CPU_COPY(mask, &set->cs_mask); 327d7f687fcSJeff Roberson LIST_INIT(&set->cs_children); 328d7f687fcSJeff Roberson refcount_init(&set->cs_ref, 1); 329d7f687fcSJeff Roberson set->cs_flags = 0; 330d7f687fcSJeff Roberson mtx_lock_spin(&cpuset_lock); 3313f289c3fSJeff Roberson set->cs_domain = domain; 332e2650af1SStefan Eßer CPU_AND(&set->cs_mask, &set->cs_mask, &parent->cs_mask); 333d7f687fcSJeff Roberson set->cs_id = id; 334d7f687fcSJeff Roberson set->cs_parent = cpuset_ref(parent); 335d7f687fcSJeff Roberson LIST_INSERT_HEAD(&parent->cs_children, set, cs_siblings); 336d7f687fcSJeff Roberson if (set->cs_id != CPUSET_INVALID) 337d7f687fcSJeff Roberson LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 338d7f687fcSJeff Roberson mtx_unlock_spin(&cpuset_lock); 339d7f687fcSJeff Roberson 34073c40187SJeff Roberson return (0); 341d7f687fcSJeff Roberson } 342d7f687fcSJeff Roberson 343d7f687fcSJeff Roberson /* 344d7f687fcSJeff Roberson * Create a new non-anonymous set with the requested parent and mask. May 345d7f687fcSJeff Roberson * return failures if the mask is invalid or a new number can not be 346d7f687fcSJeff Roberson * allocated. 34729d04ea8SKyle Evans * 34829d04ea8SKyle Evans * If *setp is not NULL, then it will be used as-is. The caller must take 34929d04ea8SKyle Evans * into account that *setp will be inserted at the head of cpuset_ids and 35029d04ea8SKyle Evans * plan any potentially conflicting cs_link usage accordingly. 351d7f687fcSJeff Roberson */ 352d7f687fcSJeff Roberson static int 353e84c2db1SJohn Baldwin cpuset_create(struct cpuset **setp, struct cpuset *parent, const cpuset_t *mask) 354d7f687fcSJeff Roberson { 355d7f687fcSJeff Roberson struct cpuset *set; 356d7f687fcSJeff Roberson cpusetid_t id; 357d7f687fcSJeff Roberson int error; 35829d04ea8SKyle Evans bool dofree; 359d7f687fcSJeff Roberson 360d7f687fcSJeff Roberson id = alloc_unr(cpuset_unr); 361d7f687fcSJeff Roberson if (id == -1) 362d7f687fcSJeff Roberson return (ENFILE); 36329d04ea8SKyle Evans dofree = (*setp == NULL); 36429d04ea8SKyle Evans if (*setp != NULL) 36529d04ea8SKyle Evans set = *setp; 36629d04ea8SKyle Evans else 3673f289c3fSJeff Roberson *setp = set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 36830b7c6f9SKyle Evans error = cpuset_init(set, parent, mask, NULL, id); 369d7f687fcSJeff Roberson if (error == 0) 370d7f687fcSJeff Roberson return (0); 371d7f687fcSJeff Roberson free_unr(cpuset_unr, id); 37229d04ea8SKyle Evans if (dofree) 373d7f687fcSJeff Roberson uma_zfree(cpuset_zone, set); 374d7f687fcSJeff Roberson 375d7f687fcSJeff Roberson return (error); 376d7f687fcSJeff Roberson } 377d7f687fcSJeff Roberson 3783f289c3fSJeff Roberson static void 3793f289c3fSJeff Roberson cpuset_freelist_add(struct setlist *list, int count) 3803f289c3fSJeff Roberson { 3813f289c3fSJeff Roberson struct cpuset *set; 3823f289c3fSJeff Roberson int i; 3833f289c3fSJeff Roberson 3843f289c3fSJeff Roberson for (i = 0; i < count; i++) { 3853f289c3fSJeff Roberson set = uma_zalloc(cpuset_zone, M_ZERO | M_WAITOK); 3863f289c3fSJeff Roberson LIST_INSERT_HEAD(list, set, cs_link); 3873f289c3fSJeff Roberson } 3883f289c3fSJeff Roberson } 3893f289c3fSJeff Roberson 3903f289c3fSJeff Roberson static void 3913f289c3fSJeff Roberson cpuset_freelist_init(struct setlist *list, int count) 3923f289c3fSJeff Roberson { 3933f289c3fSJeff Roberson 3943f289c3fSJeff Roberson LIST_INIT(list); 3953f289c3fSJeff Roberson cpuset_freelist_add(list, count); 3963f289c3fSJeff Roberson } 3973f289c3fSJeff Roberson 3983f289c3fSJeff Roberson static void 3993f289c3fSJeff Roberson cpuset_freelist_free(struct setlist *list) 4003f289c3fSJeff Roberson { 4013f289c3fSJeff Roberson struct cpuset *set; 4023f289c3fSJeff Roberson 4033f289c3fSJeff Roberson while ((set = LIST_FIRST(list)) != NULL) { 4043f289c3fSJeff Roberson LIST_REMOVE(set, cs_link); 4053f289c3fSJeff Roberson uma_zfree(cpuset_zone, set); 4063f289c3fSJeff Roberson } 4073f289c3fSJeff Roberson } 4083f289c3fSJeff Roberson 4093f289c3fSJeff Roberson static void 4103f289c3fSJeff Roberson domainset_freelist_add(struct domainlist *list, int count) 4113f289c3fSJeff Roberson { 4123f289c3fSJeff Roberson struct domainset *set; 4133f289c3fSJeff Roberson int i; 4143f289c3fSJeff Roberson 4153f289c3fSJeff Roberson for (i = 0; i < count; i++) { 4163f289c3fSJeff Roberson set = uma_zalloc(domainset_zone, M_ZERO | M_WAITOK); 4173f289c3fSJeff Roberson LIST_INSERT_HEAD(list, set, ds_link); 4183f289c3fSJeff Roberson } 4193f289c3fSJeff Roberson } 4203f289c3fSJeff Roberson 4213f289c3fSJeff Roberson static void 4223f289c3fSJeff Roberson domainset_freelist_init(struct domainlist *list, int count) 4233f289c3fSJeff Roberson { 4243f289c3fSJeff Roberson 4253f289c3fSJeff Roberson LIST_INIT(list); 4263f289c3fSJeff Roberson domainset_freelist_add(list, count); 4273f289c3fSJeff Roberson } 4283f289c3fSJeff Roberson 4293f289c3fSJeff Roberson static void 4303f289c3fSJeff Roberson domainset_freelist_free(struct domainlist *list) 4313f289c3fSJeff Roberson { 4323f289c3fSJeff Roberson struct domainset *set; 4333f289c3fSJeff Roberson 4343f289c3fSJeff Roberson while ((set = LIST_FIRST(list)) != NULL) { 4353f289c3fSJeff Roberson LIST_REMOVE(set, ds_link); 4363f289c3fSJeff Roberson uma_zfree(domainset_zone, set); 4373f289c3fSJeff Roberson } 4383f289c3fSJeff Roberson } 4393f289c3fSJeff Roberson 4403f289c3fSJeff Roberson /* Copy a domainset preserving mask and policy. */ 4413f289c3fSJeff Roberson static void 4423f289c3fSJeff Roberson domainset_copy(const struct domainset *from, struct domainset *to) 4433f289c3fSJeff Roberson { 4443f289c3fSJeff Roberson 4453f289c3fSJeff Roberson DOMAINSET_COPY(&from->ds_mask, &to->ds_mask); 4463f289c3fSJeff Roberson to->ds_policy = from->ds_policy; 4473f289c3fSJeff Roberson to->ds_prefer = from->ds_prefer; 4483f289c3fSJeff Roberson } 4493f289c3fSJeff Roberson 4503f289c3fSJeff Roberson /* Return 1 if mask and policy are equal, otherwise 0. */ 4513f289c3fSJeff Roberson static int 4523f289c3fSJeff Roberson domainset_equal(const struct domainset *one, const struct domainset *two) 4533f289c3fSJeff Roberson { 4543f289c3fSJeff Roberson 4553f289c3fSJeff Roberson return (DOMAINSET_CMP(&one->ds_mask, &two->ds_mask) == 0 && 4563f289c3fSJeff Roberson one->ds_policy == two->ds_policy && 4573f289c3fSJeff Roberson one->ds_prefer == two->ds_prefer); 4583f289c3fSJeff Roberson } 4593f289c3fSJeff Roberson 4603f289c3fSJeff Roberson /* Return 1 if child is a valid subset of parent. */ 4613f289c3fSJeff Roberson static int 4623f289c3fSJeff Roberson domainset_valid(const struct domainset *parent, const struct domainset *child) 4633f289c3fSJeff Roberson { 4643f289c3fSJeff Roberson if (child->ds_policy != DOMAINSET_POLICY_PREFER) 4653f289c3fSJeff Roberson return (DOMAINSET_SUBSET(&parent->ds_mask, &child->ds_mask)); 4663f289c3fSJeff Roberson return (DOMAINSET_ISSET(child->ds_prefer, &parent->ds_mask)); 4673f289c3fSJeff Roberson } 4683f289c3fSJeff Roberson 4693f289c3fSJeff Roberson static int 4703f289c3fSJeff Roberson domainset_restrict(const struct domainset *parent, 4713f289c3fSJeff Roberson const struct domainset *child) 4723f289c3fSJeff Roberson { 4733f289c3fSJeff Roberson if (child->ds_policy != DOMAINSET_POLICY_PREFER) 4743f289c3fSJeff Roberson return (DOMAINSET_OVERLAP(&parent->ds_mask, &child->ds_mask)); 4753f289c3fSJeff Roberson return (DOMAINSET_ISSET(child->ds_prefer, &parent->ds_mask)); 4763f289c3fSJeff Roberson } 4773f289c3fSJeff Roberson 4783f289c3fSJeff Roberson /* 4793f289c3fSJeff Roberson * Lookup or create a domainset. The key is provided in ds_mask and 4803f289c3fSJeff Roberson * ds_policy. If the domainset does not yet exist the storage in 4813f289c3fSJeff Roberson * 'domain' is used to insert. Otherwise this storage is freed to the 4823f289c3fSJeff Roberson * domainset_zone and the existing domainset is returned. 4833f289c3fSJeff Roberson */ 4843f289c3fSJeff Roberson static struct domainset * 4853f289c3fSJeff Roberson _domainset_create(struct domainset *domain, struct domainlist *freelist) 4863f289c3fSJeff Roberson { 4873f289c3fSJeff Roberson struct domainset *ndomain; 48844e4def7SMark Johnston int i, j; 4893f289c3fSJeff Roberson 490b61f3142SMark Johnston KASSERT(domain->ds_cnt <= vm_ndomains, 491b61f3142SMark Johnston ("invalid domain count in domainset %p", domain)); 492b61f3142SMark Johnston KASSERT(domain->ds_policy != DOMAINSET_POLICY_PREFER || 493b61f3142SMark Johnston domain->ds_prefer < vm_ndomains, 494b61f3142SMark Johnston ("invalid preferred domain in domains %p", domain)); 495b61f3142SMark Johnston 4963f289c3fSJeff Roberson mtx_lock_spin(&cpuset_lock); 4973f289c3fSJeff Roberson LIST_FOREACH(ndomain, &cpuset_domains, ds_link) 4983f289c3fSJeff Roberson if (domainset_equal(ndomain, domain)) 4993f289c3fSJeff Roberson break; 5003f289c3fSJeff Roberson /* 5013f289c3fSJeff Roberson * If the domain does not yet exist we insert it and initialize 5023f289c3fSJeff Roberson * various iteration helpers which are not part of the key. 5033f289c3fSJeff Roberson */ 5043f289c3fSJeff Roberson if (ndomain == NULL) { 5053f289c3fSJeff Roberson LIST_INSERT_HEAD(&cpuset_domains, domain, ds_link); 5063f289c3fSJeff Roberson domain->ds_cnt = DOMAINSET_COUNT(&domain->ds_mask); 50744e4def7SMark Johnston for (i = 0, j = 0; i < DOMAINSET_FLS(&domain->ds_mask); i++) 508e5818a53SJeff Roberson if (DOMAINSET_ISSET(i, &domain->ds_mask)) 509e5818a53SJeff Roberson domain->ds_order[j++] = i; 5103f289c3fSJeff Roberson } 5113f289c3fSJeff Roberson mtx_unlock_spin(&cpuset_lock); 5123f289c3fSJeff Roberson if (ndomain == NULL) 5133f289c3fSJeff Roberson return (domain); 5143f289c3fSJeff Roberson if (freelist != NULL) 5153f289c3fSJeff Roberson LIST_INSERT_HEAD(freelist, domain, ds_link); 5163f289c3fSJeff Roberson else 5173f289c3fSJeff Roberson uma_zfree(domainset_zone, domain); 5183f289c3fSJeff Roberson return (ndomain); 5193f289c3fSJeff Roberson 5203f289c3fSJeff Roberson } 5213f289c3fSJeff Roberson 5223f289c3fSJeff Roberson /* 52330c5525bSAndrew Gallatin * Are any of the domains in the mask empty? If so, silently 524920239efSMark Johnston * remove them and update the domainset accordingly. If only empty 525920239efSMark Johnston * domains are present, we must return failure. 52630c5525bSAndrew Gallatin */ 52730c5525bSAndrew Gallatin static bool 52830c5525bSAndrew Gallatin domainset_empty_vm(struct domainset *domain) 52930c5525bSAndrew Gallatin { 5308e697504SMark Johnston domainset_t empty; 5318e697504SMark Johnston int i, j; 53230c5525bSAndrew Gallatin 5338e697504SMark Johnston DOMAINSET_ZERO(&empty); 5348e697504SMark Johnston for (i = 0; i < vm_ndomains; i++) 5358e697504SMark Johnston if (VM_DOMAIN_EMPTY(i)) 5368e697504SMark Johnston DOMAINSET_SET(i, &empty); 5378e697504SMark Johnston if (DOMAINSET_SUBSET(&empty, &domain->ds_mask)) 5388e697504SMark Johnston return (true); 5398e697504SMark Johnston 5408e697504SMark Johnston /* Remove empty domains from the set and recompute. */ 5419825eadfSRyan Libby DOMAINSET_ANDNOT(&domain->ds_mask, &empty); 542920239efSMark Johnston domain->ds_cnt = DOMAINSET_COUNT(&domain->ds_mask); 5438e697504SMark Johnston for (i = j = 0; i < DOMAINSET_FLS(&domain->ds_mask); i++) 544920239efSMark Johnston if (DOMAINSET_ISSET(i, &domain->ds_mask)) 545920239efSMark Johnston domain->ds_order[j++] = i; 5468e697504SMark Johnston 5478e697504SMark Johnston /* Convert a PREFER policy referencing an empty domain to RR. */ 5488e697504SMark Johnston if (domain->ds_policy == DOMAINSET_POLICY_PREFER && 5498e697504SMark Johnston DOMAINSET_ISSET(domain->ds_prefer, &empty)) { 550920239efSMark Johnston domain->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; 551920239efSMark Johnston domain->ds_prefer = -1; 552920239efSMark Johnston } 55330c5525bSAndrew Gallatin 5548e697504SMark Johnston return (false); 55530c5525bSAndrew Gallatin } 55630c5525bSAndrew Gallatin 55730c5525bSAndrew Gallatin /* 5583f289c3fSJeff Roberson * Create or lookup a domainset based on the key held in 'domain'. 5593f289c3fSJeff Roberson */ 560e5818a53SJeff Roberson struct domainset * 5613f289c3fSJeff Roberson domainset_create(const struct domainset *domain) 5623f289c3fSJeff Roberson { 5633f289c3fSJeff Roberson struct domainset *ndomain; 5643f289c3fSJeff Roberson 565e5818a53SJeff Roberson /* 566e5818a53SJeff Roberson * Validate the policy. It must specify a useable policy number with 567e5818a53SJeff Roberson * only valid domains. Preferred must include the preferred domain 568e5818a53SJeff Roberson * in the mask. 569e5818a53SJeff Roberson */ 570e5818a53SJeff Roberson if (domain->ds_policy <= DOMAINSET_POLICY_INVALID || 571e5818a53SJeff Roberson domain->ds_policy > DOMAINSET_POLICY_MAX) 572e5818a53SJeff Roberson return (NULL); 573e5818a53SJeff Roberson if (domain->ds_policy == DOMAINSET_POLICY_PREFER && 574e5818a53SJeff Roberson !DOMAINSET_ISSET(domain->ds_prefer, &domain->ds_mask)) 575e5818a53SJeff Roberson return (NULL); 57629bb6c19SMark Johnston if (!DOMAINSET_SUBSET(&domainset0->ds_mask, &domain->ds_mask)) 577e5818a53SJeff Roberson return (NULL); 5783f289c3fSJeff Roberson ndomain = uma_zalloc(domainset_zone, M_WAITOK | M_ZERO); 5793f289c3fSJeff Roberson domainset_copy(domain, ndomain); 5803f289c3fSJeff Roberson return _domainset_create(ndomain, NULL); 5813f289c3fSJeff Roberson } 5823f289c3fSJeff Roberson 5833f289c3fSJeff Roberson /* 5843f289c3fSJeff Roberson * Update thread domainset pointers. 5853f289c3fSJeff Roberson */ 5863f289c3fSJeff Roberson static void 5873f289c3fSJeff Roberson domainset_notify(void) 5883f289c3fSJeff Roberson { 5893f289c3fSJeff Roberson struct thread *td; 5903f289c3fSJeff Roberson struct proc *p; 5913f289c3fSJeff Roberson 5923f289c3fSJeff Roberson sx_slock(&allproc_lock); 5933f289c3fSJeff Roberson FOREACH_PROC_IN_SYSTEM(p) { 5943f289c3fSJeff Roberson PROC_LOCK(p); 5953f289c3fSJeff Roberson if (p->p_state == PRS_NEW) { 5963f289c3fSJeff Roberson PROC_UNLOCK(p); 5973f289c3fSJeff Roberson continue; 5983f289c3fSJeff Roberson } 5993f289c3fSJeff Roberson FOREACH_THREAD_IN_PROC(p, td) { 6003f289c3fSJeff Roberson thread_lock(td); 6013f289c3fSJeff Roberson td->td_domain.dr_policy = td->td_cpuset->cs_domain; 6023f289c3fSJeff Roberson thread_unlock(td); 6033f289c3fSJeff Roberson } 6043f289c3fSJeff Roberson PROC_UNLOCK(p); 6053f289c3fSJeff Roberson } 6063f289c3fSJeff Roberson sx_sunlock(&allproc_lock); 607e5818a53SJeff Roberson kernel_object->domain.dr_policy = cpuset_kernel->cs_domain; 6083f289c3fSJeff Roberson } 6093f289c3fSJeff Roberson 6103f289c3fSJeff Roberson /* 6113f289c3fSJeff Roberson * Create a new set that is a subset of a parent. 6123f289c3fSJeff Roberson */ 6133f289c3fSJeff Roberson static struct domainset * 6143f289c3fSJeff Roberson domainset_shadow(const struct domainset *pdomain, 6153f289c3fSJeff Roberson const struct domainset *domain, struct domainlist *freelist) 6163f289c3fSJeff Roberson { 6173f289c3fSJeff Roberson struct domainset *ndomain; 6183f289c3fSJeff Roberson 6193f289c3fSJeff Roberson ndomain = LIST_FIRST(freelist); 6203f289c3fSJeff Roberson LIST_REMOVE(ndomain, ds_link); 6213f289c3fSJeff Roberson 6223f289c3fSJeff Roberson /* 6233f289c3fSJeff Roberson * Initialize the key from the request. 6243f289c3fSJeff Roberson */ 6253f289c3fSJeff Roberson domainset_copy(domain, ndomain); 6263f289c3fSJeff Roberson 6273f289c3fSJeff Roberson /* 6283f289c3fSJeff Roberson * Restrict the key by the parent. 6293f289c3fSJeff Roberson */ 6303f289c3fSJeff Roberson DOMAINSET_AND(&ndomain->ds_mask, &pdomain->ds_mask); 6313f289c3fSJeff Roberson 6323f289c3fSJeff Roberson return _domainset_create(ndomain, freelist); 6333f289c3fSJeff Roberson } 6343f289c3fSJeff Roberson 635d7f687fcSJeff Roberson /* 636d7f687fcSJeff Roberson * Recursively check for errors that would occur from applying mask to 637d7f687fcSJeff Roberson * the tree of sets starting at 'set'. Checks for sets that would become 638d7f687fcSJeff Roberson * empty as well as RDONLY flags. 639d7f687fcSJeff Roberson */ 640d7f687fcSJeff Roberson static int 641f1b18a66SKyle Evans cpuset_testupdate(struct cpuset *set, cpuset_t *mask, int augment_mask) 642d7f687fcSJeff Roberson { 643d7f687fcSJeff Roberson struct cpuset *nset; 644d7f687fcSJeff Roberson cpuset_t newmask; 645d7f687fcSJeff Roberson int error; 646d7f687fcSJeff Roberson 647d7f687fcSJeff Roberson mtx_assert(&cpuset_lock, MA_OWNED); 648d7f687fcSJeff Roberson if (set->cs_flags & CPU_SET_RDONLY) 649d7f687fcSJeff Roberson return (EPERM); 650f1b18a66SKyle Evans if (augment_mask) { 651e2650af1SStefan Eßer CPU_AND(&newmask, &set->cs_mask, mask); 652c9813d0aSJohn Baldwin } else 653c9813d0aSJohn Baldwin CPU_COPY(mask, &newmask); 654f1b18a66SKyle Evans 655f1b18a66SKyle Evans if (CPU_EMPTY(&newmask)) 656f1b18a66SKyle Evans return (EDEADLK); 65773c40187SJeff Roberson error = 0; 658d7f687fcSJeff Roberson LIST_FOREACH(nset, &set->cs_children, cs_siblings) 659c9813d0aSJohn Baldwin if ((error = cpuset_testupdate(nset, &newmask, 1)) != 0) 660d7f687fcSJeff Roberson break; 661d7f687fcSJeff Roberson return (error); 662d7f687fcSJeff Roberson } 663d7f687fcSJeff Roberson 664d7f687fcSJeff Roberson /* 665d7f687fcSJeff Roberson * Applies the mask 'mask' without checking for empty sets or permissions. 666d7f687fcSJeff Roberson */ 667d7f687fcSJeff Roberson static void 668d7f687fcSJeff Roberson cpuset_update(struct cpuset *set, cpuset_t *mask) 669d7f687fcSJeff Roberson { 670d7f687fcSJeff Roberson struct cpuset *nset; 671d7f687fcSJeff Roberson 672d7f687fcSJeff Roberson mtx_assert(&cpuset_lock, MA_OWNED); 673e2650af1SStefan Eßer CPU_AND(&set->cs_mask, &set->cs_mask, mask); 674d7f687fcSJeff Roberson LIST_FOREACH(nset, &set->cs_children, cs_siblings) 675d7f687fcSJeff Roberson cpuset_update(nset, &set->cs_mask); 676d7f687fcSJeff Roberson 677d7f687fcSJeff Roberson return; 678d7f687fcSJeff Roberson } 679d7f687fcSJeff Roberson 680d7f687fcSJeff Roberson /* 681d7f687fcSJeff Roberson * Modify the set 'set' to use a copy of the mask provided. Apply this new 682d7f687fcSJeff Roberson * mask to restrict all children in the tree. Checks for validity before 683d7f687fcSJeff Roberson * applying the changes. 684d7f687fcSJeff Roberson */ 685d7f687fcSJeff Roberson static int 686d7f687fcSJeff Roberson cpuset_modify(struct cpuset *set, cpuset_t *mask) 687d7f687fcSJeff Roberson { 68873c40187SJeff Roberson struct cpuset *root; 689d7f687fcSJeff Roberson int error; 690d7f687fcSJeff Roberson 691ba931c08SBjoern A. Zeeb error = priv_check(curthread, PRIV_SCHED_CPUSET); 692d7f687fcSJeff Roberson if (error) 693d7f687fcSJeff Roberson return (error); 69473c40187SJeff Roberson /* 69554a837c8SKyle Evans * In case we are called from within the jail, 6966aaa0b3cSBjoern A. Zeeb * we do not allow modifying the dedicated root 6976aaa0b3cSBjoern A. Zeeb * cpuset of the jail but may still allow to 69854a837c8SKyle Evans * change child sets, including subordinate jails' 69954a837c8SKyle Evans * roots. 7006aaa0b3cSBjoern A. Zeeb */ 70154a837c8SKyle Evans if ((set->cs_flags & CPU_SET_ROOT) != 0 && 70254a837c8SKyle Evans jailed(curthread->td_ucred) && 70354a837c8SKyle Evans set == curthread->td_ucred->cr_prison->pr_cpuset) 7046aaa0b3cSBjoern A. Zeeb return (EPERM); 7056aaa0b3cSBjoern A. Zeeb /* 70673c40187SJeff Roberson * Verify that we have access to this set of 70773c40187SJeff Roberson * cpus. 70873c40187SJeff Roberson */ 70954a837c8SKyle Evans if ((set->cs_flags & (CPU_SET_ROOT | CPU_SET_RDONLY)) == CPU_SET_ROOT) { 71054a837c8SKyle Evans KASSERT(set->cs_parent != NULL, 71154a837c8SKyle Evans ("jail.cpuset=%d is not a proper child of parent jail's root.", 71254a837c8SKyle Evans set->cs_id)); 71354a837c8SKyle Evans 71454a837c8SKyle Evans /* 71554a837c8SKyle Evans * cpuset_getroot() cannot work here due to how top-level jail 71654a837c8SKyle Evans * roots are constructed. Top-level jails are parented to 71754a837c8SKyle Evans * thread0's cpuset (i.e. cpuset 1) rather than the system root. 71854a837c8SKyle Evans */ 71954a837c8SKyle Evans root = set->cs_parent; 72054a837c8SKyle Evans } else { 7213f289c3fSJeff Roberson root = cpuset_getroot(set); 72254a837c8SKyle Evans } 723d7f687fcSJeff Roberson mtx_lock_spin(&cpuset_lock); 7243f289c3fSJeff Roberson if (root && !CPU_SUBSET(&root->cs_mask, mask)) { 7253f289c3fSJeff Roberson error = EINVAL; 7263f289c3fSJeff Roberson goto out; 7273f289c3fSJeff Roberson } 728c9813d0aSJohn Baldwin error = cpuset_testupdate(set, mask, 0); 729d7f687fcSJeff Roberson if (error) 730d7f687fcSJeff Roberson goto out; 731d7f687fcSJeff Roberson CPU_COPY(mask, &set->cs_mask); 732c9813d0aSJohn Baldwin cpuset_update(set, mask); 733d7f687fcSJeff Roberson out: 734d7f687fcSJeff Roberson mtx_unlock_spin(&cpuset_lock); 735d7f687fcSJeff Roberson 736d7f687fcSJeff Roberson return (error); 737d7f687fcSJeff Roberson } 738d7f687fcSJeff Roberson 739d7f687fcSJeff Roberson /* 7403f289c3fSJeff Roberson * Recursively check for errors that would occur from applying mask to 7413f289c3fSJeff Roberson * the tree of sets starting at 'set'. Checks for sets that would become 7423f289c3fSJeff Roberson * empty as well as RDONLY flags. 7433f289c3fSJeff Roberson */ 7443f289c3fSJeff Roberson static int 7453f289c3fSJeff Roberson cpuset_testupdate_domain(struct cpuset *set, struct domainset *dset, 746f1b18a66SKyle Evans struct domainset *orig, int *count, int augment_mask __unused) 7473f289c3fSJeff Roberson { 7483f289c3fSJeff Roberson struct cpuset *nset; 7493f289c3fSJeff Roberson struct domainset *domain; 7503f289c3fSJeff Roberson struct domainset newset; 7513f289c3fSJeff Roberson int error; 7523f289c3fSJeff Roberson 7533f289c3fSJeff Roberson mtx_assert(&cpuset_lock, MA_OWNED); 7543f289c3fSJeff Roberson if (set->cs_flags & CPU_SET_RDONLY) 7553f289c3fSJeff Roberson return (EPERM); 7563f289c3fSJeff Roberson domain = set->cs_domain; 7573f289c3fSJeff Roberson domainset_copy(domain, &newset); 7583f289c3fSJeff Roberson if (!domainset_equal(domain, orig)) { 7593f289c3fSJeff Roberson if (!domainset_restrict(domain, dset)) 7603f289c3fSJeff Roberson return (EDEADLK); 7613f289c3fSJeff Roberson DOMAINSET_AND(&newset.ds_mask, &dset->ds_mask); 7623f289c3fSJeff Roberson /* Count the number of domains that are changing. */ 7633f289c3fSJeff Roberson (*count)++; 7643f289c3fSJeff Roberson } 7653f289c3fSJeff Roberson error = 0; 7663f289c3fSJeff Roberson LIST_FOREACH(nset, &set->cs_children, cs_siblings) 7673f289c3fSJeff Roberson if ((error = cpuset_testupdate_domain(nset, &newset, domain, 7683f289c3fSJeff Roberson count, 1)) != 0) 7693f289c3fSJeff Roberson break; 7703f289c3fSJeff Roberson return (error); 7713f289c3fSJeff Roberson } 7723f289c3fSJeff Roberson 7733f289c3fSJeff Roberson /* 7743f289c3fSJeff Roberson * Applies the mask 'mask' without checking for empty sets or permissions. 7753f289c3fSJeff Roberson */ 7763f289c3fSJeff Roberson static void 7773f289c3fSJeff Roberson cpuset_update_domain(struct cpuset *set, struct domainset *domain, 7783f289c3fSJeff Roberson struct domainset *orig, struct domainlist *domains) 7793f289c3fSJeff Roberson { 7803f289c3fSJeff Roberson struct cpuset *nset; 7813f289c3fSJeff Roberson 7823f289c3fSJeff Roberson mtx_assert(&cpuset_lock, MA_OWNED); 7833f289c3fSJeff Roberson /* 7843f289c3fSJeff Roberson * If this domainset has changed from the parent we must calculate 7853f289c3fSJeff Roberson * a new set. Otherwise it simply inherits from the parent. When 7863f289c3fSJeff Roberson * we inherit from the parent we get a new mask and policy. If the 7873f289c3fSJeff Roberson * set is modified from the parent we keep the policy and only 7883f289c3fSJeff Roberson * update the mask. 7893f289c3fSJeff Roberson */ 7903f289c3fSJeff Roberson if (set->cs_domain != orig) { 7913f289c3fSJeff Roberson orig = set->cs_domain; 7923f289c3fSJeff Roberson set->cs_domain = domainset_shadow(domain, orig, domains); 7933f289c3fSJeff Roberson } else 7943f289c3fSJeff Roberson set->cs_domain = domain; 7953f289c3fSJeff Roberson LIST_FOREACH(nset, &set->cs_children, cs_siblings) 7963f289c3fSJeff Roberson cpuset_update_domain(nset, set->cs_domain, orig, domains); 7973f289c3fSJeff Roberson 7983f289c3fSJeff Roberson return; 7993f289c3fSJeff Roberson } 8003f289c3fSJeff Roberson 8013f289c3fSJeff Roberson /* 8023f289c3fSJeff Roberson * Modify the set 'set' to use a copy the domainset provided. Apply this new 8033f289c3fSJeff Roberson * mask to restrict all children in the tree. Checks for validity before 8043f289c3fSJeff Roberson * applying the changes. 8053f289c3fSJeff Roberson */ 8063f289c3fSJeff Roberson static int 8073f289c3fSJeff Roberson cpuset_modify_domain(struct cpuset *set, struct domainset *domain) 8083f289c3fSJeff Roberson { 8093f289c3fSJeff Roberson struct domainlist domains; 8103f289c3fSJeff Roberson struct domainset temp; 8113f289c3fSJeff Roberson struct domainset *dset; 8123f289c3fSJeff Roberson struct cpuset *root; 8133f289c3fSJeff Roberson int ndomains, needed; 8143f289c3fSJeff Roberson int error; 8153f289c3fSJeff Roberson 8163f289c3fSJeff Roberson error = priv_check(curthread, PRIV_SCHED_CPUSET); 8173f289c3fSJeff Roberson if (error) 8183f289c3fSJeff Roberson return (error); 8193f289c3fSJeff Roberson /* 8203f289c3fSJeff Roberson * In case we are called from within the jail 8213f289c3fSJeff Roberson * we do not allow modifying the dedicated root 8223f289c3fSJeff Roberson * cpuset of the jail but may still allow to 8233f289c3fSJeff Roberson * change child sets. 8243f289c3fSJeff Roberson */ 8253f289c3fSJeff Roberson if (jailed(curthread->td_ucred) && 8263f289c3fSJeff Roberson set->cs_flags & CPU_SET_ROOT) 8273f289c3fSJeff Roberson return (EPERM); 8283f289c3fSJeff Roberson domainset_freelist_init(&domains, 0); 8293f289c3fSJeff Roberson domain = domainset_create(domain); 830e07e3fa3SKyle Evans ndomains = 0; 831e07e3fa3SKyle Evans 8323f289c3fSJeff Roberson mtx_lock_spin(&cpuset_lock); 833e07e3fa3SKyle Evans for (;;) { 834e07e3fa3SKyle Evans root = cpuset_getroot(set); 8353f289c3fSJeff Roberson dset = root->cs_domain; 8363f289c3fSJeff Roberson /* 8373f289c3fSJeff Roberson * Verify that we have access to this set of domains. 8383f289c3fSJeff Roberson */ 83945cdd437SMark Johnston if (!domainset_valid(dset, domain)) { 8403f289c3fSJeff Roberson error = EINVAL; 8413f289c3fSJeff Roberson goto out; 8423f289c3fSJeff Roberson } 8433f289c3fSJeff Roberson /* 8443f289c3fSJeff Roberson * If applying prefer we keep the current set as the fallback. 8453f289c3fSJeff Roberson */ 8463f289c3fSJeff Roberson if (domain->ds_policy == DOMAINSET_POLICY_PREFER) 8473f289c3fSJeff Roberson DOMAINSET_COPY(&set->cs_domain->ds_mask, 8483f289c3fSJeff Roberson &domain->ds_mask); 8493f289c3fSJeff Roberson /* 8503f289c3fSJeff Roberson * Determine whether we can apply this set of domains and 8513f289c3fSJeff Roberson * how many new domain structures it will require. 8523f289c3fSJeff Roberson */ 8533f289c3fSJeff Roberson domainset_copy(domain, &temp); 8543f289c3fSJeff Roberson needed = 0; 8553f289c3fSJeff Roberson error = cpuset_testupdate_domain(set, &temp, set->cs_domain, 8563f289c3fSJeff Roberson &needed, 0); 8573f289c3fSJeff Roberson if (error) 8583f289c3fSJeff Roberson goto out; 859e07e3fa3SKyle Evans if (ndomains >= needed) 860e07e3fa3SKyle Evans break; 861e07e3fa3SKyle Evans 862e07e3fa3SKyle Evans /* Dropping the lock; we'll need to re-evaluate again. */ 863e07e3fa3SKyle Evans mtx_unlock_spin(&cpuset_lock); 864e07e3fa3SKyle Evans domainset_freelist_add(&domains, needed - ndomains); 865e07e3fa3SKyle Evans ndomains = needed; 866e07e3fa3SKyle Evans mtx_lock_spin(&cpuset_lock); 867e07e3fa3SKyle Evans } 8683f289c3fSJeff Roberson dset = set->cs_domain; 8693f289c3fSJeff Roberson cpuset_update_domain(set, domain, dset, &domains); 8703f289c3fSJeff Roberson out: 8713f289c3fSJeff Roberson mtx_unlock_spin(&cpuset_lock); 8723f289c3fSJeff Roberson domainset_freelist_free(&domains); 8733f289c3fSJeff Roberson if (error == 0) 8743f289c3fSJeff Roberson domainset_notify(); 8753f289c3fSJeff Roberson 8763f289c3fSJeff Roberson return (error); 8773f289c3fSJeff Roberson } 8783f289c3fSJeff Roberson 8793f289c3fSJeff Roberson /* 880d7f687fcSJeff Roberson * Resolve the 'which' parameter of several cpuset apis. 881d7f687fcSJeff Roberson * 882d7f687fcSJeff Roberson * For WHICH_PID and WHICH_TID return a locked proc and valid proc/tid. Also 883d7f687fcSJeff Roberson * checks for permission via p_cansched(). 884d7f687fcSJeff Roberson * 885d7f687fcSJeff Roberson * For WHICH_SET returns a valid set with a new reference. 886d7f687fcSJeff Roberson * 887d7f687fcSJeff Roberson * -1 may be supplied for any argument to mean the current proc/thread or 888d7f687fcSJeff Roberson * the base set of the current thread. May fail with ESRCH/EPERM. 889d7f687fcSJeff Roberson */ 8905bbb2169SAdrian Chadd int 891d7f687fcSJeff Roberson cpuset_which(cpuwhich_t which, id_t id, struct proc **pp, struct thread **tdp, 892d7f687fcSJeff Roberson struct cpuset **setp) 893d7f687fcSJeff Roberson { 894d7f687fcSJeff Roberson struct cpuset *set; 895d7f687fcSJeff Roberson struct thread *td; 896d7f687fcSJeff Roberson struct proc *p; 897d7f687fcSJeff Roberson int error; 898d7f687fcSJeff Roberson 899d7f687fcSJeff Roberson *pp = p = NULL; 900d7f687fcSJeff Roberson *tdp = td = NULL; 901d7f687fcSJeff Roberson *setp = set = NULL; 902d7f687fcSJeff Roberson switch (which) { 903d7f687fcSJeff Roberson case CPU_WHICH_PID: 904d7f687fcSJeff Roberson if (id == -1) { 905d7f687fcSJeff Roberson PROC_LOCK(curproc); 906d7f687fcSJeff Roberson p = curproc; 907d7f687fcSJeff Roberson break; 908d7f687fcSJeff Roberson } 909d7f687fcSJeff Roberson if ((p = pfind(id)) == NULL) 910d7f687fcSJeff Roberson return (ESRCH); 911d7f687fcSJeff Roberson break; 912d7f687fcSJeff Roberson case CPU_WHICH_TID: 913d7f687fcSJeff Roberson if (id == -1) { 914d7f687fcSJeff Roberson PROC_LOCK(curproc); 915d7f687fcSJeff Roberson p = curproc; 916d7f687fcSJeff Roberson td = curthread; 917d7f687fcSJeff Roberson break; 918d7f687fcSJeff Roberson } 91942fe684cSDavid Xu td = tdfind(id, -1); 920d7f687fcSJeff Roberson if (td == NULL) 921d7f687fcSJeff Roberson return (ESRCH); 92242fe684cSDavid Xu p = td->td_proc; 923d7f687fcSJeff Roberson break; 924d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 925d7f687fcSJeff Roberson if (id == -1) { 926d7f687fcSJeff Roberson thread_lock(curthread); 927a03ee000SJeff Roberson set = cpuset_refbase(curthread->td_cpuset); 928d7f687fcSJeff Roberson thread_unlock(curthread); 929d7f687fcSJeff Roberson } else 930413628a7SBjoern A. Zeeb set = cpuset_lookup(id, curthread); 931d7f687fcSJeff Roberson if (set) { 932d7f687fcSJeff Roberson *setp = set; 933d7f687fcSJeff Roberson return (0); 934d7f687fcSJeff Roberson } 935d7f687fcSJeff Roberson return (ESRCH); 936413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 937413628a7SBjoern A. Zeeb { 938413628a7SBjoern A. Zeeb /* Find `set' for prison with given id. */ 939413628a7SBjoern A. Zeeb struct prison *pr; 940413628a7SBjoern A. Zeeb 941413628a7SBjoern A. Zeeb sx_slock(&allprison_lock); 9420304c731SJamie Gritton pr = prison_find_child(curthread->td_ucred->cr_prison, id); 943413628a7SBjoern A. Zeeb sx_sunlock(&allprison_lock); 944413628a7SBjoern A. Zeeb if (pr == NULL) 945413628a7SBjoern A. Zeeb return (ESRCH); 946413628a7SBjoern A. Zeeb cpuset_ref(pr->pr_cpuset); 9470304c731SJamie Gritton *setp = pr->pr_cpuset; 948413628a7SBjoern A. Zeeb mtx_unlock(&pr->pr_mtx); 949413628a7SBjoern A. Zeeb return (0); 950413628a7SBjoern A. Zeeb } 9519b33b154SJeff Roberson case CPU_WHICH_IRQ: 952c0ae6688SJohn Baldwin case CPU_WHICH_DOMAIN: 9539b33b154SJeff Roberson return (0); 954d7f687fcSJeff Roberson default: 955d7f687fcSJeff Roberson return (EINVAL); 956d7f687fcSJeff Roberson } 957d7f687fcSJeff Roberson error = p_cansched(curthread, p); 958d7f687fcSJeff Roberson if (error) { 959d7f687fcSJeff Roberson PROC_UNLOCK(p); 960d7f687fcSJeff Roberson return (error); 961d7f687fcSJeff Roberson } 962d7f687fcSJeff Roberson if (td == NULL) 963d7f687fcSJeff Roberson td = FIRST_THREAD_IN_PROC(p); 964d7f687fcSJeff Roberson *pp = p; 965d7f687fcSJeff Roberson *tdp = td; 966d7f687fcSJeff Roberson return (0); 967d7f687fcSJeff Roberson } 968d7f687fcSJeff Roberson 969d7f687fcSJeff Roberson static int 9703f289c3fSJeff Roberson cpuset_testshadow(struct cpuset *set, const cpuset_t *mask, 9713f289c3fSJeff Roberson const struct domainset *domain) 972d7f687fcSJeff Roberson { 973d7f687fcSJeff Roberson struct cpuset *parent; 9743f289c3fSJeff Roberson struct domainset *dset; 975d7f687fcSJeff Roberson 9763f289c3fSJeff Roberson parent = cpuset_getbase(set); 9773f289c3fSJeff Roberson /* 9783f289c3fSJeff Roberson * If we are restricting a cpu mask it must be a subset of the 9793f289c3fSJeff Roberson * parent or invalid CPUs have been specified. 9803f289c3fSJeff Roberson */ 9813f289c3fSJeff Roberson if (mask != NULL && !CPU_SUBSET(&parent->cs_mask, mask)) 9823f289c3fSJeff Roberson return (EINVAL); 9833f289c3fSJeff Roberson 9843f289c3fSJeff Roberson /* 9853f289c3fSJeff Roberson * If we are restricting a domain mask it must be a subset of the 9863f289c3fSJeff Roberson * parent or invalid domains have been specified. 9873f289c3fSJeff Roberson */ 9883f289c3fSJeff Roberson dset = parent->cs_domain; 9893f289c3fSJeff Roberson if (domain != NULL && !domainset_valid(dset, domain)) 9903f289c3fSJeff Roberson return (EINVAL); 9913f289c3fSJeff Roberson 9923f289c3fSJeff Roberson return (0); 993d7f687fcSJeff Roberson } 994d7f687fcSJeff Roberson 995d7f687fcSJeff Roberson /* 9963f289c3fSJeff Roberson * Create an anonymous set with the provided mask in the space provided by 9973f289c3fSJeff Roberson * 'nset'. If the passed in set is anonymous we use its parent otherwise 9983f289c3fSJeff Roberson * the new set is a child of 'set'. 9993f289c3fSJeff Roberson */ 10003f289c3fSJeff Roberson static int 10013f289c3fSJeff Roberson cpuset_shadow(struct cpuset *set, struct cpuset **nsetp, 10023f289c3fSJeff Roberson const cpuset_t *mask, const struct domainset *domain, 10033f289c3fSJeff Roberson struct setlist *cpusets, struct domainlist *domains) 10043f289c3fSJeff Roberson { 10053f289c3fSJeff Roberson struct cpuset *parent; 10063f289c3fSJeff Roberson struct cpuset *nset; 10073f289c3fSJeff Roberson struct domainset *dset; 10083f289c3fSJeff Roberson struct domainset *d; 10093f289c3fSJeff Roberson int error; 10103f289c3fSJeff Roberson 10113f289c3fSJeff Roberson error = cpuset_testshadow(set, mask, domain); 10123f289c3fSJeff Roberson if (error) 10133f289c3fSJeff Roberson return (error); 10143f289c3fSJeff Roberson 10153f289c3fSJeff Roberson parent = cpuset_getbase(set); 10163f289c3fSJeff Roberson dset = parent->cs_domain; 10173f289c3fSJeff Roberson if (mask == NULL) 10183f289c3fSJeff Roberson mask = &set->cs_mask; 10193f289c3fSJeff Roberson if (domain != NULL) 10203f289c3fSJeff Roberson d = domainset_shadow(dset, domain, domains); 10213f289c3fSJeff Roberson else 10223f289c3fSJeff Roberson d = set->cs_domain; 10233f289c3fSJeff Roberson nset = LIST_FIRST(cpusets); 102430b7c6f9SKyle Evans error = cpuset_init(nset, parent, mask, d, CPUSET_INVALID); 10253f289c3fSJeff Roberson if (error == 0) { 10263f289c3fSJeff Roberson LIST_REMOVE(nset, cs_link); 10273f289c3fSJeff Roberson *nsetp = nset; 10283f289c3fSJeff Roberson } 10293f289c3fSJeff Roberson return (error); 10303f289c3fSJeff Roberson } 10313f289c3fSJeff Roberson 10323f289c3fSJeff Roberson static struct cpuset * 10333f289c3fSJeff Roberson cpuset_update_thread(struct thread *td, struct cpuset *nset) 10343f289c3fSJeff Roberson { 10353f289c3fSJeff Roberson struct cpuset *tdset; 10363f289c3fSJeff Roberson 10373f289c3fSJeff Roberson tdset = td->td_cpuset; 10383f289c3fSJeff Roberson td->td_cpuset = nset; 10393f289c3fSJeff Roberson td->td_domain.dr_policy = nset->cs_domain; 10403f289c3fSJeff Roberson sched_affinity(td); 10413f289c3fSJeff Roberson 10423f289c3fSJeff Roberson return (tdset); 10433f289c3fSJeff Roberson } 10443f289c3fSJeff Roberson 10453f289c3fSJeff Roberson static int 10463f289c3fSJeff Roberson cpuset_setproc_test_maskthread(struct cpuset *tdset, cpuset_t *mask, 10473f289c3fSJeff Roberson struct domainset *domain) 10483f289c3fSJeff Roberson { 10493f289c3fSJeff Roberson struct cpuset *parent; 10503f289c3fSJeff Roberson 10513f289c3fSJeff Roberson parent = cpuset_getbase(tdset); 10523f289c3fSJeff Roberson if (mask == NULL) 10533f289c3fSJeff Roberson mask = &tdset->cs_mask; 10543f289c3fSJeff Roberson if (domain == NULL) 10553f289c3fSJeff Roberson domain = tdset->cs_domain; 10563f289c3fSJeff Roberson return cpuset_testshadow(parent, mask, domain); 10573f289c3fSJeff Roberson } 10583f289c3fSJeff Roberson 10593f289c3fSJeff Roberson static int 10603f289c3fSJeff Roberson cpuset_setproc_maskthread(struct cpuset *tdset, cpuset_t *mask, 10613f289c3fSJeff Roberson struct domainset *domain, struct cpuset **nsetp, 10623f289c3fSJeff Roberson struct setlist *freelist, struct domainlist *domainlist) 10633f289c3fSJeff Roberson { 10643f289c3fSJeff Roberson struct cpuset *parent; 10653f289c3fSJeff Roberson 10663f289c3fSJeff Roberson parent = cpuset_getbase(tdset); 10673f289c3fSJeff Roberson if (mask == NULL) 10683f289c3fSJeff Roberson mask = &tdset->cs_mask; 10693f289c3fSJeff Roberson if (domain == NULL) 10703f289c3fSJeff Roberson domain = tdset->cs_domain; 10713f289c3fSJeff Roberson return cpuset_shadow(parent, nsetp, mask, domain, freelist, 10723f289c3fSJeff Roberson domainlist); 10733f289c3fSJeff Roberson } 10743f289c3fSJeff Roberson 10753f289c3fSJeff Roberson static int 10763f289c3fSJeff Roberson cpuset_setproc_setthread_mask(struct cpuset *tdset, struct cpuset *set, 10773f289c3fSJeff Roberson cpuset_t *mask, struct domainset *domain) 10783f289c3fSJeff Roberson { 10793f289c3fSJeff Roberson struct cpuset *parent; 10803f289c3fSJeff Roberson 10813f289c3fSJeff Roberson parent = cpuset_getbase(tdset); 10823f289c3fSJeff Roberson 10833f289c3fSJeff Roberson /* 10843f289c3fSJeff Roberson * If the thread restricted its mask then apply that same 10853f289c3fSJeff Roberson * restriction to the new set, otherwise take it wholesale. 10863f289c3fSJeff Roberson */ 10873f289c3fSJeff Roberson if (CPU_CMP(&tdset->cs_mask, &parent->cs_mask) != 0) { 1088e2650af1SStefan Eßer CPU_AND(mask, &tdset->cs_mask, &set->cs_mask); 10893f289c3fSJeff Roberson } else 10903f289c3fSJeff Roberson CPU_COPY(&set->cs_mask, mask); 10913f289c3fSJeff Roberson 10923f289c3fSJeff Roberson /* 10933f289c3fSJeff Roberson * If the thread restricted the domain then we apply the 10943f289c3fSJeff Roberson * restriction to the new set but retain the policy. 10953f289c3fSJeff Roberson */ 10963f289c3fSJeff Roberson if (tdset->cs_domain != parent->cs_domain) { 10973f289c3fSJeff Roberson domainset_copy(tdset->cs_domain, domain); 10983f289c3fSJeff Roberson DOMAINSET_AND(&domain->ds_mask, &set->cs_domain->ds_mask); 10993f289c3fSJeff Roberson } else 11003f289c3fSJeff Roberson domainset_copy(set->cs_domain, domain); 11013f289c3fSJeff Roberson 11023f289c3fSJeff Roberson if (CPU_EMPTY(mask) || DOMAINSET_EMPTY(&domain->ds_mask)) 11033f289c3fSJeff Roberson return (EDEADLK); 11043f289c3fSJeff Roberson 11053f289c3fSJeff Roberson return (0); 11063f289c3fSJeff Roberson } 11073f289c3fSJeff Roberson 11083f289c3fSJeff Roberson static int 11093f289c3fSJeff Roberson cpuset_setproc_test_setthread(struct cpuset *tdset, struct cpuset *set) 11103f289c3fSJeff Roberson { 11113f289c3fSJeff Roberson struct domainset domain; 11123f289c3fSJeff Roberson cpuset_t mask; 11133f289c3fSJeff Roberson 11143f289c3fSJeff Roberson if (tdset->cs_id != CPUSET_INVALID) 11153f289c3fSJeff Roberson return (0); 11163f289c3fSJeff Roberson return cpuset_setproc_setthread_mask(tdset, set, &mask, &domain); 11173f289c3fSJeff Roberson } 11183f289c3fSJeff Roberson 11193f289c3fSJeff Roberson static int 11203f289c3fSJeff Roberson cpuset_setproc_setthread(struct cpuset *tdset, struct cpuset *set, 11213f289c3fSJeff Roberson struct cpuset **nsetp, struct setlist *freelist, 11223f289c3fSJeff Roberson struct domainlist *domainlist) 11233f289c3fSJeff Roberson { 11243f289c3fSJeff Roberson struct domainset domain; 11253f289c3fSJeff Roberson cpuset_t mask; 11263f289c3fSJeff Roberson int error; 11273f289c3fSJeff Roberson 11283f289c3fSJeff Roberson /* 11293f289c3fSJeff Roberson * If we're replacing on a thread that has not constrained the 11303f289c3fSJeff Roberson * original set we can simply accept the new set. 11313f289c3fSJeff Roberson */ 11323f289c3fSJeff Roberson if (tdset->cs_id != CPUSET_INVALID) { 11333f289c3fSJeff Roberson *nsetp = cpuset_ref(set); 11343f289c3fSJeff Roberson return (0); 11353f289c3fSJeff Roberson } 11363f289c3fSJeff Roberson error = cpuset_setproc_setthread_mask(tdset, set, &mask, &domain); 11373f289c3fSJeff Roberson if (error) 11383f289c3fSJeff Roberson return (error); 11393f289c3fSJeff Roberson 1140dac521ebSKyle Evans return cpuset_shadow(set, nsetp, &mask, &domain, freelist, 11413f289c3fSJeff Roberson domainlist); 11423f289c3fSJeff Roberson } 11433f289c3fSJeff Roberson 1144d431dea5SKyle Evans static int 1145d431dea5SKyle Evans cpuset_setproc_newbase(struct thread *td, struct cpuset *set, 1146d431dea5SKyle Evans struct cpuset *nroot, struct cpuset **nsetp, 1147d431dea5SKyle Evans struct setlist *cpusets, struct domainlist *domainlist) 1148d431dea5SKyle Evans { 1149d431dea5SKyle Evans struct domainset ndomain; 1150d431dea5SKyle Evans cpuset_t nmask; 1151d431dea5SKyle Evans struct cpuset *pbase; 1152d431dea5SKyle Evans int error; 1153d431dea5SKyle Evans 1154d431dea5SKyle Evans pbase = cpuset_getbase(td->td_cpuset); 1155d431dea5SKyle Evans 1156d431dea5SKyle Evans /* Copy process mask, then further apply the new root mask. */ 1157e2650af1SStefan Eßer CPU_AND(&nmask, &pbase->cs_mask, &nroot->cs_mask); 1158d431dea5SKyle Evans 1159d431dea5SKyle Evans domainset_copy(pbase->cs_domain, &ndomain); 1160d431dea5SKyle Evans DOMAINSET_AND(&ndomain.ds_mask, &set->cs_domain->ds_mask); 1161d431dea5SKyle Evans 1162d431dea5SKyle Evans /* Policy is too restrictive, will not work. */ 1163d431dea5SKyle Evans if (CPU_EMPTY(&nmask) || DOMAINSET_EMPTY(&ndomain.ds_mask)) 1164d431dea5SKyle Evans return (EDEADLK); 1165d431dea5SKyle Evans 11663f289c3fSJeff Roberson /* 1167d431dea5SKyle Evans * Remove pbase from the freelist in advance, it'll be pushed to 1168d431dea5SKyle Evans * cpuset_ids on success. We assume here that cpuset_create() will not 1169d431dea5SKyle Evans * touch pbase on failure, and we just enqueue it back to the freelist 1170d431dea5SKyle Evans * to remain in a consistent state. 1171d431dea5SKyle Evans */ 1172d431dea5SKyle Evans pbase = LIST_FIRST(cpusets); 1173d431dea5SKyle Evans LIST_REMOVE(pbase, cs_link); 1174d431dea5SKyle Evans error = cpuset_create(&pbase, set, &nmask); 1175d431dea5SKyle Evans if (error != 0) { 1176d431dea5SKyle Evans LIST_INSERT_HEAD(cpusets, pbase, cs_link); 1177d431dea5SKyle Evans return (error); 1178d431dea5SKyle Evans } 1179d431dea5SKyle Evans 1180d431dea5SKyle Evans /* Duplicates some work from above... oh well. */ 1181d431dea5SKyle Evans pbase->cs_domain = domainset_shadow(set->cs_domain, &ndomain, 1182d431dea5SKyle Evans domainlist); 1183d431dea5SKyle Evans *nsetp = pbase; 1184d431dea5SKyle Evans return (0); 1185d431dea5SKyle Evans } 1186d431dea5SKyle Evans 1187d431dea5SKyle Evans /* 1188d431dea5SKyle Evans * Handle four cases for updating an entire process. 1189d7f687fcSJeff Roberson * 1190d431dea5SKyle Evans * 1) Set is non-null and the process is not rebasing onto a new root. This 1191d431dea5SKyle Evans * reparents all anonymous sets to the provided set and replaces all 1192d431dea5SKyle Evans * non-anonymous td_cpusets with the provided set. 1193d431dea5SKyle Evans * 2) Set is non-null and the process is rebasing onto a new root. This 1194d431dea5SKyle Evans * creates a new base set if the process previously had its own base set, 1195d431dea5SKyle Evans * then reparents all anonymous sets either to that set or the provided set 1196d431dea5SKyle Evans * if one was not created. Non-anonymous sets are similarly replaced. 1197d431dea5SKyle Evans * 3) Mask is non-null. This replaces or creates anonymous sets for every 11983f289c3fSJeff Roberson * thread with the existing base as a parent. 1199d431dea5SKyle Evans * 4) domain is non-null. This creates anonymous sets for every thread 12003f289c3fSJeff Roberson * and replaces the domain set. 1201d7f687fcSJeff Roberson * 1202d7f687fcSJeff Roberson * This is overly complicated because we can't allocate while holding a 1203d7f687fcSJeff Roberson * spinlock and spinlocks must be held while changing and examining thread 1204d7f687fcSJeff Roberson * state. 1205d7f687fcSJeff Roberson */ 1206d7f687fcSJeff Roberson static int 12073f289c3fSJeff Roberson cpuset_setproc(pid_t pid, struct cpuset *set, cpuset_t *mask, 1208d431dea5SKyle Evans struct domainset *domain, bool rebase) 1209d7f687fcSJeff Roberson { 1210d7f687fcSJeff Roberson struct setlist freelist; 1211d7f687fcSJeff Roberson struct setlist droplist; 12123f289c3fSJeff Roberson struct domainlist domainlist; 1213d431dea5SKyle Evans struct cpuset *base, *nset, *nroot, *tdroot; 1214d7f687fcSJeff Roberson struct thread *td; 1215d7f687fcSJeff Roberson struct proc *p; 1216d431dea5SKyle Evans int needed; 1217d7f687fcSJeff Roberson int nfree; 1218d7f687fcSJeff Roberson int error; 1219f299c47bSAllan Jude 1220d7f687fcSJeff Roberson /* 1221d7f687fcSJeff Roberson * The algorithm requires two passes due to locking considerations. 1222d7f687fcSJeff Roberson * 1223d7f687fcSJeff Roberson * 1) Lookup the process and acquire the locks in the required order. 1224d7f687fcSJeff Roberson * 2) If enough cpusets have not been allocated release the locks and 1225d7f687fcSJeff Roberson * allocate them. Loop. 1226d7f687fcSJeff Roberson */ 12273f289c3fSJeff Roberson cpuset_freelist_init(&freelist, 1); 12283f289c3fSJeff Roberson domainset_freelist_init(&domainlist, 1); 12293f289c3fSJeff Roberson nfree = 1; 1230d7f687fcSJeff Roberson LIST_INIT(&droplist); 1231d7f687fcSJeff Roberson nfree = 0; 1232d431dea5SKyle Evans base = set; 1233d431dea5SKyle Evans nroot = NULL; 1234d431dea5SKyle Evans if (set != NULL) 1235d431dea5SKyle Evans nroot = cpuset_getroot(set); 1236d7f687fcSJeff Roberson for (;;) { 1237d7f687fcSJeff Roberson error = cpuset_which(CPU_WHICH_PID, pid, &p, &td, &nset); 1238d7f687fcSJeff Roberson if (error) 1239d7f687fcSJeff Roberson goto out; 1240d431dea5SKyle Evans tdroot = cpuset_getroot(td->td_cpuset); 1241d431dea5SKyle Evans needed = p->p_numthreads; 1242d431dea5SKyle Evans if (set != NULL && rebase && tdroot != nroot) 1243d431dea5SKyle Evans needed++; 1244d431dea5SKyle Evans if (nfree >= needed) 1245d7f687fcSJeff Roberson break; 1246d7f687fcSJeff Roberson PROC_UNLOCK(p); 1247d431dea5SKyle Evans if (nfree < needed) { 1248d431dea5SKyle Evans cpuset_freelist_add(&freelist, needed - nfree); 1249d431dea5SKyle Evans domainset_freelist_add(&domainlist, needed - nfree); 1250d431dea5SKyle Evans nfree = needed; 1251d7f687fcSJeff Roberson } 1252d7f687fcSJeff Roberson } 1253d7f687fcSJeff Roberson PROC_LOCK_ASSERT(p, MA_OWNED); 1254d431dea5SKyle Evans 1255d431dea5SKyle Evans /* 1256d431dea5SKyle Evans * If we're changing roots and the root set is what has been specified 1257d431dea5SKyle Evans * as the parent, then we'll check if the process was previously using 1258d431dea5SKyle Evans * the root set and, if it wasn't, create a new base with the process's 1259d431dea5SKyle Evans * mask applied to it. 126060c4ec80SKyle Evans * 126160c4ec80SKyle Evans * If the new root is incompatible with the existing mask, then we allow 126260c4ec80SKyle Evans * the process to take on the new root if and only if they have 126360c4ec80SKyle Evans * privilege to widen their mask anyways. Unprivileged processes get 126460c4ec80SKyle Evans * rejected with EDEADLK. 1265d431dea5SKyle Evans */ 1266d431dea5SKyle Evans if (set != NULL && rebase && nroot != tdroot) { 1267d431dea5SKyle Evans cpusetid_t base_id, root_id; 1268d431dea5SKyle Evans 1269d431dea5SKyle Evans root_id = td->td_ucred->cr_prison->pr_cpuset->cs_id; 1270d431dea5SKyle Evans base_id = cpuset_getbase(td->td_cpuset)->cs_id; 1271d431dea5SKyle Evans 1272d431dea5SKyle Evans if (base_id != root_id) { 1273d431dea5SKyle Evans error = cpuset_setproc_newbase(td, set, nroot, &base, 1274d431dea5SKyle Evans &freelist, &domainlist); 127560c4ec80SKyle Evans if (error == EDEADLK && 127660c4ec80SKyle Evans priv_check(td, PRIV_SCHED_CPUSET) == 0) 127760c4ec80SKyle Evans error = 0; 1278d431dea5SKyle Evans if (error != 0) 1279d431dea5SKyle Evans goto unlock_out; 1280d431dea5SKyle Evans } 1281d431dea5SKyle Evans } 1282d431dea5SKyle Evans 1283d7f687fcSJeff Roberson /* 1284d7f687fcSJeff Roberson * Now that the appropriate locks are held and we have enough cpusets, 128573c40187SJeff Roberson * make sure the operation will succeed before applying changes. The 128673c40187SJeff Roberson * proc lock prevents td_cpuset from changing between calls. 1287d7f687fcSJeff Roberson */ 1288d7f687fcSJeff Roberson error = 0; 1289d7f687fcSJeff Roberson FOREACH_THREAD_IN_PROC(p, td) { 129073c40187SJeff Roberson thread_lock(td); 12913f289c3fSJeff Roberson if (set != NULL) 12923f289c3fSJeff Roberson error = cpuset_setproc_test_setthread(td->td_cpuset, 1293d431dea5SKyle Evans base); 12943f289c3fSJeff Roberson else 12953f289c3fSJeff Roberson error = cpuset_setproc_test_maskthread(td->td_cpuset, 12963f289c3fSJeff Roberson mask, domain); 129773c40187SJeff Roberson thread_unlock(td); 129873c40187SJeff Roberson if (error) 129973c40187SJeff Roberson goto unlock_out; 130073c40187SJeff Roberson } 130173c40187SJeff Roberson /* 130273c40187SJeff Roberson * Replace each thread's cpuset while using deferred release. We 1303374ae2a3SJeff Roberson * must do this because the thread lock must be held while operating 1304374ae2a3SJeff Roberson * on the thread and this limits the type of operations allowed. 130573c40187SJeff Roberson */ 130673c40187SJeff Roberson FOREACH_THREAD_IN_PROC(p, td) { 1307d7f687fcSJeff Roberson thread_lock(td); 13083f289c3fSJeff Roberson if (set != NULL) 1309d431dea5SKyle Evans error = cpuset_setproc_setthread(td->td_cpuset, base, 13103f289c3fSJeff Roberson &nset, &freelist, &domainlist); 1311d7f687fcSJeff Roberson else 13123f289c3fSJeff Roberson error = cpuset_setproc_maskthread(td->td_cpuset, mask, 13133f289c3fSJeff Roberson domain, &nset, &freelist, &domainlist); 1314d7f687fcSJeff Roberson if (error) { 1315d7f687fcSJeff Roberson thread_unlock(td); 1316d7f687fcSJeff Roberson break; 1317d7f687fcSJeff Roberson } 13183f289c3fSJeff Roberson cpuset_rel_defer(&droplist, cpuset_update_thread(td, nset)); 1319d7f687fcSJeff Roberson thread_unlock(td); 1320d7f687fcSJeff Roberson } 132173c40187SJeff Roberson unlock_out: 1322d7f687fcSJeff Roberson PROC_UNLOCK(p); 1323d7f687fcSJeff Roberson out: 1324d431dea5SKyle Evans if (base != NULL && base != set) 1325d431dea5SKyle Evans cpuset_rel(base); 1326d7f687fcSJeff Roberson while ((nset = LIST_FIRST(&droplist)) != NULL) 1327d7f687fcSJeff Roberson cpuset_rel_complete(nset); 13283f289c3fSJeff Roberson cpuset_freelist_free(&freelist); 13293f289c3fSJeff Roberson domainset_freelist_free(&domainlist); 1330d7f687fcSJeff Roberson return (error); 1331d7f687fcSJeff Roberson } 1332d7f687fcSJeff Roberson 1333e5818a53SJeff Roberson static int 1334e5818a53SJeff Roberson bitset_strprint(char *buf, size_t bufsiz, const struct bitset *set, int setlen) 1335e5818a53SJeff Roberson { 1336e5818a53SJeff Roberson size_t bytes; 1337e5818a53SJeff Roberson int i, once; 1338e5818a53SJeff Roberson char *p; 1339e5818a53SJeff Roberson 1340e5818a53SJeff Roberson once = 0; 1341e5818a53SJeff Roberson p = buf; 1342e5818a53SJeff Roberson for (i = 0; i < __bitset_words(setlen); i++) { 1343e5818a53SJeff Roberson if (once != 0) { 1344e5818a53SJeff Roberson if (bufsiz < 1) 1345e5818a53SJeff Roberson return (0); 1346e5818a53SJeff Roberson *p = ','; 1347e5818a53SJeff Roberson p++; 1348e5818a53SJeff Roberson bufsiz--; 1349e5818a53SJeff Roberson } else 1350e5818a53SJeff Roberson once = 1; 1351e5818a53SJeff Roberson if (bufsiz < sizeof(__STRING(ULONG_MAX))) 1352e5818a53SJeff Roberson return (0); 1353e5818a53SJeff Roberson bytes = snprintf(p, bufsiz, "%lx", set->__bits[i]); 1354e5818a53SJeff Roberson p += bytes; 1355e5818a53SJeff Roberson bufsiz -= bytes; 1356e5818a53SJeff Roberson } 1357e5818a53SJeff Roberson return (p - buf); 1358e5818a53SJeff Roberson } 1359e5818a53SJeff Roberson 1360e5818a53SJeff Roberson static int 1361e5818a53SJeff Roberson bitset_strscan(struct bitset *set, int setlen, const char *buf) 1362e5818a53SJeff Roberson { 1363e5818a53SJeff Roberson int i, ret; 1364e5818a53SJeff Roberson const char *p; 1365e5818a53SJeff Roberson 1366e5818a53SJeff Roberson BIT_ZERO(setlen, set); 1367e5818a53SJeff Roberson p = buf; 1368e5818a53SJeff Roberson for (i = 0; i < __bitset_words(setlen); i++) { 1369e5818a53SJeff Roberson if (*p == ',') { 1370e5818a53SJeff Roberson p++; 1371e5818a53SJeff Roberson continue; 1372e5818a53SJeff Roberson } 1373e5818a53SJeff Roberson ret = sscanf(p, "%lx", &set->__bits[i]); 1374e5818a53SJeff Roberson if (ret == 0 || ret == -1) 1375e5818a53SJeff Roberson break; 1376e5818a53SJeff Roberson while (isxdigit(*p)) 1377e5818a53SJeff Roberson p++; 1378e5818a53SJeff Roberson } 1379e5818a53SJeff Roberson return (p - buf); 1380e5818a53SJeff Roberson } 1381e5818a53SJeff Roberson 1382d7f687fcSJeff Roberson /* 138371a19bdcSAttilio Rao * Return a string representing a valid layout for a cpuset_t object. 138471a19bdcSAttilio Rao * It expects an incoming buffer at least sized as CPUSETBUFSIZ. 138571a19bdcSAttilio Rao */ 138671a19bdcSAttilio Rao char * 138771a19bdcSAttilio Rao cpusetobj_strprint(char *buf, const cpuset_t *set) 138871a19bdcSAttilio Rao { 138971a19bdcSAttilio Rao 1390e5818a53SJeff Roberson bitset_strprint(buf, CPUSETBUFSIZ, (const struct bitset *)set, 1391e5818a53SJeff Roberson CPU_SETSIZE); 139271a19bdcSAttilio Rao return (buf); 139371a19bdcSAttilio Rao } 139471a19bdcSAttilio Rao 139571a19bdcSAttilio Rao /* 1396e3709597SAttilio Rao * Build a valid cpuset_t object from a string representation. 1397e3709597SAttilio Rao * It expects an incoming buffer at least sized as CPUSETBUFSIZ. 1398e3709597SAttilio Rao */ 1399e3709597SAttilio Rao int 1400e3709597SAttilio Rao cpusetobj_strscan(cpuset_t *set, const char *buf) 1401e3709597SAttilio Rao { 1402e5818a53SJeff Roberson char p; 1403e3709597SAttilio Rao 1404e3709597SAttilio Rao if (strlen(buf) > CPUSETBUFSIZ - 1) 1405e3709597SAttilio Rao return (-1); 1406e3709597SAttilio Rao 1407e5818a53SJeff Roberson p = buf[bitset_strscan((struct bitset *)set, CPU_SETSIZE, buf)]; 1408e5818a53SJeff Roberson if (p != '\0') 1409e3709597SAttilio Rao return (-1); 1410e3709597SAttilio Rao 1411e3709597SAttilio Rao return (0); 1412e3709597SAttilio Rao } 1413e3709597SAttilio Rao 1414e3709597SAttilio Rao /* 1415e5818a53SJeff Roberson * Handle a domainset specifier in the sysctl tree. A poiner to a pointer to 1416e5818a53SJeff Roberson * a domainset is in arg1. If the user specifies a valid domainset the 1417e5818a53SJeff Roberson * pointer is updated. 1418e5818a53SJeff Roberson * 1419e5818a53SJeff Roberson * Format is: 1420e5818a53SJeff Roberson * hex mask word 0,hex mask word 1,...:decimal policy:decimal preferred 1421e5818a53SJeff Roberson */ 1422e5818a53SJeff Roberson int 1423e5818a53SJeff Roberson sysctl_handle_domainset(SYSCTL_HANDLER_ARGS) 1424e5818a53SJeff Roberson { 1425e5818a53SJeff Roberson char buf[DOMAINSETBUFSIZ]; 1426e5818a53SJeff Roberson struct domainset *dset; 1427e5818a53SJeff Roberson struct domainset key; 1428e5818a53SJeff Roberson int policy, prefer, error; 1429e5818a53SJeff Roberson char *p; 1430e5818a53SJeff Roberson 1431e5818a53SJeff Roberson dset = *(struct domainset **)arg1; 1432e5818a53SJeff Roberson error = 0; 1433e5818a53SJeff Roberson 1434e5818a53SJeff Roberson if (dset != NULL) { 1435e5818a53SJeff Roberson p = buf + bitset_strprint(buf, DOMAINSETBUFSIZ, 1436e5818a53SJeff Roberson (const struct bitset *)&dset->ds_mask, DOMAINSET_SETSIZE); 1437e5818a53SJeff Roberson sprintf(p, ":%d:%d", dset->ds_policy, dset->ds_prefer); 1438e5818a53SJeff Roberson } else 1439e5818a53SJeff Roberson sprintf(buf, "<NULL>"); 1440e5818a53SJeff Roberson error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 1441e5818a53SJeff Roberson if (error != 0 || req->newptr == NULL) 1442e5818a53SJeff Roberson return (error); 1443e5818a53SJeff Roberson 1444e5818a53SJeff Roberson /* 1445e5818a53SJeff Roberson * Read in and validate the string. 1446e5818a53SJeff Roberson */ 1447e5818a53SJeff Roberson memset(&key, 0, sizeof(key)); 1448e5818a53SJeff Roberson p = &buf[bitset_strscan((struct bitset *)&key.ds_mask, 1449e5818a53SJeff Roberson DOMAINSET_SETSIZE, buf)]; 1450e5818a53SJeff Roberson if (p == buf) 1451e5818a53SJeff Roberson return (EINVAL); 1452e5818a53SJeff Roberson if (sscanf(p, ":%d:%d", &policy, &prefer) != 2) 1453e5818a53SJeff Roberson return (EINVAL); 1454e5818a53SJeff Roberson key.ds_policy = policy; 1455e5818a53SJeff Roberson key.ds_prefer = prefer; 1456e5818a53SJeff Roberson 1457e5818a53SJeff Roberson /* Domainset_create() validates the policy.*/ 1458e5818a53SJeff Roberson dset = domainset_create(&key); 1459e5818a53SJeff Roberson if (dset == NULL) 1460e5818a53SJeff Roberson return (EINVAL); 1461e5818a53SJeff Roberson *(struct domainset **)arg1 = dset; 1462e5818a53SJeff Roberson 1463e5818a53SJeff Roberson return (error); 1464e5818a53SJeff Roberson } 1465e5818a53SJeff Roberson 1466e5818a53SJeff Roberson /* 14673f289c3fSJeff Roberson * Apply an anonymous mask or a domain to a single thread. 1468d7f687fcSJeff Roberson */ 14693f289c3fSJeff Roberson static int 14703f289c3fSJeff Roberson _cpuset_setthread(lwpid_t id, cpuset_t *mask, struct domainset *domain) 1471d7f687fcSJeff Roberson { 14723f289c3fSJeff Roberson struct setlist cpusets; 14733f289c3fSJeff Roberson struct domainlist domainlist; 1474d7f687fcSJeff Roberson struct cpuset *nset; 1475d7f687fcSJeff Roberson struct cpuset *set; 1476d7f687fcSJeff Roberson struct thread *td; 1477d7f687fcSJeff Roberson struct proc *p; 1478d7f687fcSJeff Roberson int error; 1479d7f687fcSJeff Roberson 14803f289c3fSJeff Roberson cpuset_freelist_init(&cpusets, 1); 14813f289c3fSJeff Roberson domainset_freelist_init(&domainlist, domain != NULL); 14828bd75bddSJeff Roberson error = cpuset_which(CPU_WHICH_TID, id, &p, &td, &set); 1483d7f687fcSJeff Roberson if (error) 1484d7f687fcSJeff Roberson goto out; 1485a03ee000SJeff Roberson set = NULL; 1486d7f687fcSJeff Roberson thread_lock(td); 14873f289c3fSJeff Roberson error = cpuset_shadow(td->td_cpuset, &nset, mask, domain, 14883f289c3fSJeff Roberson &cpusets, &domainlist); 14893f289c3fSJeff Roberson if (error == 0) 14903f289c3fSJeff Roberson set = cpuset_update_thread(td, nset); 1491d7f687fcSJeff Roberson thread_unlock(td); 1492d7f687fcSJeff Roberson PROC_UNLOCK(p); 1493a03ee000SJeff Roberson if (set) 1494a03ee000SJeff Roberson cpuset_rel(set); 1495d7f687fcSJeff Roberson out: 14963f289c3fSJeff Roberson cpuset_freelist_free(&cpusets); 14973f289c3fSJeff Roberson domainset_freelist_free(&domainlist); 1498d7f687fcSJeff Roberson return (error); 1499d7f687fcSJeff Roberson } 1500d7f687fcSJeff Roberson 1501d7f687fcSJeff Roberson /* 15023f289c3fSJeff Roberson * Apply an anonymous mask to a single thread. 15033f289c3fSJeff Roberson */ 15043f289c3fSJeff Roberson int 15053f289c3fSJeff Roberson cpuset_setthread(lwpid_t id, cpuset_t *mask) 15063f289c3fSJeff Roberson { 15073f289c3fSJeff Roberson 15083f289c3fSJeff Roberson return _cpuset_setthread(id, mask, NULL); 15093f289c3fSJeff Roberson } 15103f289c3fSJeff Roberson 15113f289c3fSJeff Roberson /* 151281198539SAlexander V. Chernikov * Apply new cpumask to the ithread. 151381198539SAlexander V. Chernikov */ 151481198539SAlexander V. Chernikov int 15157f7528fcSAdrian Chadd cpuset_setithread(lwpid_t id, int cpu) 151681198539SAlexander V. Chernikov { 151781198539SAlexander V. Chernikov cpuset_t mask; 151881198539SAlexander V. Chernikov 151981198539SAlexander V. Chernikov CPU_ZERO(&mask); 152081198539SAlexander V. Chernikov if (cpu == NOCPU) 152181198539SAlexander V. Chernikov CPU_COPY(cpuset_root, &mask); 152281198539SAlexander V. Chernikov else 152381198539SAlexander V. Chernikov CPU_SET(cpu, &mask); 1524e5818a53SJeff Roberson return _cpuset_setthread(id, &mask, NULL); 1525e5818a53SJeff Roberson } 1526c1d9ecf2SAlexander V. Chernikov 152781198539SAlexander V. Chernikov /* 1528662e7fa8SMark Johnston * Initialize static domainsets after NUMA information is available. This is 1529920239efSMark Johnston * called before memory allocators are initialized. 1530662e7fa8SMark Johnston */ 1531662e7fa8SMark Johnston void 1532662e7fa8SMark Johnston domainset_init(void) 1533662e7fa8SMark Johnston { 1534662e7fa8SMark Johnston struct domainset *dset; 1535662e7fa8SMark Johnston int i; 1536662e7fa8SMark Johnston 153729bb6c19SMark Johnston dset = &domainset_firsttouch; 153829bb6c19SMark Johnston DOMAINSET_COPY(&all_domains, &dset->ds_mask); 153929bb6c19SMark Johnston dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH; 154029bb6c19SMark Johnston dset->ds_prefer = -1; 154129bb6c19SMark Johnston _domainset_create(dset, NULL); 154229bb6c19SMark Johnston 154329bb6c19SMark Johnston dset = &domainset_interleave; 154429bb6c19SMark Johnston DOMAINSET_COPY(&all_domains, &dset->ds_mask); 154529bb6c19SMark Johnston dset->ds_policy = DOMAINSET_POLICY_INTERLEAVE; 154629bb6c19SMark Johnston dset->ds_prefer = -1; 154729bb6c19SMark Johnston _domainset_create(dset, NULL); 154829bb6c19SMark Johnston 1549662e7fa8SMark Johnston dset = &domainset_roundrobin; 1550662e7fa8SMark Johnston DOMAINSET_COPY(&all_domains, &dset->ds_mask); 1551662e7fa8SMark Johnston dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; 1552662e7fa8SMark Johnston dset->ds_prefer = -1; 1553662e7fa8SMark Johnston _domainset_create(dset, NULL); 1554662e7fa8SMark Johnston 1555662e7fa8SMark Johnston for (i = 0; i < vm_ndomains; i++) { 15569978bd99SMark Johnston dset = &domainset_fixed[i]; 15579978bd99SMark Johnston DOMAINSET_ZERO(&dset->ds_mask); 15589978bd99SMark Johnston DOMAINSET_SET(i, &dset->ds_mask); 15599978bd99SMark Johnston dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; 15609978bd99SMark Johnston _domainset_create(dset, NULL); 15619978bd99SMark Johnston 1562662e7fa8SMark Johnston dset = &domainset_prefer[i]; 1563662e7fa8SMark Johnston DOMAINSET_COPY(&all_domains, &dset->ds_mask); 1564662e7fa8SMark Johnston dset->ds_policy = DOMAINSET_POLICY_PREFER; 1565662e7fa8SMark Johnston dset->ds_prefer = i; 1566662e7fa8SMark Johnston _domainset_create(dset, NULL); 1567662e7fa8SMark Johnston } 1568662e7fa8SMark Johnston } 1569662e7fa8SMark Johnston 1570662e7fa8SMark Johnston /* 157129bb6c19SMark Johnston * Define the domainsets for cpuset 0, 1 and cpuset 2. 157281198539SAlexander V. Chernikov */ 15733f289c3fSJeff Roberson void 15743f289c3fSJeff Roberson domainset_zero(void) 15753f289c3fSJeff Roberson { 1576920239efSMark Johnston struct domainset *dset, *tmp; 15773f289c3fSJeff Roberson 15783f289c3fSJeff Roberson mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE); 15793f289c3fSJeff Roberson 158029bb6c19SMark Johnston domainset0 = &domainset_firsttouch; 158129bb6c19SMark Johnston curthread->td_domain.dr_policy = domainset0; 1582e5818a53SJeff Roberson 158329bb6c19SMark Johnston domainset2 = &domainset_interleave; 158429bb6c19SMark Johnston kernel_object->domain.dr_policy = domainset2; 1585662e7fa8SMark Johnston 1586662e7fa8SMark Johnston /* Remove empty domains from the global policies. */ 1587920239efSMark Johnston LIST_FOREACH_SAFE(dset, &cpuset_domains, ds_link, tmp) 1588920239efSMark Johnston if (domainset_empty_vm(dset)) 1589920239efSMark Johnston LIST_REMOVE(dset, ds_link); 15903f289c3fSJeff Roberson } 159181198539SAlexander V. Chernikov 159281198539SAlexander V. Chernikov /* 1593e5818a53SJeff Roberson * Creates system-wide cpusets and the cpuset for thread0 including three 1594c0ae6688SJohn Baldwin * sets: 1595d7f687fcSJeff Roberson * 1596d7f687fcSJeff Roberson * 0 - The root set which should represent all valid processors in the 159730b7c6f9SKyle Evans * system. This set is immutable. 1598d7f687fcSJeff Roberson * 1 - The default set which all processes are a member of until changed. 1599d7f687fcSJeff Roberson * This allows an administrator to move all threads off of given cpus to 1600d7f687fcSJeff Roberson * dedicate them to high priority tasks or save power etc. 1601e5818a53SJeff Roberson * 2 - The kernel set which allows restriction and policy to be applied only 1602e5818a53SJeff Roberson * to kernel threads and the kernel_object. 1603d7f687fcSJeff Roberson */ 1604d7f687fcSJeff Roberson struct cpuset * 1605d7f687fcSJeff Roberson cpuset_thread0(void) 1606d7f687fcSJeff Roberson { 1607d7f687fcSJeff Roberson struct cpuset *set; 160827a3c9d7SJeff Roberson int i; 1609a6c7423aSMatt Macy int error __unused; 1610d7f687fcSJeff Roberson 1611d7f687fcSJeff Roberson cpuset_zone = uma_zcreate("cpuset", sizeof(struct cpuset), NULL, NULL, 1612e5818a53SJeff Roberson NULL, NULL, UMA_ALIGN_CACHE, 0); 16133f289c3fSJeff Roberson domainset_zone = uma_zcreate("domainset", sizeof(struct domainset), 1614e5818a53SJeff Roberson NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 161581198539SAlexander V. Chernikov 1616d7f687fcSJeff Roberson /* 1617e5818a53SJeff Roberson * Create the root system set (0) for the whole machine. Doesn't use 1618d7f687fcSJeff Roberson * cpuset_create() due to NULL parent. 1619d7f687fcSJeff Roberson */ 1620d7f687fcSJeff Roberson set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 162127a3c9d7SJeff Roberson CPU_COPY(&all_cpus, &set->cs_mask); 1622d7f687fcSJeff Roberson LIST_INIT(&set->cs_children); 1623d7f687fcSJeff Roberson LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 16241a7bb896SMateusz Guzik refcount_init(&set->cs_ref, 1); 162527a3c9d7SJeff Roberson set->cs_flags = CPU_SET_ROOT | CPU_SET_RDONLY; 162629bb6c19SMark Johnston set->cs_domain = domainset0; 1627d7f687fcSJeff Roberson cpuset_zero = set; 1628a03ee000SJeff Roberson cpuset_root = &set->cs_mask; 162981198539SAlexander V. Chernikov 1630d7f687fcSJeff Roberson /* 1631e5818a53SJeff Roberson * Now derive a default (1), modifiable set from that to give out. 1632d7f687fcSJeff Roberson */ 16333f289c3fSJeff Roberson set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 163430b7c6f9SKyle Evans error = cpuset_init(set, cpuset_zero, NULL, NULL, 1); 1635d7f687fcSJeff Roberson KASSERT(error == 0, ("Error creating default set: %d\n", error)); 163681198539SAlexander V. Chernikov cpuset_default = set; 1637e5818a53SJeff Roberson /* 1638e5818a53SJeff Roberson * Create the kernel set (2). 1639e5818a53SJeff Roberson */ 1640e5818a53SJeff Roberson set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 164130b7c6f9SKyle Evans error = cpuset_init(set, cpuset_zero, NULL, NULL, 2); 1642e5818a53SJeff Roberson KASSERT(error == 0, ("Error creating kernel set: %d\n", error)); 164329bb6c19SMark Johnston set->cs_domain = domainset2; 1644e5818a53SJeff Roberson cpuset_kernel = set; 164581198539SAlexander V. Chernikov 1646d7f687fcSJeff Roberson /* 1647d7f687fcSJeff Roberson * Initialize the unit allocator. 0 and 1 are allocated above. 1648d7f687fcSJeff Roberson */ 1649d57cd5ccSStephen J. Kiernan cpuset_unr = new_unrhdr(3, INT_MAX, NULL); 1650d7f687fcSJeff Roberson 165127a3c9d7SJeff Roberson /* 165227a3c9d7SJeff Roberson * If MD code has not initialized per-domain cpusets, place all 165327a3c9d7SJeff Roberson * CPUs in domain 0. 165427a3c9d7SJeff Roberson */ 165527a3c9d7SJeff Roberson for (i = 0; i < MAXMEMDOM; i++) 165627a3c9d7SJeff Roberson if (!CPU_EMPTY(&cpuset_domain[i])) 165727a3c9d7SJeff Roberson goto domains_set; 165827a3c9d7SJeff Roberson CPU_COPY(&all_cpus, &cpuset_domain[0]); 165927a3c9d7SJeff Roberson domains_set: 166027a3c9d7SJeff Roberson 1661e5818a53SJeff Roberson return (cpuset_default); 1662e5818a53SJeff Roberson } 1663e5818a53SJeff Roberson 1664e5818a53SJeff Roberson void 1665e5818a53SJeff Roberson cpuset_kernthread(struct thread *td) 1666e5818a53SJeff Roberson { 1667e5818a53SJeff Roberson struct cpuset *set; 1668e5818a53SJeff Roberson 1669e5818a53SJeff Roberson thread_lock(td); 1670e5818a53SJeff Roberson set = td->td_cpuset; 1671e5818a53SJeff Roberson td->td_cpuset = cpuset_ref(cpuset_kernel); 1672e5818a53SJeff Roberson thread_unlock(td); 1673e5818a53SJeff Roberson cpuset_rel(set); 1674d7f687fcSJeff Roberson } 1675d7f687fcSJeff Roberson 1676d7f687fcSJeff Roberson /* 1677413628a7SBjoern A. Zeeb * Create a cpuset, which would be cpuset_create() but 1678413628a7SBjoern A. Zeeb * mark the new 'set' as root. 1679413628a7SBjoern A. Zeeb * 168047479a8cSBjoern A. Zeeb * We are not going to reparent the td to it. Use cpuset_setproc_update_set() 168147479a8cSBjoern A. Zeeb * for that. 1682413628a7SBjoern A. Zeeb * 1683413628a7SBjoern A. Zeeb * In case of no error, returns the set in *setp locked with a reference. 1684413628a7SBjoern A. Zeeb */ 1685413628a7SBjoern A. Zeeb int 16860304c731SJamie Gritton cpuset_create_root(struct prison *pr, struct cpuset **setp) 1687413628a7SBjoern A. Zeeb { 1688413628a7SBjoern A. Zeeb struct cpuset *set; 1689413628a7SBjoern A. Zeeb int error; 1690413628a7SBjoern A. Zeeb 16910304c731SJamie Gritton KASSERT(pr != NULL, ("[%s:%d] invalid pr", __func__, __LINE__)); 1692413628a7SBjoern A. Zeeb KASSERT(setp != NULL, ("[%s:%d] invalid setp", __func__, __LINE__)); 1693413628a7SBjoern A. Zeeb 169429d04ea8SKyle Evans set = NULL; 169529d04ea8SKyle Evans error = cpuset_create(&set, pr->pr_cpuset, &pr->pr_cpuset->cs_mask); 1696413628a7SBjoern A. Zeeb if (error) 1697413628a7SBjoern A. Zeeb return (error); 1698413628a7SBjoern A. Zeeb 169929d04ea8SKyle Evans KASSERT(set != NULL, ("[%s:%d] cpuset_create returned invalid data", 1700413628a7SBjoern A. Zeeb __func__, __LINE__)); 1701413628a7SBjoern A. Zeeb 1702413628a7SBjoern A. Zeeb /* Mark the set as root. */ 1703413628a7SBjoern A. Zeeb set->cs_flags |= CPU_SET_ROOT; 170429d04ea8SKyle Evans *setp = set; 1705413628a7SBjoern A. Zeeb 1706413628a7SBjoern A. Zeeb return (0); 1707413628a7SBjoern A. Zeeb } 1708413628a7SBjoern A. Zeeb 1709413628a7SBjoern A. Zeeb int 1710413628a7SBjoern A. Zeeb cpuset_setproc_update_set(struct proc *p, struct cpuset *set) 1711413628a7SBjoern A. Zeeb { 1712413628a7SBjoern A. Zeeb int error; 1713413628a7SBjoern A. Zeeb 1714413628a7SBjoern A. Zeeb KASSERT(p != NULL, ("[%s:%d] invalid proc", __func__, __LINE__)); 1715413628a7SBjoern A. Zeeb KASSERT(set != NULL, ("[%s:%d] invalid set", __func__, __LINE__)); 1716413628a7SBjoern A. Zeeb 1717413628a7SBjoern A. Zeeb cpuset_ref(set); 1718d431dea5SKyle Evans error = cpuset_setproc(p->p_pid, set, NULL, NULL, true); 1719413628a7SBjoern A. Zeeb if (error) 1720413628a7SBjoern A. Zeeb return (error); 1721413628a7SBjoern A. Zeeb cpuset_rel(set); 1722413628a7SBjoern A. Zeeb return (0); 1723413628a7SBjoern A. Zeeb } 1724413628a7SBjoern A. Zeeb 17259eb997cbSMark Johnston /* 17269eb997cbSMark Johnston * In Capability mode, the only accesses that are permitted are to the current 17279eb997cbSMark Johnston * thread and process' CPU and domain sets. 17289eb997cbSMark Johnston */ 17299eb997cbSMark Johnston static int 17309eb997cbSMark Johnston cpuset_check_capabilities(struct thread *td, cpulevel_t level, cpuwhich_t which, 17319eb997cbSMark Johnston id_t id) 17329eb997cbSMark Johnston { 17339eb997cbSMark Johnston if (IN_CAPABILITY_MODE(td)) { 17349eb997cbSMark Johnston if (level != CPU_LEVEL_WHICH) 17359eb997cbSMark Johnston return (ECAPMODE); 17369eb997cbSMark Johnston if (which != CPU_WHICH_TID && which != CPU_WHICH_PID) 17379eb997cbSMark Johnston return (ECAPMODE); 173869b565d7SMark Johnston if (id != -1 && 173969b565d7SMark Johnston !(which == CPU_WHICH_TID && id == td->td_tid) && 174069b565d7SMark Johnston !(which == CPU_WHICH_PID && id == td->td_proc->p_pid)) 17419eb997cbSMark Johnston return (ECAPMODE); 17429eb997cbSMark Johnston } 17439eb997cbSMark Johnston return (0); 17449eb997cbSMark Johnston } 17459eb997cbSMark Johnston 174647a57144SJustin Hibbits static const struct cpuset_copy_cb copy_set = { 17474a3e5133SMark Johnston .cpuset_copyin = copyin, 17484a3e5133SMark Johnston .cpuset_copyout = copyout 174947a57144SJustin Hibbits }; 175047a57144SJustin Hibbits 1751d7f687fcSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 1752d7f687fcSJeff Roberson struct cpuset_args { 1753d7f687fcSJeff Roberson cpusetid_t *setid; 1754d7f687fcSJeff Roberson }; 1755d7f687fcSJeff Roberson #endif 1756d7f687fcSJeff Roberson int 17578451d0ddSKip Macy sys_cpuset(struct thread *td, struct cpuset_args *uap) 1758d7f687fcSJeff Roberson { 1759d7f687fcSJeff Roberson struct cpuset *root; 1760d7f687fcSJeff Roberson struct cpuset *set; 1761d7f687fcSJeff Roberson int error; 1762d7f687fcSJeff Roberson 1763d7f687fcSJeff Roberson thread_lock(td); 1764a03ee000SJeff Roberson root = cpuset_refroot(td->td_cpuset); 1765d7f687fcSJeff Roberson thread_unlock(td); 176629d04ea8SKyle Evans set = NULL; 1767d7f687fcSJeff Roberson error = cpuset_create(&set, root, &root->cs_mask); 1768d7f687fcSJeff Roberson cpuset_rel(root); 1769d7f687fcSJeff Roberson if (error) 1770d7f687fcSJeff Roberson return (error); 1771d7f687fcSJeff Roberson error = copyout(&set->cs_id, uap->setid, sizeof(set->cs_id)); 1772a03ee000SJeff Roberson if (error == 0) 1773d431dea5SKyle Evans error = cpuset_setproc(-1, set, NULL, NULL, false); 1774d7f687fcSJeff Roberson cpuset_rel(set); 1775d7f687fcSJeff Roberson return (error); 1776d7f687fcSJeff Roberson } 1777d7f687fcSJeff Roberson 1778d7f687fcSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 1779d7f687fcSJeff Roberson struct cpuset_setid_args { 1780d7f687fcSJeff Roberson cpuwhich_t which; 1781d7f687fcSJeff Roberson id_t id; 1782d7f687fcSJeff Roberson cpusetid_t setid; 1783d7f687fcSJeff Roberson }; 1784d7f687fcSJeff Roberson #endif 1785d7f687fcSJeff Roberson int 17868451d0ddSKip Macy sys_cpuset_setid(struct thread *td, struct cpuset_setid_args *uap) 1787d7f687fcSJeff Roberson { 1788ea2ebdc1SEdward Tomasz Napierala 1789ea2ebdc1SEdward Tomasz Napierala return (kern_cpuset_setid(td, uap->which, uap->id, uap->setid)); 1790ea2ebdc1SEdward Tomasz Napierala } 1791ea2ebdc1SEdward Tomasz Napierala 1792ea2ebdc1SEdward Tomasz Napierala int 1793ea2ebdc1SEdward Tomasz Napierala kern_cpuset_setid(struct thread *td, cpuwhich_t which, 1794ea2ebdc1SEdward Tomasz Napierala id_t id, cpusetid_t setid) 1795ea2ebdc1SEdward Tomasz Napierala { 1796d7f687fcSJeff Roberson struct cpuset *set; 1797d7f687fcSJeff Roberson int error; 1798d7f687fcSJeff Roberson 1799d7f687fcSJeff Roberson /* 1800d7f687fcSJeff Roberson * Presently we only support per-process sets. 1801d7f687fcSJeff Roberson */ 1802ea2ebdc1SEdward Tomasz Napierala if (which != CPU_WHICH_PID) 1803d7f687fcSJeff Roberson return (EINVAL); 1804ea2ebdc1SEdward Tomasz Napierala set = cpuset_lookup(setid, td); 1805d7f687fcSJeff Roberson if (set == NULL) 1806d7f687fcSJeff Roberson return (ESRCH); 1807d431dea5SKyle Evans error = cpuset_setproc(id, set, NULL, NULL, false); 1808d7f687fcSJeff Roberson cpuset_rel(set); 1809d7f687fcSJeff Roberson return (error); 1810d7f687fcSJeff Roberson } 1811d7f687fcSJeff Roberson 1812d7f687fcSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 1813d7f687fcSJeff Roberson struct cpuset_getid_args { 1814d7f687fcSJeff Roberson cpulevel_t level; 1815d7f687fcSJeff Roberson cpuwhich_t which; 1816d7f687fcSJeff Roberson id_t id; 1817d7f687fcSJeff Roberson cpusetid_t *setid; 18182b69bb1fSKevin Lo }; 1819d7f687fcSJeff Roberson #endif 1820d7f687fcSJeff Roberson int 18218451d0ddSKip Macy sys_cpuset_getid(struct thread *td, struct cpuset_getid_args *uap) 1822d7f687fcSJeff Roberson { 1823ea2ebdc1SEdward Tomasz Napierala 1824ea2ebdc1SEdward Tomasz Napierala return (kern_cpuset_getid(td, uap->level, uap->which, uap->id, 1825ea2ebdc1SEdward Tomasz Napierala uap->setid)); 1826ea2ebdc1SEdward Tomasz Napierala } 1827ea2ebdc1SEdward Tomasz Napierala 1828ea2ebdc1SEdward Tomasz Napierala int 1829ea2ebdc1SEdward Tomasz Napierala kern_cpuset_getid(struct thread *td, cpulevel_t level, cpuwhich_t which, 1830ea2ebdc1SEdward Tomasz Napierala id_t id, cpusetid_t *setid) 1831ea2ebdc1SEdward Tomasz Napierala { 1832d7f687fcSJeff Roberson struct cpuset *nset; 1833d7f687fcSJeff Roberson struct cpuset *set; 1834d7f687fcSJeff Roberson struct thread *ttd; 1835d7f687fcSJeff Roberson struct proc *p; 1836ea2ebdc1SEdward Tomasz Napierala cpusetid_t tmpid; 1837d7f687fcSJeff Roberson int error; 1838d7f687fcSJeff Roberson 1839ea2ebdc1SEdward Tomasz Napierala if (level == CPU_LEVEL_WHICH && which != CPU_WHICH_CPUSET) 1840d7f687fcSJeff Roberson return (EINVAL); 1841ea2ebdc1SEdward Tomasz Napierala error = cpuset_which(which, id, &p, &ttd, &set); 1842d7f687fcSJeff Roberson if (error) 1843d7f687fcSJeff Roberson return (error); 1844ea2ebdc1SEdward Tomasz Napierala switch (which) { 1845d7f687fcSJeff Roberson case CPU_WHICH_TID: 1846d7f687fcSJeff Roberson case CPU_WHICH_PID: 1847d7f687fcSJeff Roberson thread_lock(ttd); 1848a03ee000SJeff Roberson set = cpuset_refbase(ttd->td_cpuset); 1849d7f687fcSJeff Roberson thread_unlock(ttd); 1850d7f687fcSJeff Roberson PROC_UNLOCK(p); 1851d7f687fcSJeff Roberson break; 1852d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 1853413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 1854d7f687fcSJeff Roberson break; 18559b33b154SJeff Roberson case CPU_WHICH_IRQ: 1856c0ae6688SJohn Baldwin case CPU_WHICH_DOMAIN: 18579b33b154SJeff Roberson return (EINVAL); 1858d7f687fcSJeff Roberson } 1859ea2ebdc1SEdward Tomasz Napierala switch (level) { 1860d7f687fcSJeff Roberson case CPU_LEVEL_ROOT: 1861a03ee000SJeff Roberson nset = cpuset_refroot(set); 1862d7f687fcSJeff Roberson cpuset_rel(set); 1863d7f687fcSJeff Roberson set = nset; 1864d7f687fcSJeff Roberson break; 1865d7f687fcSJeff Roberson case CPU_LEVEL_CPUSET: 1866d7f687fcSJeff Roberson break; 1867d7f687fcSJeff Roberson case CPU_LEVEL_WHICH: 1868d7f687fcSJeff Roberson break; 1869d7f687fcSJeff Roberson } 1870ea2ebdc1SEdward Tomasz Napierala tmpid = set->cs_id; 1871d7f687fcSJeff Roberson cpuset_rel(set); 1872d7f687fcSJeff Roberson if (error == 0) 1873a1d0659cSJung-uk Kim error = copyout(&tmpid, setid, sizeof(tmpid)); 1874d7f687fcSJeff Roberson 1875d7f687fcSJeff Roberson return (error); 1876d7f687fcSJeff Roberson } 1877d7f687fcSJeff Roberson 1878d7f687fcSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 1879d7f687fcSJeff Roberson struct cpuset_getaffinity_args { 1880d7f687fcSJeff Roberson cpulevel_t level; 1881d7f687fcSJeff Roberson cpuwhich_t which; 18827f64829aSRuslan Ermilov id_t id; 18837f64829aSRuslan Ermilov size_t cpusetsize; 18847f64829aSRuslan Ermilov cpuset_t *mask; 1885d7f687fcSJeff Roberson }; 1886d7f687fcSJeff Roberson #endif 1887d7f687fcSJeff Roberson int 18888451d0ddSKip Macy sys_cpuset_getaffinity(struct thread *td, struct cpuset_getaffinity_args *uap) 1889d7f687fcSJeff Roberson { 189096ee4310SEdward Tomasz Napierala 189196ee4310SEdward Tomasz Napierala return (kern_cpuset_getaffinity(td, uap->level, uap->which, 189247a57144SJustin Hibbits uap->id, uap->cpusetsize, uap->mask, ©_set)); 189396ee4310SEdward Tomasz Napierala } 189496ee4310SEdward Tomasz Napierala 189596ee4310SEdward Tomasz Napierala int 189696ee4310SEdward Tomasz Napierala kern_cpuset_getaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, 189747a57144SJustin Hibbits id_t id, size_t cpusetsize, cpuset_t *maskp, const struct cpuset_copy_cb *cb) 189896ee4310SEdward Tomasz Napierala { 1899d7f687fcSJeff Roberson struct thread *ttd; 1900d7f687fcSJeff Roberson struct cpuset *nset; 1901d7f687fcSJeff Roberson struct cpuset *set; 1902d7f687fcSJeff Roberson struct proc *p; 1903d7f687fcSJeff Roberson cpuset_t *mask; 1904d7f687fcSJeff Roberson int error; 19057f64829aSRuslan Ermilov size_t size; 1906d7f687fcSJeff Roberson 19079eb997cbSMark Johnston error = cpuset_check_capabilities(td, level, which, id); 19089eb997cbSMark Johnston if (error != 0) 19099eb997cbSMark Johnston return (error); 1910f35093f8SDmitry Chagin mask = malloc(sizeof(cpuset_t), M_TEMP, M_WAITOK | M_ZERO); 191196ee4310SEdward Tomasz Napierala error = cpuset_which(which, id, &p, &ttd, &set); 1912d7f687fcSJeff Roberson if (error) 1913d7f687fcSJeff Roberson goto out; 191496ee4310SEdward Tomasz Napierala switch (level) { 1915d7f687fcSJeff Roberson case CPU_LEVEL_ROOT: 1916d7f687fcSJeff Roberson case CPU_LEVEL_CPUSET: 191796ee4310SEdward Tomasz Napierala switch (which) { 1918d7f687fcSJeff Roberson case CPU_WHICH_TID: 1919d7f687fcSJeff Roberson case CPU_WHICH_PID: 1920d7f687fcSJeff Roberson thread_lock(ttd); 1921d7f687fcSJeff Roberson set = cpuset_ref(ttd->td_cpuset); 1922d7f687fcSJeff Roberson thread_unlock(ttd); 1923d7f687fcSJeff Roberson break; 1924d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 1925413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 1926d7f687fcSJeff Roberson break; 19279b33b154SJeff Roberson case CPU_WHICH_IRQ: 192829dfb631SConrad Meyer case CPU_WHICH_INTRHANDLER: 192929dfb631SConrad Meyer case CPU_WHICH_ITHREAD: 1930c0ae6688SJohn Baldwin case CPU_WHICH_DOMAIN: 19319b33b154SJeff Roberson error = EINVAL; 19329b33b154SJeff Roberson goto out; 1933d7f687fcSJeff Roberson } 193496ee4310SEdward Tomasz Napierala if (level == CPU_LEVEL_ROOT) 1935a03ee000SJeff Roberson nset = cpuset_refroot(set); 1936d7f687fcSJeff Roberson else 1937a03ee000SJeff Roberson nset = cpuset_refbase(set); 1938d7f687fcSJeff Roberson CPU_COPY(&nset->cs_mask, mask); 1939d7f687fcSJeff Roberson cpuset_rel(nset); 1940d7f687fcSJeff Roberson break; 1941d7f687fcSJeff Roberson case CPU_LEVEL_WHICH: 194296ee4310SEdward Tomasz Napierala switch (which) { 1943d7f687fcSJeff Roberson case CPU_WHICH_TID: 1944d7f687fcSJeff Roberson thread_lock(ttd); 1945d7f687fcSJeff Roberson CPU_COPY(&ttd->td_cpuset->cs_mask, mask); 1946d7f687fcSJeff Roberson thread_unlock(ttd); 1947d7f687fcSJeff Roberson break; 1948d7f687fcSJeff Roberson case CPU_WHICH_PID: 1949d7f687fcSJeff Roberson FOREACH_THREAD_IN_PROC(p, ttd) { 1950d7f687fcSJeff Roberson thread_lock(ttd); 1951e2650af1SStefan Eßer CPU_OR(mask, mask, &ttd->td_cpuset->cs_mask); 1952d7f687fcSJeff Roberson thread_unlock(ttd); 1953d7f687fcSJeff Roberson } 1954d7f687fcSJeff Roberson break; 1955d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 1956413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 1957d7f687fcSJeff Roberson CPU_COPY(&set->cs_mask, mask); 1958d7f687fcSJeff Roberson break; 19599b33b154SJeff Roberson case CPU_WHICH_IRQ: 196029dfb631SConrad Meyer case CPU_WHICH_INTRHANDLER: 196129dfb631SConrad Meyer case CPU_WHICH_ITHREAD: 196229dfb631SConrad Meyer error = intr_getaffinity(id, which, mask); 19639b33b154SJeff Roberson break; 1964c0ae6688SJohn Baldwin case CPU_WHICH_DOMAIN: 196596ee4310SEdward Tomasz Napierala if (id < 0 || id >= MAXMEMDOM) 1966c0ae6688SJohn Baldwin error = ESRCH; 1967c0ae6688SJohn Baldwin else 196896ee4310SEdward Tomasz Napierala CPU_COPY(&cpuset_domain[id], mask); 1969c0ae6688SJohn Baldwin break; 1970d7f687fcSJeff Roberson } 1971d7f687fcSJeff Roberson break; 1972d7f687fcSJeff Roberson default: 1973d7f687fcSJeff Roberson error = EINVAL; 1974d7f687fcSJeff Roberson break; 1975d7f687fcSJeff Roberson } 1976d7f687fcSJeff Roberson if (set) 1977d7f687fcSJeff Roberson cpuset_rel(set); 1978d7f687fcSJeff Roberson if (p) 1979d7f687fcSJeff Roberson PROC_UNLOCK(p); 1980f35093f8SDmitry Chagin if (error == 0) { 1981f35093f8SDmitry Chagin if (cpusetsize < howmany(CPU_FLS(mask), NBBY)) { 1982f35093f8SDmitry Chagin error = ERANGE; 1983f35093f8SDmitry Chagin goto out; 1984f35093f8SDmitry Chagin } 1985f35093f8SDmitry Chagin size = min(cpusetsize, sizeof(cpuset_t)); 19864a3e5133SMark Johnston error = cb->cpuset_copyout(mask, maskp, size); 1987f35093f8SDmitry Chagin if (error != 0) 1988f35093f8SDmitry Chagin goto out; 1989f35093f8SDmitry Chagin if (cpusetsize > size) { 1990f35093f8SDmitry Chagin char *end; 1991f35093f8SDmitry Chagin char *cp; 1992f35093f8SDmitry Chagin int rv; 1993f35093f8SDmitry Chagin 1994f35093f8SDmitry Chagin end = cp = (char *)&maskp->__bits; 1995f35093f8SDmitry Chagin end += cpusetsize; 1996f35093f8SDmitry Chagin cp += size; 1997f35093f8SDmitry Chagin while (cp != end) { 1998f35093f8SDmitry Chagin rv = subyte(cp, 0); 1999f35093f8SDmitry Chagin if (rv == -1) { 2000f35093f8SDmitry Chagin error = EFAULT; 2001f35093f8SDmitry Chagin goto out; 2002f35093f8SDmitry Chagin } 2003f35093f8SDmitry Chagin cp++; 2004f35093f8SDmitry Chagin } 2005f35093f8SDmitry Chagin } 2006586ed321SDmitry Chagin #ifdef KTRACE 2007586ed321SDmitry Chagin if ( KTRPOINT(td, KTR_STRUCT)) 2008586ed321SDmitry Chagin ktrcpuset(mask, size); 2009586ed321SDmitry Chagin #endif 2010f35093f8SDmitry Chagin } 2011d7f687fcSJeff Roberson out: 2012d7f687fcSJeff Roberson free(mask, M_TEMP); 2013d7f687fcSJeff Roberson return (error); 2014d7f687fcSJeff Roberson } 2015d7f687fcSJeff Roberson 2016d7f687fcSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 2017d7f687fcSJeff Roberson struct cpuset_setaffinity_args { 2018d7f687fcSJeff Roberson cpulevel_t level; 2019d7f687fcSJeff Roberson cpuwhich_t which; 20207f64829aSRuslan Ermilov id_t id; 20217f64829aSRuslan Ermilov size_t cpusetsize; 20227f64829aSRuslan Ermilov const cpuset_t *mask; 2023d7f687fcSJeff Roberson }; 2024d7f687fcSJeff Roberson #endif 2025d7f687fcSJeff Roberson int 20268451d0ddSKip Macy sys_cpuset_setaffinity(struct thread *td, struct cpuset_setaffinity_args *uap) 2027d7f687fcSJeff Roberson { 202896ee4310SEdward Tomasz Napierala 2029f35093f8SDmitry Chagin return (user_cpuset_setaffinity(td, uap->level, uap->which, 203047a57144SJustin Hibbits uap->id, uap->cpusetsize, uap->mask, ©_set)); 203196ee4310SEdward Tomasz Napierala } 203296ee4310SEdward Tomasz Napierala 203396ee4310SEdward Tomasz Napierala int 203496ee4310SEdward Tomasz Napierala kern_cpuset_setaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, 2035f35093f8SDmitry Chagin id_t id, cpuset_t *mask) 203696ee4310SEdward Tomasz Napierala { 2037d7f687fcSJeff Roberson struct cpuset *nset; 2038d7f687fcSJeff Roberson struct cpuset *set; 2039d7f687fcSJeff Roberson struct thread *ttd; 2040d7f687fcSJeff Roberson struct proc *p; 2041d7f687fcSJeff Roberson int error; 2042d7f687fcSJeff Roberson 2043586ed321SDmitry Chagin #ifdef KTRACE 2044586ed321SDmitry Chagin if (KTRPOINT(td, KTR_STRUCT)) 2045586ed321SDmitry Chagin ktrcpuset(mask, sizeof(cpuset_t)); 2046586ed321SDmitry Chagin #endif 20479eb997cbSMark Johnston error = cpuset_check_capabilities(td, level, which, id); 20489eb997cbSMark Johnston if (error != 0) 20499eb997cbSMark Johnston return (error); 2050f35093f8SDmitry Chagin if (CPU_EMPTY(mask)) 2051f35093f8SDmitry Chagin return (EDEADLK); 205296ee4310SEdward Tomasz Napierala switch (level) { 2053d7f687fcSJeff Roberson case CPU_LEVEL_ROOT: 2054d7f687fcSJeff Roberson case CPU_LEVEL_CPUSET: 205596ee4310SEdward Tomasz Napierala error = cpuset_which(which, id, &p, &ttd, &set); 2056d7f687fcSJeff Roberson if (error) 2057d7f687fcSJeff Roberson break; 205896ee4310SEdward Tomasz Napierala switch (which) { 2059d7f687fcSJeff Roberson case CPU_WHICH_TID: 2060d7f687fcSJeff Roberson case CPU_WHICH_PID: 2061d7f687fcSJeff Roberson thread_lock(ttd); 2062d7f687fcSJeff Roberson set = cpuset_ref(ttd->td_cpuset); 2063d7f687fcSJeff Roberson thread_unlock(ttd); 2064c6440f72SJeff Roberson PROC_UNLOCK(p); 2065d7f687fcSJeff Roberson break; 2066d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 2067413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 2068d7f687fcSJeff Roberson break; 20699b33b154SJeff Roberson case CPU_WHICH_IRQ: 207029dfb631SConrad Meyer case CPU_WHICH_INTRHANDLER: 207129dfb631SConrad Meyer case CPU_WHICH_ITHREAD: 2072c0ae6688SJohn Baldwin case CPU_WHICH_DOMAIN: 2073f35093f8SDmitry Chagin return (EINVAL); 2074d7f687fcSJeff Roberson } 207596ee4310SEdward Tomasz Napierala if (level == CPU_LEVEL_ROOT) 2076a03ee000SJeff Roberson nset = cpuset_refroot(set); 2077d7f687fcSJeff Roberson else 2078a03ee000SJeff Roberson nset = cpuset_refbase(set); 2079d7f687fcSJeff Roberson error = cpuset_modify(nset, mask); 2080d7f687fcSJeff Roberson cpuset_rel(nset); 2081d7f687fcSJeff Roberson cpuset_rel(set); 2082d7f687fcSJeff Roberson break; 2083d7f687fcSJeff Roberson case CPU_LEVEL_WHICH: 208496ee4310SEdward Tomasz Napierala switch (which) { 2085d7f687fcSJeff Roberson case CPU_WHICH_TID: 208696ee4310SEdward Tomasz Napierala error = cpuset_setthread(id, mask); 2087d7f687fcSJeff Roberson break; 2088d7f687fcSJeff Roberson case CPU_WHICH_PID: 2089d431dea5SKyle Evans error = cpuset_setproc(id, NULL, mask, NULL, false); 2090d7f687fcSJeff Roberson break; 2091d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 2092413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 209396ee4310SEdward Tomasz Napierala error = cpuset_which(which, id, &p, &ttd, &set); 2094d7f687fcSJeff Roberson if (error == 0) { 2095d7f687fcSJeff Roberson error = cpuset_modify(set, mask); 2096d7f687fcSJeff Roberson cpuset_rel(set); 2097d7f687fcSJeff Roberson } 2098d7f687fcSJeff Roberson break; 20999b33b154SJeff Roberson case CPU_WHICH_IRQ: 210029dfb631SConrad Meyer case CPU_WHICH_INTRHANDLER: 210129dfb631SConrad Meyer case CPU_WHICH_ITHREAD: 210229dfb631SConrad Meyer error = intr_setaffinity(id, which, mask); 21039b33b154SJeff Roberson break; 2104d7f687fcSJeff Roberson default: 2105d7f687fcSJeff Roberson error = EINVAL; 2106d7f687fcSJeff Roberson break; 2107d7f687fcSJeff Roberson } 2108d7f687fcSJeff Roberson break; 2109d7f687fcSJeff Roberson default: 2110d7f687fcSJeff Roberson error = EINVAL; 2111d7f687fcSJeff Roberson break; 2112d7f687fcSJeff Roberson } 2113f35093f8SDmitry Chagin return (error); 2114f35093f8SDmitry Chagin } 2115f35093f8SDmitry Chagin 2116f35093f8SDmitry Chagin int 2117f35093f8SDmitry Chagin user_cpuset_setaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, 211847a57144SJustin Hibbits id_t id, size_t cpusetsize, const cpuset_t *maskp, const struct cpuset_copy_cb *cb) 2119f35093f8SDmitry Chagin { 2120f35093f8SDmitry Chagin cpuset_t *mask; 2121f35093f8SDmitry Chagin int error; 2122f35093f8SDmitry Chagin size_t size; 2123f35093f8SDmitry Chagin 2124f35093f8SDmitry Chagin size = min(cpusetsize, sizeof(cpuset_t)); 2125f35093f8SDmitry Chagin mask = malloc(sizeof(cpuset_t), M_TEMP, M_WAITOK | M_ZERO); 21264a3e5133SMark Johnston error = cb->cpuset_copyin(maskp, mask, size); 2127f35093f8SDmitry Chagin if (error) 2128f35093f8SDmitry Chagin goto out; 2129f35093f8SDmitry Chagin /* 2130f35093f8SDmitry Chagin * Verify that no high bits are set. 2131f35093f8SDmitry Chagin */ 2132f35093f8SDmitry Chagin if (cpusetsize > sizeof(cpuset_t)) { 2133f35093f8SDmitry Chagin const char *end, *cp; 2134f35093f8SDmitry Chagin int val; 2135f35093f8SDmitry Chagin end = cp = (const char *)&maskp->__bits; 2136f35093f8SDmitry Chagin end += cpusetsize; 2137f35093f8SDmitry Chagin cp += sizeof(cpuset_t); 2138f35093f8SDmitry Chagin 2139f35093f8SDmitry Chagin while (cp != end) { 2140f35093f8SDmitry Chagin val = fubyte(cp); 2141f35093f8SDmitry Chagin if (val == -1) { 2142f35093f8SDmitry Chagin error = EFAULT; 2143f35093f8SDmitry Chagin goto out; 2144f35093f8SDmitry Chagin } 2145f35093f8SDmitry Chagin if (val != 0) { 2146f35093f8SDmitry Chagin error = EINVAL; 2147f35093f8SDmitry Chagin goto out; 2148f35093f8SDmitry Chagin } 2149f35093f8SDmitry Chagin cp++; 2150f35093f8SDmitry Chagin } 2151f35093f8SDmitry Chagin } 2152f35093f8SDmitry Chagin error = kern_cpuset_setaffinity(td, level, which, id, mask); 2153f35093f8SDmitry Chagin 2154d7f687fcSJeff Roberson out: 2155d7f687fcSJeff Roberson free(mask, M_TEMP); 2156d7f687fcSJeff Roberson return (error); 2157d7f687fcSJeff Roberson } 2158dea0ed66SBjoern A. Zeeb 21593f289c3fSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 21603f289c3fSJeff Roberson struct cpuset_getdomain_args { 21613f289c3fSJeff Roberson cpulevel_t level; 21623f289c3fSJeff Roberson cpuwhich_t which; 21633f289c3fSJeff Roberson id_t id; 21643f289c3fSJeff Roberson size_t domainsetsize; 21653f289c3fSJeff Roberson domainset_t *mask; 21663f289c3fSJeff Roberson int *policy; 21673f289c3fSJeff Roberson }; 21683f289c3fSJeff Roberson #endif 21693f289c3fSJeff Roberson int 21703f289c3fSJeff Roberson sys_cpuset_getdomain(struct thread *td, struct cpuset_getdomain_args *uap) 2171dea0ed66SBjoern A. Zeeb { 2172dea0ed66SBjoern A. Zeeb 21733f289c3fSJeff Roberson return (kern_cpuset_getdomain(td, uap->level, uap->which, 217447a57144SJustin Hibbits uap->id, uap->domainsetsize, uap->mask, uap->policy, ©_set)); 21753f289c3fSJeff Roberson } 21763f289c3fSJeff Roberson 21773f289c3fSJeff Roberson int 21783f289c3fSJeff Roberson kern_cpuset_getdomain(struct thread *td, cpulevel_t level, cpuwhich_t which, 217947a57144SJustin Hibbits id_t id, size_t domainsetsize, domainset_t *maskp, int *policyp, 218047a57144SJustin Hibbits const struct cpuset_copy_cb *cb) 21813f289c3fSJeff Roberson { 21823f289c3fSJeff Roberson struct domainset outset; 21833f289c3fSJeff Roberson struct thread *ttd; 21843f289c3fSJeff Roberson struct cpuset *nset; 21853f289c3fSJeff Roberson struct cpuset *set; 21863f289c3fSJeff Roberson struct domainset *dset; 21873f289c3fSJeff Roberson struct proc *p; 21883f289c3fSJeff Roberson domainset_t *mask; 21893f289c3fSJeff Roberson int error; 21903f289c3fSJeff Roberson 21913f289c3fSJeff Roberson if (domainsetsize < sizeof(domainset_t) || 21923f289c3fSJeff Roberson domainsetsize > DOMAINSET_MAXSIZE / NBBY) 21933f289c3fSJeff Roberson return (ERANGE); 21949eb997cbSMark Johnston error = cpuset_check_capabilities(td, level, which, id); 21959eb997cbSMark Johnston if (error != 0) 21969eb997cbSMark Johnston return (error); 21973f289c3fSJeff Roberson mask = malloc(domainsetsize, M_TEMP, M_WAITOK | M_ZERO); 21983f289c3fSJeff Roberson bzero(&outset, sizeof(outset)); 21993f289c3fSJeff Roberson error = cpuset_which(which, id, &p, &ttd, &set); 22003f289c3fSJeff Roberson if (error) 22013f289c3fSJeff Roberson goto out; 22023f289c3fSJeff Roberson switch (level) { 22033f289c3fSJeff Roberson case CPU_LEVEL_ROOT: 22043f289c3fSJeff Roberson case CPU_LEVEL_CPUSET: 22053f289c3fSJeff Roberson switch (which) { 22063f289c3fSJeff Roberson case CPU_WHICH_TID: 22073f289c3fSJeff Roberson case CPU_WHICH_PID: 22083f289c3fSJeff Roberson thread_lock(ttd); 22093f289c3fSJeff Roberson set = cpuset_ref(ttd->td_cpuset); 22103f289c3fSJeff Roberson thread_unlock(ttd); 22113f289c3fSJeff Roberson break; 22123f289c3fSJeff Roberson case CPU_WHICH_CPUSET: 22133f289c3fSJeff Roberson case CPU_WHICH_JAIL: 22143f289c3fSJeff Roberson break; 22153f289c3fSJeff Roberson case CPU_WHICH_IRQ: 22163f289c3fSJeff Roberson case CPU_WHICH_INTRHANDLER: 22173f289c3fSJeff Roberson case CPU_WHICH_ITHREAD: 22183f289c3fSJeff Roberson case CPU_WHICH_DOMAIN: 22193f289c3fSJeff Roberson error = EINVAL; 22203f289c3fSJeff Roberson goto out; 22213f289c3fSJeff Roberson } 22223f289c3fSJeff Roberson if (level == CPU_LEVEL_ROOT) 22233f289c3fSJeff Roberson nset = cpuset_refroot(set); 22243f289c3fSJeff Roberson else 22253f289c3fSJeff Roberson nset = cpuset_refbase(set); 22263f289c3fSJeff Roberson domainset_copy(nset->cs_domain, &outset); 22273f289c3fSJeff Roberson cpuset_rel(nset); 22283f289c3fSJeff Roberson break; 22293f289c3fSJeff Roberson case CPU_LEVEL_WHICH: 22303f289c3fSJeff Roberson switch (which) { 22313f289c3fSJeff Roberson case CPU_WHICH_TID: 22323f289c3fSJeff Roberson thread_lock(ttd); 22333f289c3fSJeff Roberson domainset_copy(ttd->td_cpuset->cs_domain, &outset); 22343f289c3fSJeff Roberson thread_unlock(ttd); 22353f289c3fSJeff Roberson break; 22363f289c3fSJeff Roberson case CPU_WHICH_PID: 22373f289c3fSJeff Roberson FOREACH_THREAD_IN_PROC(p, ttd) { 22383f289c3fSJeff Roberson thread_lock(ttd); 22393f289c3fSJeff Roberson dset = ttd->td_cpuset->cs_domain; 22403f289c3fSJeff Roberson /* Show all domains in the proc. */ 22413f289c3fSJeff Roberson DOMAINSET_OR(&outset.ds_mask, &dset->ds_mask); 22423f289c3fSJeff Roberson /* Last policy wins. */ 22433f289c3fSJeff Roberson outset.ds_policy = dset->ds_policy; 22443f289c3fSJeff Roberson outset.ds_prefer = dset->ds_prefer; 22453f289c3fSJeff Roberson thread_unlock(ttd); 22463f289c3fSJeff Roberson } 22473f289c3fSJeff Roberson break; 22483f289c3fSJeff Roberson case CPU_WHICH_CPUSET: 22493f289c3fSJeff Roberson case CPU_WHICH_JAIL: 22503f289c3fSJeff Roberson domainset_copy(set->cs_domain, &outset); 22513f289c3fSJeff Roberson break; 22523f289c3fSJeff Roberson case CPU_WHICH_IRQ: 22533f289c3fSJeff Roberson case CPU_WHICH_INTRHANDLER: 22543f289c3fSJeff Roberson case CPU_WHICH_ITHREAD: 22553f289c3fSJeff Roberson case CPU_WHICH_DOMAIN: 22563f289c3fSJeff Roberson error = EINVAL; 22573f289c3fSJeff Roberson break; 22583f289c3fSJeff Roberson } 22593f289c3fSJeff Roberson break; 22603f289c3fSJeff Roberson default: 22613f289c3fSJeff Roberson error = EINVAL; 22623f289c3fSJeff Roberson break; 22633f289c3fSJeff Roberson } 22643f289c3fSJeff Roberson if (set) 22653f289c3fSJeff Roberson cpuset_rel(set); 22663f289c3fSJeff Roberson if (p) 22673f289c3fSJeff Roberson PROC_UNLOCK(p); 22683f289c3fSJeff Roberson /* 22693f289c3fSJeff Roberson * Translate prefer into a set containing only the preferred domain, 22703f289c3fSJeff Roberson * not the entire fallback set. 22713f289c3fSJeff Roberson */ 22723f289c3fSJeff Roberson if (outset.ds_policy == DOMAINSET_POLICY_PREFER) { 22733f289c3fSJeff Roberson DOMAINSET_ZERO(&outset.ds_mask); 22743f289c3fSJeff Roberson DOMAINSET_SET(outset.ds_prefer, &outset.ds_mask); 22753f289c3fSJeff Roberson } 22763f289c3fSJeff Roberson DOMAINSET_COPY(&outset.ds_mask, mask); 22773f289c3fSJeff Roberson if (error == 0) 22784a3e5133SMark Johnston error = cb->cpuset_copyout(mask, maskp, domainsetsize); 22793f289c3fSJeff Roberson if (error == 0) 2280dd51fec3SBrooks Davis if (suword32(policyp, outset.ds_policy) != 0) 2281dd51fec3SBrooks Davis error = EFAULT; 22823f289c3fSJeff Roberson out: 22833f289c3fSJeff Roberson free(mask, M_TEMP); 22843f289c3fSJeff Roberson return (error); 22853f289c3fSJeff Roberson } 22863f289c3fSJeff Roberson 22873f289c3fSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 22883f289c3fSJeff Roberson struct cpuset_setdomain_args { 22893f289c3fSJeff Roberson cpulevel_t level; 22903f289c3fSJeff Roberson cpuwhich_t which; 22913f289c3fSJeff Roberson id_t id; 22923f289c3fSJeff Roberson size_t domainsetsize; 22933f289c3fSJeff Roberson domainset_t *mask; 22943f289c3fSJeff Roberson int policy; 22953f289c3fSJeff Roberson }; 22963f289c3fSJeff Roberson #endif 22973f289c3fSJeff Roberson int 22983f289c3fSJeff Roberson sys_cpuset_setdomain(struct thread *td, struct cpuset_setdomain_args *uap) 22993f289c3fSJeff Roberson { 23003f289c3fSJeff Roberson 23013f289c3fSJeff Roberson return (kern_cpuset_setdomain(td, uap->level, uap->which, 230247a57144SJustin Hibbits uap->id, uap->domainsetsize, uap->mask, uap->policy, ©_set)); 23033f289c3fSJeff Roberson } 23043f289c3fSJeff Roberson 23053f289c3fSJeff Roberson int 23063f289c3fSJeff Roberson kern_cpuset_setdomain(struct thread *td, cpulevel_t level, cpuwhich_t which, 230747a57144SJustin Hibbits id_t id, size_t domainsetsize, const domainset_t *maskp, int policy, 230847a57144SJustin Hibbits const struct cpuset_copy_cb *cb) 23093f289c3fSJeff Roberson { 23103f289c3fSJeff Roberson struct cpuset *nset; 23113f289c3fSJeff Roberson struct cpuset *set; 23123f289c3fSJeff Roberson struct thread *ttd; 23133f289c3fSJeff Roberson struct proc *p; 23143f289c3fSJeff Roberson struct domainset domain; 23153f289c3fSJeff Roberson domainset_t *mask; 23163f289c3fSJeff Roberson int error; 23173f289c3fSJeff Roberson 23183f289c3fSJeff Roberson if (domainsetsize < sizeof(domainset_t) || 23193f289c3fSJeff Roberson domainsetsize > DOMAINSET_MAXSIZE / NBBY) 23203f289c3fSJeff Roberson return (ERANGE); 232170d66bcfSEric van Gyzen if (policy <= DOMAINSET_POLICY_INVALID || 232270d66bcfSEric van Gyzen policy > DOMAINSET_POLICY_MAX) 232370d66bcfSEric van Gyzen return (EINVAL); 23249eb997cbSMark Johnston error = cpuset_check_capabilities(td, level, which, id); 23259eb997cbSMark Johnston if (error != 0) 23269eb997cbSMark Johnston return (error); 23273f289c3fSJeff Roberson memset(&domain, 0, sizeof(domain)); 23283f289c3fSJeff Roberson mask = malloc(domainsetsize, M_TEMP, M_WAITOK | M_ZERO); 23294a3e5133SMark Johnston error = cb->cpuset_copyin(maskp, mask, domainsetsize); 23303f289c3fSJeff Roberson if (error) 23313f289c3fSJeff Roberson goto out; 23323f289c3fSJeff Roberson /* 23333f289c3fSJeff Roberson * Verify that no high bits are set. 23343f289c3fSJeff Roberson */ 23353f289c3fSJeff Roberson if (domainsetsize > sizeof(domainset_t)) { 23363f289c3fSJeff Roberson char *end; 23373f289c3fSJeff Roberson char *cp; 23383f289c3fSJeff Roberson 23393f289c3fSJeff Roberson end = cp = (char *)&mask->__bits; 23403f289c3fSJeff Roberson end += domainsetsize; 23413f289c3fSJeff Roberson cp += sizeof(domainset_t); 23423f289c3fSJeff Roberson while (cp != end) 23433f289c3fSJeff Roberson if (*cp++ != 0) { 23443f289c3fSJeff Roberson error = EINVAL; 23453f289c3fSJeff Roberson goto out; 23463f289c3fSJeff Roberson } 23473f289c3fSJeff Roberson } 2348f1b18a66SKyle Evans if (DOMAINSET_EMPTY(mask)) { 2349f1b18a66SKyle Evans error = EDEADLK; 2350f1b18a66SKyle Evans goto out; 2351f1b18a66SKyle Evans } 23523f289c3fSJeff Roberson DOMAINSET_COPY(mask, &domain.ds_mask); 23533f289c3fSJeff Roberson domain.ds_policy = policy; 23543f289c3fSJeff Roberson 235563cdd18eSMark Johnston /* 235663cdd18eSMark Johnston * Sanitize the provided mask. 235763cdd18eSMark Johnston */ 235863cdd18eSMark Johnston if (!DOMAINSET_SUBSET(&all_domains, &domain.ds_mask)) { 235963cdd18eSMark Johnston error = EINVAL; 236063cdd18eSMark Johnston goto out; 236163cdd18eSMark Johnston } 236263cdd18eSMark Johnston 23633f289c3fSJeff Roberson /* Translate preferred policy into a mask and fallback. */ 23643f289c3fSJeff Roberson if (policy == DOMAINSET_POLICY_PREFER) { 23653f289c3fSJeff Roberson /* Only support a single preferred domain. */ 236670d66bcfSEric van Gyzen if (DOMAINSET_COUNT(&domain.ds_mask) != 1) { 236770d66bcfSEric van Gyzen error = EINVAL; 236870d66bcfSEric van Gyzen goto out; 236970d66bcfSEric van Gyzen } 23703f289c3fSJeff Roberson domain.ds_prefer = DOMAINSET_FFS(&domain.ds_mask) - 1; 23713f289c3fSJeff Roberson /* This will be constrained by domainset_shadow(). */ 237263cdd18eSMark Johnston DOMAINSET_COPY(&all_domains, &domain.ds_mask); 23733f289c3fSJeff Roberson } 23743f289c3fSJeff Roberson 237530c5525bSAndrew Gallatin /* 237630c5525bSAndrew Gallatin * When given an impossible policy, fall back to interleaving 237763cdd18eSMark Johnston * across all domains. 237830c5525bSAndrew Gallatin */ 237930c5525bSAndrew Gallatin if (domainset_empty_vm(&domain)) 238029bb6c19SMark Johnston domainset_copy(domainset2, &domain); 238130c5525bSAndrew Gallatin 23823f289c3fSJeff Roberson switch (level) { 23833f289c3fSJeff Roberson case CPU_LEVEL_ROOT: 23843f289c3fSJeff Roberson case CPU_LEVEL_CPUSET: 23853f289c3fSJeff Roberson error = cpuset_which(which, id, &p, &ttd, &set); 23863f289c3fSJeff Roberson if (error) 23873f289c3fSJeff Roberson break; 23883f289c3fSJeff Roberson switch (which) { 23893f289c3fSJeff Roberson case CPU_WHICH_TID: 23903f289c3fSJeff Roberson case CPU_WHICH_PID: 23913f289c3fSJeff Roberson thread_lock(ttd); 23923f289c3fSJeff Roberson set = cpuset_ref(ttd->td_cpuset); 23933f289c3fSJeff Roberson thread_unlock(ttd); 23943f289c3fSJeff Roberson PROC_UNLOCK(p); 23953f289c3fSJeff Roberson break; 23963f289c3fSJeff Roberson case CPU_WHICH_CPUSET: 23973f289c3fSJeff Roberson case CPU_WHICH_JAIL: 23983f289c3fSJeff Roberson break; 23993f289c3fSJeff Roberson case CPU_WHICH_IRQ: 24003f289c3fSJeff Roberson case CPU_WHICH_INTRHANDLER: 24013f289c3fSJeff Roberson case CPU_WHICH_ITHREAD: 24023f289c3fSJeff Roberson case CPU_WHICH_DOMAIN: 24033f289c3fSJeff Roberson error = EINVAL; 24043f289c3fSJeff Roberson goto out; 24053f289c3fSJeff Roberson } 24063f289c3fSJeff Roberson if (level == CPU_LEVEL_ROOT) 24073f289c3fSJeff Roberson nset = cpuset_refroot(set); 24083f289c3fSJeff Roberson else 24093f289c3fSJeff Roberson nset = cpuset_refbase(set); 24103f289c3fSJeff Roberson error = cpuset_modify_domain(nset, &domain); 24113f289c3fSJeff Roberson cpuset_rel(nset); 24123f289c3fSJeff Roberson cpuset_rel(set); 24133f289c3fSJeff Roberson break; 24143f289c3fSJeff Roberson case CPU_LEVEL_WHICH: 24153f289c3fSJeff Roberson switch (which) { 24163f289c3fSJeff Roberson case CPU_WHICH_TID: 24173f289c3fSJeff Roberson error = _cpuset_setthread(id, NULL, &domain); 24183f289c3fSJeff Roberson break; 24193f289c3fSJeff Roberson case CPU_WHICH_PID: 2420d431dea5SKyle Evans error = cpuset_setproc(id, NULL, NULL, &domain, false); 24213f289c3fSJeff Roberson break; 24223f289c3fSJeff Roberson case CPU_WHICH_CPUSET: 24233f289c3fSJeff Roberson case CPU_WHICH_JAIL: 24243f289c3fSJeff Roberson error = cpuset_which(which, id, &p, &ttd, &set); 24253f289c3fSJeff Roberson if (error == 0) { 24263f289c3fSJeff Roberson error = cpuset_modify_domain(set, &domain); 24273f289c3fSJeff Roberson cpuset_rel(set); 24283f289c3fSJeff Roberson } 24293f289c3fSJeff Roberson break; 24303f289c3fSJeff Roberson case CPU_WHICH_IRQ: 24313f289c3fSJeff Roberson case CPU_WHICH_INTRHANDLER: 24323f289c3fSJeff Roberson case CPU_WHICH_ITHREAD: 24333f289c3fSJeff Roberson default: 24343f289c3fSJeff Roberson error = EINVAL; 24353f289c3fSJeff Roberson break; 24363f289c3fSJeff Roberson } 24373f289c3fSJeff Roberson break; 24383f289c3fSJeff Roberson default: 24393f289c3fSJeff Roberson error = EINVAL; 24403f289c3fSJeff Roberson break; 24413f289c3fSJeff Roberson } 24423f289c3fSJeff Roberson out: 24433f289c3fSJeff Roberson free(mask, M_TEMP); 24443f289c3fSJeff Roberson return (error); 24453f289c3fSJeff Roberson } 24463f289c3fSJeff Roberson 24473f289c3fSJeff Roberson #ifdef DDB 2448e5818a53SJeff Roberson 24493f289c3fSJeff Roberson static void 24503f289c3fSJeff Roberson ddb_display_bitset(const struct bitset *set, int size) 24513f289c3fSJeff Roberson { 24523f289c3fSJeff Roberson int bit, once; 24533f289c3fSJeff Roberson 24543f289c3fSJeff Roberson for (once = 0, bit = 0; bit < size; bit++) { 24553f289c3fSJeff Roberson if (CPU_ISSET(bit, set)) { 2456dea0ed66SBjoern A. Zeeb if (once == 0) { 24573f289c3fSJeff Roberson db_printf("%d", bit); 2458dea0ed66SBjoern A. Zeeb once = 1; 2459dea0ed66SBjoern A. Zeeb } else 24603f289c3fSJeff Roberson db_printf(",%d", bit); 2461dea0ed66SBjoern A. Zeeb } 2462dea0ed66SBjoern A. Zeeb } 2463cd32bd7aSJohn Baldwin if (once == 0) 2464cd32bd7aSJohn Baldwin db_printf("<none>"); 2465cd32bd7aSJohn Baldwin } 2466cd32bd7aSJohn Baldwin 24673f289c3fSJeff Roberson void 24683f289c3fSJeff Roberson ddb_display_cpuset(const cpuset_t *set) 24693f289c3fSJeff Roberson { 24703f289c3fSJeff Roberson ddb_display_bitset((const struct bitset *)set, CPU_SETSIZE); 24713f289c3fSJeff Roberson } 24723f289c3fSJeff Roberson 24733f289c3fSJeff Roberson static void 24743f289c3fSJeff Roberson ddb_display_domainset(const domainset_t *set) 24753f289c3fSJeff Roberson { 24763f289c3fSJeff Roberson ddb_display_bitset((const struct bitset *)set, DOMAINSET_SETSIZE); 24773f289c3fSJeff Roberson } 24783f289c3fSJeff Roberson 2479cd32bd7aSJohn Baldwin DB_SHOW_COMMAND(cpusets, db_show_cpusets) 2480cd32bd7aSJohn Baldwin { 2481cd32bd7aSJohn Baldwin struct cpuset *set; 2482cd32bd7aSJohn Baldwin 2483cd32bd7aSJohn Baldwin LIST_FOREACH(set, &cpuset_ids, cs_link) { 2484cd32bd7aSJohn Baldwin db_printf("set=%p id=%-6u ref=%-6d flags=0x%04x parent id=%d\n", 24851a7bb896SMateusz Guzik set, set->cs_id, refcount_load(&set->cs_ref), set->cs_flags, 2486cd32bd7aSJohn Baldwin (set->cs_parent != NULL) ? set->cs_parent->cs_id : 0); 24873f289c3fSJeff Roberson db_printf(" cpu mask="); 2488cd32bd7aSJohn Baldwin ddb_display_cpuset(&set->cs_mask); 2489dea0ed66SBjoern A. Zeeb db_printf("\n"); 24903f289c3fSJeff Roberson db_printf(" domain policy %d prefer %d mask=", 24913f289c3fSJeff Roberson set->cs_domain->ds_policy, set->cs_domain->ds_prefer); 24923f289c3fSJeff Roberson ddb_display_domainset(&set->cs_domain->ds_mask); 24933f289c3fSJeff Roberson db_printf("\n"); 2494dea0ed66SBjoern A. Zeeb if (db_pager_quit) 2495dea0ed66SBjoern A. Zeeb break; 2496dea0ed66SBjoern A. Zeeb } 2497dea0ed66SBjoern A. Zeeb } 24983f289c3fSJeff Roberson 24993f289c3fSJeff Roberson DB_SHOW_COMMAND(domainsets, db_show_domainsets) 25003f289c3fSJeff Roberson { 25013f289c3fSJeff Roberson struct domainset *set; 25023f289c3fSJeff Roberson 25033f289c3fSJeff Roberson LIST_FOREACH(set, &cpuset_domains, ds_link) { 2504e5818a53SJeff Roberson db_printf("set=%p policy %d prefer %d cnt %d\n", 2505e5818a53SJeff Roberson set, set->ds_policy, set->ds_prefer, set->ds_cnt); 25063f289c3fSJeff Roberson db_printf(" mask ="); 25073f289c3fSJeff Roberson ddb_display_domainset(&set->ds_mask); 25083f289c3fSJeff Roberson db_printf("\n"); 25093f289c3fSJeff Roberson } 25103f289c3fSJeff Roberson } 2511dea0ed66SBjoern A. Zeeb #endif /* DDB */ 2512