1d7f687fcSJeff Roberson /*- 28a36da99SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 38a36da99SPedro F. Giffuni * 4d7f687fcSJeff Roberson * Copyright (c) 2008, Jeffrey Roberson <jeff@freebsd.org> 5d7f687fcSJeff Roberson * All rights reserved. 6d7f687fcSJeff Roberson * 73bc8c68dSJeff Roberson * Copyright (c) 2008 Nokia Corporation 83bc8c68dSJeff Roberson * All rights reserved. 93bc8c68dSJeff Roberson * 10d7f687fcSJeff Roberson * Redistribution and use in source and binary forms, with or without 11d7f687fcSJeff Roberson * modification, are permitted provided that the following conditions 12d7f687fcSJeff Roberson * are met: 13d7f687fcSJeff Roberson * 1. Redistributions of source code must retain the above copyright 14d7f687fcSJeff Roberson * notice unmodified, this list of conditions, and the following 15d7f687fcSJeff Roberson * disclaimer. 16d7f687fcSJeff Roberson * 2. Redistributions in binary form must reproduce the above copyright 17d7f687fcSJeff Roberson * notice, this list of conditions and the following disclaimer in the 18d7f687fcSJeff Roberson * documentation and/or other materials provided with the distribution. 19d7f687fcSJeff Roberson * 20d7f687fcSJeff Roberson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21d7f687fcSJeff Roberson * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22d7f687fcSJeff Roberson * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23d7f687fcSJeff Roberson * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24d7f687fcSJeff Roberson * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25d7f687fcSJeff Roberson * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26d7f687fcSJeff Roberson * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27d7f687fcSJeff Roberson * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28d7f687fcSJeff Roberson * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29d7f687fcSJeff Roberson * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30d7f687fcSJeff Roberson * 31d7f687fcSJeff Roberson */ 32d7f687fcSJeff Roberson 33d7f687fcSJeff Roberson #include <sys/cdefs.h> 34d7f687fcSJeff Roberson __FBSDID("$FreeBSD$"); 35d7f687fcSJeff Roberson 36dea0ed66SBjoern A. Zeeb #include "opt_ddb.h" 37586ed321SDmitry Chagin #include "opt_ktrace.h" 38dea0ed66SBjoern A. Zeeb 39d7f687fcSJeff Roberson #include <sys/param.h> 40d7f687fcSJeff Roberson #include <sys/systm.h> 41e5818a53SJeff Roberson #include <sys/sysctl.h> 42e5818a53SJeff Roberson #include <sys/ctype.h> 43d7f687fcSJeff Roberson #include <sys/sysproto.h> 440304c731SJamie Gritton #include <sys/jail.h> 45d7f687fcSJeff Roberson #include <sys/kernel.h> 46d7f687fcSJeff Roberson #include <sys/lock.h> 47d7f687fcSJeff Roberson #include <sys/malloc.h> 48d7f687fcSJeff Roberson #include <sys/mutex.h> 49d7f687fcSJeff Roberson #include <sys/priv.h> 50d7f687fcSJeff Roberson #include <sys/proc.h> 51d7f687fcSJeff Roberson #include <sys/refcount.h> 52d7f687fcSJeff Roberson #include <sys/sched.h> 53d7f687fcSJeff Roberson #include <sys/smp.h> 54d7f687fcSJeff Roberson #include <sys/syscallsubr.h> 55f299c47bSAllan Jude #include <sys/capsicum.h> 56d7f687fcSJeff Roberson #include <sys/cpuset.h> 573f289c3fSJeff Roberson #include <sys/domainset.h> 58d7f687fcSJeff Roberson #include <sys/sx.h> 59d7f687fcSJeff Roberson #include <sys/queue.h> 60e3709597SAttilio Rao #include <sys/libkern.h> 61d7f687fcSJeff Roberson #include <sys/limits.h> 62a03ee000SJeff Roberson #include <sys/bus.h> 63a03ee000SJeff Roberson #include <sys/interrupt.h> 643f289c3fSJeff Roberson #include <sys/vmmeter.h> 65586ed321SDmitry Chagin #include <sys/ktrace.h> 66d7f687fcSJeff Roberson 67d7f687fcSJeff Roberson #include <vm/uma.h> 68c0ae6688SJohn Baldwin #include <vm/vm.h> 693f289c3fSJeff Roberson #include <vm/vm_object.h> 7030c5525bSAndrew Gallatin #include <vm/vm_page.h> 7130c5525bSAndrew Gallatin #include <vm/vm_pageout.h> 72e5818a53SJeff Roberson #include <vm/vm_extern.h> 7330c5525bSAndrew Gallatin #include <vm/vm_param.h> 7430c5525bSAndrew Gallatin #include <vm/vm_phys.h> 7530c5525bSAndrew Gallatin #include <vm/vm_pagequeue.h> 76d7f687fcSJeff Roberson 77dea0ed66SBjoern A. Zeeb #ifdef DDB 78dea0ed66SBjoern A. Zeeb #include <ddb/ddb.h> 79dea0ed66SBjoern A. Zeeb #endif /* DDB */ 80dea0ed66SBjoern A. Zeeb 81d7f687fcSJeff Roberson /* 82d7f687fcSJeff Roberson * cpusets provide a mechanism for creating and manipulating sets of 83d7f687fcSJeff Roberson * processors for the purpose of constraining the scheduling of threads to 84d7f687fcSJeff Roberson * specific processors. 85d7f687fcSJeff Roberson * 86d7f687fcSJeff Roberson * Each process belongs to an identified set, by default this is set 1. Each 87d7f687fcSJeff Roberson * thread may further restrict the cpus it may run on to a subset of this 88d7f687fcSJeff Roberson * named set. This creates an anonymous set which other threads and processes 89d7f687fcSJeff Roberson * may not join by number. 90d7f687fcSJeff Roberson * 91d7f687fcSJeff Roberson * The named set is referred to herein as the 'base' set to avoid ambiguity. 92d7f687fcSJeff Roberson * This set is usually a child of a 'root' set while the anonymous set may 93d7f687fcSJeff Roberson * simply be referred to as a mask. In the syscall api these are referred to 94d7f687fcSJeff Roberson * as the ROOT, CPUSET, and MASK levels where CPUSET is called 'base' here. 95d7f687fcSJeff Roberson * 96d7f687fcSJeff Roberson * Threads inherit their set from their creator whether it be anonymous or 97d7f687fcSJeff Roberson * not. This means that anonymous sets are immutable because they may be 98d7f687fcSJeff Roberson * shared. To modify an anonymous set a new set is created with the desired 99d7f687fcSJeff Roberson * mask and the same parent as the existing anonymous set. This gives the 10093902625SJohn Baldwin * illusion of each thread having a private mask. 101d7f687fcSJeff Roberson * 102d7f687fcSJeff Roberson * Via the syscall apis a user may ask to retrieve or modify the root, base, 103d7f687fcSJeff Roberson * or mask that is discovered via a pid, tid, or setid. Modifying a set 104d7f687fcSJeff Roberson * modifies all numbered and anonymous child sets to comply with the new mask. 105d7f687fcSJeff Roberson * Modifying a pid or tid's mask applies only to that tid but must still 106d7f687fcSJeff Roberson * exist within the assigned parent set. 107d7f687fcSJeff Roberson * 10886855bf5SJohn Baldwin * A thread may not be assigned to a group separate from other threads in 109d7f687fcSJeff Roberson * the process. This is to remove ambiguity when the setid is queried with 110d7f687fcSJeff Roberson * a pid argument. There is no other technical limitation. 111d7f687fcSJeff Roberson * 112d7f687fcSJeff Roberson * This somewhat complex arrangement is intended to make it easy for 113d7f687fcSJeff Roberson * applications to query available processors and bind their threads to 114d7f687fcSJeff Roberson * specific processors while also allowing administrators to dynamically 115d7f687fcSJeff Roberson * reprovision by changing sets which apply to groups of processes. 116d7f687fcSJeff Roberson * 117d7f687fcSJeff Roberson * A simple application should not concern itself with sets at all and 118d7f687fcSJeff Roberson * rather apply masks to its own threads via CPU_WHICH_TID and a -1 id 11993902625SJohn Baldwin * meaning 'curthread'. It may query available cpus for that tid with a 120d7f687fcSJeff Roberson * getaffinity call using (CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, ...). 121d7f687fcSJeff Roberson */ 122e5818a53SJeff Roberson 123e5818a53SJeff Roberson LIST_HEAD(domainlist, domainset); 12429bb6c19SMark Johnston struct domainset __read_mostly domainset_firsttouch; 1259978bd99SMark Johnston struct domainset __read_mostly domainset_fixed[MAXMEMDOM]; 12629bb6c19SMark Johnston struct domainset __read_mostly domainset_interleave; 127662e7fa8SMark Johnston struct domainset __read_mostly domainset_prefer[MAXMEMDOM]; 128662e7fa8SMark Johnston struct domainset __read_mostly domainset_roundrobin; 129e5818a53SJeff Roberson 130d7f687fcSJeff Roberson static uma_zone_t cpuset_zone; 1313f289c3fSJeff Roberson static uma_zone_t domainset_zone; 132d7f687fcSJeff Roberson static struct mtx cpuset_lock; 133d7f687fcSJeff Roberson static struct setlist cpuset_ids; 1343f289c3fSJeff Roberson static struct domainlist cpuset_domains; 135d7f687fcSJeff Roberson static struct unrhdr *cpuset_unr; 136e5818a53SJeff Roberson static struct cpuset *cpuset_zero, *cpuset_default, *cpuset_kernel; 13729bb6c19SMark Johnston static struct domainset *domainset0, *domainset2; 138a03ee000SJeff Roberson 139444528c0SDavid Xu /* Return the size of cpuset_t at the kernel level */ 14060aa2c85SJonathan Anderson SYSCTL_INT(_kern_sched, OID_AUTO, cpusetsize, CTLFLAG_RD | CTLFLAG_CAPRD, 141f0188618SHans Petter Selasky SYSCTL_NULL_INT_PTR, sizeof(cpuset_t), "sizeof(cpuset_t)"); 142444528c0SDavid Xu 143a03ee000SJeff Roberson cpuset_t *cpuset_root; 144c0ae6688SJohn Baldwin cpuset_t cpuset_domain[MAXMEMDOM]; 145d7f687fcSJeff Roberson 1463f289c3fSJeff Roberson static int domainset_valid(const struct domainset *, const struct domainset *); 1473f289c3fSJeff Roberson 1483f289c3fSJeff Roberson /* 1493f289c3fSJeff Roberson * Find the first non-anonymous set starting from 'set'. 1503f289c3fSJeff Roberson */ 1513f289c3fSJeff Roberson static struct cpuset * 1523f289c3fSJeff Roberson cpuset_getbase(struct cpuset *set) 1533f289c3fSJeff Roberson { 1543f289c3fSJeff Roberson 1553f289c3fSJeff Roberson if (set->cs_id == CPUSET_INVALID) 1563f289c3fSJeff Roberson set = set->cs_parent; 1573f289c3fSJeff Roberson return (set); 1583f289c3fSJeff Roberson } 1593f289c3fSJeff Roberson 1603f289c3fSJeff Roberson /* 1613f289c3fSJeff Roberson * Walks up the tree from 'set' to find the root. 1623f289c3fSJeff Roberson */ 1633f289c3fSJeff Roberson static struct cpuset * 1643f289c3fSJeff Roberson cpuset_getroot(struct cpuset *set) 1653f289c3fSJeff Roberson { 1663f289c3fSJeff Roberson 1673f289c3fSJeff Roberson while ((set->cs_flags & CPU_SET_ROOT) == 0 && set->cs_parent != NULL) 1683f289c3fSJeff Roberson set = set->cs_parent; 1693f289c3fSJeff Roberson return (set); 1703f289c3fSJeff Roberson } 1713f289c3fSJeff Roberson 172d7f687fcSJeff Roberson /* 173d7f687fcSJeff Roberson * Acquire a reference to a cpuset, all pointers must be tracked with refs. 174d7f687fcSJeff Roberson */ 175d7f687fcSJeff Roberson struct cpuset * 176d7f687fcSJeff Roberson cpuset_ref(struct cpuset *set) 177d7f687fcSJeff Roberson { 178d7f687fcSJeff Roberson 179d7f687fcSJeff Roberson refcount_acquire(&set->cs_ref); 180d7f687fcSJeff Roberson return (set); 181d7f687fcSJeff Roberson } 182d7f687fcSJeff Roberson 183d7f687fcSJeff Roberson /* 1847a8f695aSBjoern A. Zeeb * Walks up the tree from 'set' to find the root. Returns the root 1857a8f695aSBjoern A. Zeeb * referenced. 1867a8f695aSBjoern A. Zeeb */ 1877a8f695aSBjoern A. Zeeb static struct cpuset * 1887a8f695aSBjoern A. Zeeb cpuset_refroot(struct cpuset *set) 1897a8f695aSBjoern A. Zeeb { 1907a8f695aSBjoern A. Zeeb 1913f289c3fSJeff Roberson return (cpuset_ref(cpuset_getroot(set))); 1927a8f695aSBjoern A. Zeeb } 1937a8f695aSBjoern A. Zeeb 1947a8f695aSBjoern A. Zeeb /* 1957a8f695aSBjoern A. Zeeb * Find the first non-anonymous set starting from 'set'. Returns this set 1967a8f695aSBjoern A. Zeeb * referenced. May return the passed in set with an extra ref if it is 1977a8f695aSBjoern A. Zeeb * not anonymous. 1987a8f695aSBjoern A. Zeeb */ 1997a8f695aSBjoern A. Zeeb static struct cpuset * 2007a8f695aSBjoern A. Zeeb cpuset_refbase(struct cpuset *set) 2017a8f695aSBjoern A. Zeeb { 2027a8f695aSBjoern A. Zeeb 2033f289c3fSJeff Roberson return (cpuset_ref(cpuset_getbase(set))); 2047a8f695aSBjoern A. Zeeb } 2057a8f695aSBjoern A. Zeeb 2067a8f695aSBjoern A. Zeeb /* 20793902625SJohn Baldwin * Release a reference in a context where it is safe to allocate. 208d7f687fcSJeff Roberson */ 209d7f687fcSJeff Roberson void 210d7f687fcSJeff Roberson cpuset_rel(struct cpuset *set) 211d7f687fcSJeff Roberson { 212d7f687fcSJeff Roberson cpusetid_t id; 213d7f687fcSJeff Roberson 214b2780e85SKyle Evans if (refcount_release_if_not_last(&set->cs_ref)) 215d7f687fcSJeff Roberson return; 216d7f687fcSJeff Roberson mtx_lock_spin(&cpuset_lock); 217b2780e85SKyle Evans if (!refcount_release(&set->cs_ref)) { 218b2780e85SKyle Evans mtx_unlock_spin(&cpuset_lock); 219b2780e85SKyle Evans return; 220b2780e85SKyle Evans } 221d7f687fcSJeff Roberson LIST_REMOVE(set, cs_siblings); 222d7f687fcSJeff Roberson id = set->cs_id; 223d7f687fcSJeff Roberson if (id != CPUSET_INVALID) 224d7f687fcSJeff Roberson LIST_REMOVE(set, cs_link); 225d7f687fcSJeff Roberson mtx_unlock_spin(&cpuset_lock); 226d7f687fcSJeff Roberson cpuset_rel(set->cs_parent); 227d7f687fcSJeff Roberson uma_zfree(cpuset_zone, set); 228d7f687fcSJeff Roberson if (id != CPUSET_INVALID) 229d7f687fcSJeff Roberson free_unr(cpuset_unr, id); 230d7f687fcSJeff Roberson } 231d7f687fcSJeff Roberson 232d7f687fcSJeff Roberson /* 233d7f687fcSJeff Roberson * Deferred release must be used when in a context that is not safe to 234d7f687fcSJeff Roberson * allocate/free. This places any unreferenced sets on the list 'head'. 235d7f687fcSJeff Roberson */ 236d7f687fcSJeff Roberson static void 237d7f687fcSJeff Roberson cpuset_rel_defer(struct setlist *head, struct cpuset *set) 238d7f687fcSJeff Roberson { 239d7f687fcSJeff Roberson 240b2780e85SKyle Evans if (refcount_release_if_not_last(&set->cs_ref)) 241d7f687fcSJeff Roberson return; 242d7f687fcSJeff Roberson mtx_lock_spin(&cpuset_lock); 243b2780e85SKyle Evans if (!refcount_release(&set->cs_ref)) { 244b2780e85SKyle Evans mtx_unlock_spin(&cpuset_lock); 245b2780e85SKyle Evans return; 246b2780e85SKyle Evans } 247d7f687fcSJeff Roberson LIST_REMOVE(set, cs_siblings); 248d7f687fcSJeff Roberson if (set->cs_id != CPUSET_INVALID) 249d7f687fcSJeff Roberson LIST_REMOVE(set, cs_link); 250d7f687fcSJeff Roberson LIST_INSERT_HEAD(head, set, cs_link); 251d7f687fcSJeff Roberson mtx_unlock_spin(&cpuset_lock); 252d7f687fcSJeff Roberson } 253d7f687fcSJeff Roberson 254d7f687fcSJeff Roberson /* 255d7f687fcSJeff Roberson * Complete a deferred release. Removes the set from the list provided to 256d7f687fcSJeff Roberson * cpuset_rel_defer. 257d7f687fcSJeff Roberson */ 258d7f687fcSJeff Roberson static void 259d7f687fcSJeff Roberson cpuset_rel_complete(struct cpuset *set) 260d7f687fcSJeff Roberson { 2619c83dab9SKyle Evans cpusetid_t id; 2629c83dab9SKyle Evans 2639c83dab9SKyle Evans id = set->cs_id; 264d7f687fcSJeff Roberson LIST_REMOVE(set, cs_link); 265d7f687fcSJeff Roberson cpuset_rel(set->cs_parent); 266d7f687fcSJeff Roberson uma_zfree(cpuset_zone, set); 2679c83dab9SKyle Evans if (id != CPUSET_INVALID) 2689c83dab9SKyle Evans free_unr(cpuset_unr, id); 269d7f687fcSJeff Roberson } 270d7f687fcSJeff Roberson 271d7f687fcSJeff Roberson /* 272d7f687fcSJeff Roberson * Find a set based on an id. Returns it with a ref. 273d7f687fcSJeff Roberson */ 274d7f687fcSJeff Roberson static struct cpuset * 275413628a7SBjoern A. Zeeb cpuset_lookup(cpusetid_t setid, struct thread *td) 276d7f687fcSJeff Roberson { 277d7f687fcSJeff Roberson struct cpuset *set; 278d7f687fcSJeff Roberson 279d7f687fcSJeff Roberson if (setid == CPUSET_INVALID) 280d7f687fcSJeff Roberson return (NULL); 281d7f687fcSJeff Roberson mtx_lock_spin(&cpuset_lock); 282d7f687fcSJeff Roberson LIST_FOREACH(set, &cpuset_ids, cs_link) 283d7f687fcSJeff Roberson if (set->cs_id == setid) 284d7f687fcSJeff Roberson break; 285d7f687fcSJeff Roberson if (set) 286d7f687fcSJeff Roberson cpuset_ref(set); 287d7f687fcSJeff Roberson mtx_unlock_spin(&cpuset_lock); 288413628a7SBjoern A. Zeeb 289413628a7SBjoern A. Zeeb KASSERT(td != NULL, ("[%s:%d] td is NULL", __func__, __LINE__)); 290413628a7SBjoern A. Zeeb if (set != NULL && jailed(td->td_ucred)) { 2910304c731SJamie Gritton struct cpuset *jset, *tset; 292413628a7SBjoern A. Zeeb 2930304c731SJamie Gritton jset = td->td_ucred->cr_prison->pr_cpuset; 2940304c731SJamie Gritton for (tset = set; tset != NULL; tset = tset->cs_parent) 2950304c731SJamie Gritton if (tset == jset) 2960304c731SJamie Gritton break; 2970304c731SJamie Gritton if (tset == NULL) { 298413628a7SBjoern A. Zeeb cpuset_rel(set); 299413628a7SBjoern A. Zeeb set = NULL; 300413628a7SBjoern A. Zeeb } 301413628a7SBjoern A. Zeeb } 302413628a7SBjoern A. Zeeb 303d7f687fcSJeff Roberson return (set); 304d7f687fcSJeff Roberson } 305d7f687fcSJeff Roberson 306d7f687fcSJeff Roberson /* 30730b7c6f9SKyle Evans * Initialize a set in the space provided in 'set' with the provided parameters. 308d7f687fcSJeff Roberson * The set is returned with a single ref. May return EDEADLK if the set 309d7f687fcSJeff Roberson * will have no valid cpu based on restrictions from the parent. 310d7f687fcSJeff Roberson */ 311d7f687fcSJeff Roberson static int 31230b7c6f9SKyle Evans cpuset_init(struct cpuset *set, struct cpuset *parent, 3133f289c3fSJeff Roberson const cpuset_t *mask, struct domainset *domain, cpusetid_t id) 314d7f687fcSJeff Roberson { 315d7f687fcSJeff Roberson 3163f289c3fSJeff Roberson if (domain == NULL) 3173f289c3fSJeff Roberson domain = parent->cs_domain; 3183f289c3fSJeff Roberson if (mask == NULL) 3193f289c3fSJeff Roberson mask = &parent->cs_mask; 32073c40187SJeff Roberson if (!CPU_OVERLAP(&parent->cs_mask, mask)) 32173c40187SJeff Roberson return (EDEADLK); 3223f289c3fSJeff Roberson /* The domain must be prepared ahead of time. */ 3233f289c3fSJeff Roberson if (!domainset_valid(parent->cs_domain, domain)) 3243f289c3fSJeff Roberson return (EDEADLK); 325d7f687fcSJeff Roberson CPU_COPY(mask, &set->cs_mask); 326d7f687fcSJeff Roberson LIST_INIT(&set->cs_children); 327d7f687fcSJeff Roberson refcount_init(&set->cs_ref, 1); 328d7f687fcSJeff Roberson set->cs_flags = 0; 329d7f687fcSJeff Roberson mtx_lock_spin(&cpuset_lock); 3303f289c3fSJeff Roberson set->cs_domain = domain; 331e2650af1SStefan Eßer CPU_AND(&set->cs_mask, &set->cs_mask, &parent->cs_mask); 332d7f687fcSJeff Roberson set->cs_id = id; 333d7f687fcSJeff Roberson set->cs_parent = cpuset_ref(parent); 334d7f687fcSJeff Roberson LIST_INSERT_HEAD(&parent->cs_children, set, cs_siblings); 335d7f687fcSJeff Roberson if (set->cs_id != CPUSET_INVALID) 336d7f687fcSJeff Roberson LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 337d7f687fcSJeff Roberson mtx_unlock_spin(&cpuset_lock); 338d7f687fcSJeff Roberson 33973c40187SJeff Roberson return (0); 340d7f687fcSJeff Roberson } 341d7f687fcSJeff Roberson 342d7f687fcSJeff Roberson /* 343d7f687fcSJeff Roberson * Create a new non-anonymous set with the requested parent and mask. May 344d7f687fcSJeff Roberson * return failures if the mask is invalid or a new number can not be 345d7f687fcSJeff Roberson * allocated. 34629d04ea8SKyle Evans * 34729d04ea8SKyle Evans * If *setp is not NULL, then it will be used as-is. The caller must take 34829d04ea8SKyle Evans * into account that *setp will be inserted at the head of cpuset_ids and 34929d04ea8SKyle Evans * plan any potentially conflicting cs_link usage accordingly. 350d7f687fcSJeff Roberson */ 351d7f687fcSJeff Roberson static int 352e84c2db1SJohn Baldwin cpuset_create(struct cpuset **setp, struct cpuset *parent, const cpuset_t *mask) 353d7f687fcSJeff Roberson { 354d7f687fcSJeff Roberson struct cpuset *set; 355d7f687fcSJeff Roberson cpusetid_t id; 356d7f687fcSJeff Roberson int error; 35729d04ea8SKyle Evans bool dofree; 358d7f687fcSJeff Roberson 359d7f687fcSJeff Roberson id = alloc_unr(cpuset_unr); 360d7f687fcSJeff Roberson if (id == -1) 361d7f687fcSJeff Roberson return (ENFILE); 36229d04ea8SKyle Evans dofree = (*setp == NULL); 36329d04ea8SKyle Evans if (*setp != NULL) 36429d04ea8SKyle Evans set = *setp; 36529d04ea8SKyle Evans else 3663f289c3fSJeff Roberson *setp = set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 36730b7c6f9SKyle Evans error = cpuset_init(set, parent, mask, NULL, id); 368d7f687fcSJeff Roberson if (error == 0) 369d7f687fcSJeff Roberson return (0); 370d7f687fcSJeff Roberson free_unr(cpuset_unr, id); 37129d04ea8SKyle Evans if (dofree) 372d7f687fcSJeff Roberson uma_zfree(cpuset_zone, set); 373d7f687fcSJeff Roberson 374d7f687fcSJeff Roberson return (error); 375d7f687fcSJeff Roberson } 376d7f687fcSJeff Roberson 3773f289c3fSJeff Roberson static void 3783f289c3fSJeff Roberson cpuset_freelist_add(struct setlist *list, int count) 3793f289c3fSJeff Roberson { 3803f289c3fSJeff Roberson struct cpuset *set; 3813f289c3fSJeff Roberson int i; 3823f289c3fSJeff Roberson 3833f289c3fSJeff Roberson for (i = 0; i < count; i++) { 3843f289c3fSJeff Roberson set = uma_zalloc(cpuset_zone, M_ZERO | M_WAITOK); 3853f289c3fSJeff Roberson LIST_INSERT_HEAD(list, set, cs_link); 3863f289c3fSJeff Roberson } 3873f289c3fSJeff Roberson } 3883f289c3fSJeff Roberson 3893f289c3fSJeff Roberson static void 3903f289c3fSJeff Roberson cpuset_freelist_init(struct setlist *list, int count) 3913f289c3fSJeff Roberson { 3923f289c3fSJeff Roberson 3933f289c3fSJeff Roberson LIST_INIT(list); 3943f289c3fSJeff Roberson cpuset_freelist_add(list, count); 3953f289c3fSJeff Roberson } 3963f289c3fSJeff Roberson 3973f289c3fSJeff Roberson static void 3983f289c3fSJeff Roberson cpuset_freelist_free(struct setlist *list) 3993f289c3fSJeff Roberson { 4003f289c3fSJeff Roberson struct cpuset *set; 4013f289c3fSJeff Roberson 4023f289c3fSJeff Roberson while ((set = LIST_FIRST(list)) != NULL) { 4033f289c3fSJeff Roberson LIST_REMOVE(set, cs_link); 4043f289c3fSJeff Roberson uma_zfree(cpuset_zone, set); 4053f289c3fSJeff Roberson } 4063f289c3fSJeff Roberson } 4073f289c3fSJeff Roberson 4083f289c3fSJeff Roberson static void 4093f289c3fSJeff Roberson domainset_freelist_add(struct domainlist *list, int count) 4103f289c3fSJeff Roberson { 4113f289c3fSJeff Roberson struct domainset *set; 4123f289c3fSJeff Roberson int i; 4133f289c3fSJeff Roberson 4143f289c3fSJeff Roberson for (i = 0; i < count; i++) { 4153f289c3fSJeff Roberson set = uma_zalloc(domainset_zone, M_ZERO | M_WAITOK); 4163f289c3fSJeff Roberson LIST_INSERT_HEAD(list, set, ds_link); 4173f289c3fSJeff Roberson } 4183f289c3fSJeff Roberson } 4193f289c3fSJeff Roberson 4203f289c3fSJeff Roberson static void 4213f289c3fSJeff Roberson domainset_freelist_init(struct domainlist *list, int count) 4223f289c3fSJeff Roberson { 4233f289c3fSJeff Roberson 4243f289c3fSJeff Roberson LIST_INIT(list); 4253f289c3fSJeff Roberson domainset_freelist_add(list, count); 4263f289c3fSJeff Roberson } 4273f289c3fSJeff Roberson 4283f289c3fSJeff Roberson static void 4293f289c3fSJeff Roberson domainset_freelist_free(struct domainlist *list) 4303f289c3fSJeff Roberson { 4313f289c3fSJeff Roberson struct domainset *set; 4323f289c3fSJeff Roberson 4333f289c3fSJeff Roberson while ((set = LIST_FIRST(list)) != NULL) { 4343f289c3fSJeff Roberson LIST_REMOVE(set, ds_link); 4353f289c3fSJeff Roberson uma_zfree(domainset_zone, set); 4363f289c3fSJeff Roberson } 4373f289c3fSJeff Roberson } 4383f289c3fSJeff Roberson 4393f289c3fSJeff Roberson /* Copy a domainset preserving mask and policy. */ 4403f289c3fSJeff Roberson static void 4413f289c3fSJeff Roberson domainset_copy(const struct domainset *from, struct domainset *to) 4423f289c3fSJeff Roberson { 4433f289c3fSJeff Roberson 4443f289c3fSJeff Roberson DOMAINSET_COPY(&from->ds_mask, &to->ds_mask); 4453f289c3fSJeff Roberson to->ds_policy = from->ds_policy; 4463f289c3fSJeff Roberson to->ds_prefer = from->ds_prefer; 4473f289c3fSJeff Roberson } 4483f289c3fSJeff Roberson 4493f289c3fSJeff Roberson /* Return 1 if mask and policy are equal, otherwise 0. */ 4503f289c3fSJeff Roberson static int 4513f289c3fSJeff Roberson domainset_equal(const struct domainset *one, const struct domainset *two) 4523f289c3fSJeff Roberson { 4533f289c3fSJeff Roberson 4543f289c3fSJeff Roberson return (DOMAINSET_CMP(&one->ds_mask, &two->ds_mask) == 0 && 4553f289c3fSJeff Roberson one->ds_policy == two->ds_policy && 4563f289c3fSJeff Roberson one->ds_prefer == two->ds_prefer); 4573f289c3fSJeff Roberson } 4583f289c3fSJeff Roberson 4593f289c3fSJeff Roberson /* Return 1 if child is a valid subset of parent. */ 4603f289c3fSJeff Roberson static int 4613f289c3fSJeff Roberson domainset_valid(const struct domainset *parent, const struct domainset *child) 4623f289c3fSJeff Roberson { 4633f289c3fSJeff Roberson if (child->ds_policy != DOMAINSET_POLICY_PREFER) 4643f289c3fSJeff Roberson return (DOMAINSET_SUBSET(&parent->ds_mask, &child->ds_mask)); 4653f289c3fSJeff Roberson return (DOMAINSET_ISSET(child->ds_prefer, &parent->ds_mask)); 4663f289c3fSJeff Roberson } 4673f289c3fSJeff Roberson 4683f289c3fSJeff Roberson static int 4693f289c3fSJeff Roberson domainset_restrict(const struct domainset *parent, 4703f289c3fSJeff Roberson const struct domainset *child) 4713f289c3fSJeff Roberson { 4723f289c3fSJeff Roberson if (child->ds_policy != DOMAINSET_POLICY_PREFER) 4733f289c3fSJeff Roberson return (DOMAINSET_OVERLAP(&parent->ds_mask, &child->ds_mask)); 4743f289c3fSJeff Roberson return (DOMAINSET_ISSET(child->ds_prefer, &parent->ds_mask)); 4753f289c3fSJeff Roberson } 4763f289c3fSJeff Roberson 4773f289c3fSJeff Roberson /* 4783f289c3fSJeff Roberson * Lookup or create a domainset. The key is provided in ds_mask and 4793f289c3fSJeff Roberson * ds_policy. If the domainset does not yet exist the storage in 4803f289c3fSJeff Roberson * 'domain' is used to insert. Otherwise this storage is freed to the 4813f289c3fSJeff Roberson * domainset_zone and the existing domainset is returned. 4823f289c3fSJeff Roberson */ 4833f289c3fSJeff Roberson static struct domainset * 4843f289c3fSJeff Roberson _domainset_create(struct domainset *domain, struct domainlist *freelist) 4853f289c3fSJeff Roberson { 4863f289c3fSJeff Roberson struct domainset *ndomain; 48744e4def7SMark Johnston int i, j; 4883f289c3fSJeff Roberson 489b61f3142SMark Johnston KASSERT(domain->ds_cnt <= vm_ndomains, 490b61f3142SMark Johnston ("invalid domain count in domainset %p", domain)); 491b61f3142SMark Johnston KASSERT(domain->ds_policy != DOMAINSET_POLICY_PREFER || 492b61f3142SMark Johnston domain->ds_prefer < vm_ndomains, 493b61f3142SMark Johnston ("invalid preferred domain in domains %p", domain)); 494b61f3142SMark Johnston 4953f289c3fSJeff Roberson mtx_lock_spin(&cpuset_lock); 4963f289c3fSJeff Roberson LIST_FOREACH(ndomain, &cpuset_domains, ds_link) 4973f289c3fSJeff Roberson if (domainset_equal(ndomain, domain)) 4983f289c3fSJeff Roberson break; 4993f289c3fSJeff Roberson /* 5003f289c3fSJeff Roberson * If the domain does not yet exist we insert it and initialize 5013f289c3fSJeff Roberson * various iteration helpers which are not part of the key. 5023f289c3fSJeff Roberson */ 5033f289c3fSJeff Roberson if (ndomain == NULL) { 5043f289c3fSJeff Roberson LIST_INSERT_HEAD(&cpuset_domains, domain, ds_link); 5053f289c3fSJeff Roberson domain->ds_cnt = DOMAINSET_COUNT(&domain->ds_mask); 50644e4def7SMark Johnston for (i = 0, j = 0; i < DOMAINSET_FLS(&domain->ds_mask); i++) 507e5818a53SJeff Roberson if (DOMAINSET_ISSET(i, &domain->ds_mask)) 508e5818a53SJeff Roberson domain->ds_order[j++] = i; 5093f289c3fSJeff Roberson } 5103f289c3fSJeff Roberson mtx_unlock_spin(&cpuset_lock); 5113f289c3fSJeff Roberson if (ndomain == NULL) 5123f289c3fSJeff Roberson return (domain); 5133f289c3fSJeff Roberson if (freelist != NULL) 5143f289c3fSJeff Roberson LIST_INSERT_HEAD(freelist, domain, ds_link); 5153f289c3fSJeff Roberson else 5163f289c3fSJeff Roberson uma_zfree(domainset_zone, domain); 5173f289c3fSJeff Roberson return (ndomain); 5183f289c3fSJeff Roberson 5193f289c3fSJeff Roberson } 5203f289c3fSJeff Roberson 5213f289c3fSJeff Roberson /* 52230c5525bSAndrew Gallatin * Are any of the domains in the mask empty? If so, silently 523920239efSMark Johnston * remove them and update the domainset accordingly. If only empty 524920239efSMark Johnston * domains are present, we must return failure. 52530c5525bSAndrew Gallatin */ 52630c5525bSAndrew Gallatin static bool 52730c5525bSAndrew Gallatin domainset_empty_vm(struct domainset *domain) 52830c5525bSAndrew Gallatin { 5298e697504SMark Johnston domainset_t empty; 5308e697504SMark Johnston int i, j; 53130c5525bSAndrew Gallatin 5328e697504SMark Johnston DOMAINSET_ZERO(&empty); 5338e697504SMark Johnston for (i = 0; i < vm_ndomains; i++) 5348e697504SMark Johnston if (VM_DOMAIN_EMPTY(i)) 5358e697504SMark Johnston DOMAINSET_SET(i, &empty); 5368e697504SMark Johnston if (DOMAINSET_SUBSET(&empty, &domain->ds_mask)) 5378e697504SMark Johnston return (true); 5388e697504SMark Johnston 5398e697504SMark Johnston /* Remove empty domains from the set and recompute. */ 5409825eadfSRyan Libby DOMAINSET_ANDNOT(&domain->ds_mask, &empty); 541920239efSMark Johnston domain->ds_cnt = DOMAINSET_COUNT(&domain->ds_mask); 5428e697504SMark Johnston for (i = j = 0; i < DOMAINSET_FLS(&domain->ds_mask); i++) 543920239efSMark Johnston if (DOMAINSET_ISSET(i, &domain->ds_mask)) 544920239efSMark Johnston domain->ds_order[j++] = i; 5458e697504SMark Johnston 5468e697504SMark Johnston /* Convert a PREFER policy referencing an empty domain to RR. */ 5478e697504SMark Johnston if (domain->ds_policy == DOMAINSET_POLICY_PREFER && 5488e697504SMark Johnston DOMAINSET_ISSET(domain->ds_prefer, &empty)) { 549920239efSMark Johnston domain->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; 550920239efSMark Johnston domain->ds_prefer = -1; 551920239efSMark Johnston } 55230c5525bSAndrew Gallatin 5538e697504SMark Johnston return (false); 55430c5525bSAndrew Gallatin } 55530c5525bSAndrew Gallatin 55630c5525bSAndrew Gallatin /* 5573f289c3fSJeff Roberson * Create or lookup a domainset based on the key held in 'domain'. 5583f289c3fSJeff Roberson */ 559e5818a53SJeff Roberson struct domainset * 5603f289c3fSJeff Roberson domainset_create(const struct domainset *domain) 5613f289c3fSJeff Roberson { 5623f289c3fSJeff Roberson struct domainset *ndomain; 5633f289c3fSJeff Roberson 564e5818a53SJeff Roberson /* 565e5818a53SJeff Roberson * Validate the policy. It must specify a useable policy number with 566e5818a53SJeff Roberson * only valid domains. Preferred must include the preferred domain 567e5818a53SJeff Roberson * in the mask. 568e5818a53SJeff Roberson */ 569e5818a53SJeff Roberson if (domain->ds_policy <= DOMAINSET_POLICY_INVALID || 570e5818a53SJeff Roberson domain->ds_policy > DOMAINSET_POLICY_MAX) 571e5818a53SJeff Roberson return (NULL); 572e5818a53SJeff Roberson if (domain->ds_policy == DOMAINSET_POLICY_PREFER && 573e5818a53SJeff Roberson !DOMAINSET_ISSET(domain->ds_prefer, &domain->ds_mask)) 574e5818a53SJeff Roberson return (NULL); 57529bb6c19SMark Johnston if (!DOMAINSET_SUBSET(&domainset0->ds_mask, &domain->ds_mask)) 576e5818a53SJeff Roberson return (NULL); 5773f289c3fSJeff Roberson ndomain = uma_zalloc(domainset_zone, M_WAITOK | M_ZERO); 5783f289c3fSJeff Roberson domainset_copy(domain, ndomain); 5793f289c3fSJeff Roberson return _domainset_create(ndomain, NULL); 5803f289c3fSJeff Roberson } 5813f289c3fSJeff Roberson 5823f289c3fSJeff Roberson /* 5833f289c3fSJeff Roberson * Update thread domainset pointers. 5843f289c3fSJeff Roberson */ 5853f289c3fSJeff Roberson static void 5863f289c3fSJeff Roberson domainset_notify(void) 5873f289c3fSJeff Roberson { 5883f289c3fSJeff Roberson struct thread *td; 5893f289c3fSJeff Roberson struct proc *p; 5903f289c3fSJeff Roberson 5913f289c3fSJeff Roberson sx_slock(&allproc_lock); 5923f289c3fSJeff Roberson FOREACH_PROC_IN_SYSTEM(p) { 5933f289c3fSJeff Roberson PROC_LOCK(p); 5943f289c3fSJeff Roberson if (p->p_state == PRS_NEW) { 5953f289c3fSJeff Roberson PROC_UNLOCK(p); 5963f289c3fSJeff Roberson continue; 5973f289c3fSJeff Roberson } 5983f289c3fSJeff Roberson FOREACH_THREAD_IN_PROC(p, td) { 5993f289c3fSJeff Roberson thread_lock(td); 6003f289c3fSJeff Roberson td->td_domain.dr_policy = td->td_cpuset->cs_domain; 6013f289c3fSJeff Roberson thread_unlock(td); 6023f289c3fSJeff Roberson } 6033f289c3fSJeff Roberson PROC_UNLOCK(p); 6043f289c3fSJeff Roberson } 6053f289c3fSJeff Roberson sx_sunlock(&allproc_lock); 606e5818a53SJeff Roberson kernel_object->domain.dr_policy = cpuset_kernel->cs_domain; 6073f289c3fSJeff Roberson } 6083f289c3fSJeff Roberson 6093f289c3fSJeff Roberson /* 6103f289c3fSJeff Roberson * Create a new set that is a subset of a parent. 6113f289c3fSJeff Roberson */ 6123f289c3fSJeff Roberson static struct domainset * 6133f289c3fSJeff Roberson domainset_shadow(const struct domainset *pdomain, 6143f289c3fSJeff Roberson const struct domainset *domain, struct domainlist *freelist) 6153f289c3fSJeff Roberson { 6163f289c3fSJeff Roberson struct domainset *ndomain; 6173f289c3fSJeff Roberson 6183f289c3fSJeff Roberson ndomain = LIST_FIRST(freelist); 6193f289c3fSJeff Roberson LIST_REMOVE(ndomain, ds_link); 6203f289c3fSJeff Roberson 6213f289c3fSJeff Roberson /* 6223f289c3fSJeff Roberson * Initialize the key from the request. 6233f289c3fSJeff Roberson */ 6243f289c3fSJeff Roberson domainset_copy(domain, ndomain); 6253f289c3fSJeff Roberson 6263f289c3fSJeff Roberson /* 6273f289c3fSJeff Roberson * Restrict the key by the parent. 6283f289c3fSJeff Roberson */ 6293f289c3fSJeff Roberson DOMAINSET_AND(&ndomain->ds_mask, &pdomain->ds_mask); 6303f289c3fSJeff Roberson 6313f289c3fSJeff Roberson return _domainset_create(ndomain, freelist); 6323f289c3fSJeff Roberson } 6333f289c3fSJeff Roberson 634d7f687fcSJeff Roberson /* 635d7f687fcSJeff Roberson * Recursively check for errors that would occur from applying mask to 636d7f687fcSJeff Roberson * the tree of sets starting at 'set'. Checks for sets that would become 637d7f687fcSJeff Roberson * empty as well as RDONLY flags. 638d7f687fcSJeff Roberson */ 639d7f687fcSJeff Roberson static int 640f1b18a66SKyle Evans cpuset_testupdate(struct cpuset *set, cpuset_t *mask, int augment_mask) 641d7f687fcSJeff Roberson { 642d7f687fcSJeff Roberson struct cpuset *nset; 643d7f687fcSJeff Roberson cpuset_t newmask; 644d7f687fcSJeff Roberson int error; 645d7f687fcSJeff Roberson 646d7f687fcSJeff Roberson mtx_assert(&cpuset_lock, MA_OWNED); 647d7f687fcSJeff Roberson if (set->cs_flags & CPU_SET_RDONLY) 648d7f687fcSJeff Roberson return (EPERM); 649f1b18a66SKyle Evans if (augment_mask) { 650e2650af1SStefan Eßer CPU_AND(&newmask, &set->cs_mask, mask); 651c9813d0aSJohn Baldwin } else 652c9813d0aSJohn Baldwin CPU_COPY(mask, &newmask); 653f1b18a66SKyle Evans 654f1b18a66SKyle Evans if (CPU_EMPTY(&newmask)) 655f1b18a66SKyle Evans return (EDEADLK); 65673c40187SJeff Roberson error = 0; 657d7f687fcSJeff Roberson LIST_FOREACH(nset, &set->cs_children, cs_siblings) 658c9813d0aSJohn Baldwin if ((error = cpuset_testupdate(nset, &newmask, 1)) != 0) 659d7f687fcSJeff Roberson break; 660d7f687fcSJeff Roberson return (error); 661d7f687fcSJeff Roberson } 662d7f687fcSJeff Roberson 663d7f687fcSJeff Roberson /* 664d7f687fcSJeff Roberson * Applies the mask 'mask' without checking for empty sets or permissions. 665d7f687fcSJeff Roberson */ 666d7f687fcSJeff Roberson static void 667d7f687fcSJeff Roberson cpuset_update(struct cpuset *set, cpuset_t *mask) 668d7f687fcSJeff Roberson { 669d7f687fcSJeff Roberson struct cpuset *nset; 670d7f687fcSJeff Roberson 671d7f687fcSJeff Roberson mtx_assert(&cpuset_lock, MA_OWNED); 672e2650af1SStefan Eßer CPU_AND(&set->cs_mask, &set->cs_mask, mask); 673d7f687fcSJeff Roberson LIST_FOREACH(nset, &set->cs_children, cs_siblings) 674d7f687fcSJeff Roberson cpuset_update(nset, &set->cs_mask); 675d7f687fcSJeff Roberson 676d7f687fcSJeff Roberson return; 677d7f687fcSJeff Roberson } 678d7f687fcSJeff Roberson 679d7f687fcSJeff Roberson /* 680d7f687fcSJeff Roberson * Modify the set 'set' to use a copy of the mask provided. Apply this new 681d7f687fcSJeff Roberson * mask to restrict all children in the tree. Checks for validity before 682d7f687fcSJeff Roberson * applying the changes. 683d7f687fcSJeff Roberson */ 684d7f687fcSJeff Roberson static int 685d7f687fcSJeff Roberson cpuset_modify(struct cpuset *set, cpuset_t *mask) 686d7f687fcSJeff Roberson { 68773c40187SJeff Roberson struct cpuset *root; 688d7f687fcSJeff Roberson int error; 689d7f687fcSJeff Roberson 690ba931c08SBjoern A. Zeeb error = priv_check(curthread, PRIV_SCHED_CPUSET); 691d7f687fcSJeff Roberson if (error) 692d7f687fcSJeff Roberson return (error); 69373c40187SJeff Roberson /* 69454a837c8SKyle Evans * In case we are called from within the jail, 6956aaa0b3cSBjoern A. Zeeb * we do not allow modifying the dedicated root 6966aaa0b3cSBjoern A. Zeeb * cpuset of the jail but may still allow to 69754a837c8SKyle Evans * change child sets, including subordinate jails' 69854a837c8SKyle Evans * roots. 6996aaa0b3cSBjoern A. Zeeb */ 70054a837c8SKyle Evans if ((set->cs_flags & CPU_SET_ROOT) != 0 && 70154a837c8SKyle Evans jailed(curthread->td_ucred) && 70254a837c8SKyle Evans set == curthread->td_ucred->cr_prison->pr_cpuset) 7036aaa0b3cSBjoern A. Zeeb return (EPERM); 7046aaa0b3cSBjoern A. Zeeb /* 70573c40187SJeff Roberson * Verify that we have access to this set of 70673c40187SJeff Roberson * cpus. 70773c40187SJeff Roberson */ 70854a837c8SKyle Evans if ((set->cs_flags & (CPU_SET_ROOT | CPU_SET_RDONLY)) == CPU_SET_ROOT) { 70954a837c8SKyle Evans KASSERT(set->cs_parent != NULL, 71054a837c8SKyle Evans ("jail.cpuset=%d is not a proper child of parent jail's root.", 71154a837c8SKyle Evans set->cs_id)); 71254a837c8SKyle Evans 71354a837c8SKyle Evans /* 71454a837c8SKyle Evans * cpuset_getroot() cannot work here due to how top-level jail 71554a837c8SKyle Evans * roots are constructed. Top-level jails are parented to 71654a837c8SKyle Evans * thread0's cpuset (i.e. cpuset 1) rather than the system root. 71754a837c8SKyle Evans */ 71854a837c8SKyle Evans root = set->cs_parent; 71954a837c8SKyle Evans } else { 7203f289c3fSJeff Roberson root = cpuset_getroot(set); 72154a837c8SKyle Evans } 722d7f687fcSJeff Roberson mtx_lock_spin(&cpuset_lock); 7233f289c3fSJeff Roberson if (root && !CPU_SUBSET(&root->cs_mask, mask)) { 7243f289c3fSJeff Roberson error = EINVAL; 7253f289c3fSJeff Roberson goto out; 7263f289c3fSJeff Roberson } 727c9813d0aSJohn Baldwin error = cpuset_testupdate(set, mask, 0); 728d7f687fcSJeff Roberson if (error) 729d7f687fcSJeff Roberson goto out; 730d7f687fcSJeff Roberson CPU_COPY(mask, &set->cs_mask); 731c9813d0aSJohn Baldwin cpuset_update(set, mask); 732d7f687fcSJeff Roberson out: 733d7f687fcSJeff Roberson mtx_unlock_spin(&cpuset_lock); 734d7f687fcSJeff Roberson 735d7f687fcSJeff Roberson return (error); 736d7f687fcSJeff Roberson } 737d7f687fcSJeff Roberson 738d7f687fcSJeff Roberson /* 7393f289c3fSJeff Roberson * Recursively check for errors that would occur from applying mask to 7403f289c3fSJeff Roberson * the tree of sets starting at 'set'. Checks for sets that would become 7413f289c3fSJeff Roberson * empty as well as RDONLY flags. 7423f289c3fSJeff Roberson */ 7433f289c3fSJeff Roberson static int 7443f289c3fSJeff Roberson cpuset_testupdate_domain(struct cpuset *set, struct domainset *dset, 745f1b18a66SKyle Evans struct domainset *orig, int *count, int augment_mask __unused) 7463f289c3fSJeff Roberson { 7473f289c3fSJeff Roberson struct cpuset *nset; 7483f289c3fSJeff Roberson struct domainset *domain; 7493f289c3fSJeff Roberson struct domainset newset; 7503f289c3fSJeff Roberson int error; 7513f289c3fSJeff Roberson 7523f289c3fSJeff Roberson mtx_assert(&cpuset_lock, MA_OWNED); 7533f289c3fSJeff Roberson if (set->cs_flags & CPU_SET_RDONLY) 7543f289c3fSJeff Roberson return (EPERM); 7553f289c3fSJeff Roberson domain = set->cs_domain; 7563f289c3fSJeff Roberson domainset_copy(domain, &newset); 7573f289c3fSJeff Roberson if (!domainset_equal(domain, orig)) { 7583f289c3fSJeff Roberson if (!domainset_restrict(domain, dset)) 7593f289c3fSJeff Roberson return (EDEADLK); 7603f289c3fSJeff Roberson DOMAINSET_AND(&newset.ds_mask, &dset->ds_mask); 7613f289c3fSJeff Roberson /* Count the number of domains that are changing. */ 7623f289c3fSJeff Roberson (*count)++; 7633f289c3fSJeff Roberson } 7643f289c3fSJeff Roberson error = 0; 7653f289c3fSJeff Roberson LIST_FOREACH(nset, &set->cs_children, cs_siblings) 7663f289c3fSJeff Roberson if ((error = cpuset_testupdate_domain(nset, &newset, domain, 7673f289c3fSJeff Roberson count, 1)) != 0) 7683f289c3fSJeff Roberson break; 7693f289c3fSJeff Roberson return (error); 7703f289c3fSJeff Roberson } 7713f289c3fSJeff Roberson 7723f289c3fSJeff Roberson /* 7733f289c3fSJeff Roberson * Applies the mask 'mask' without checking for empty sets or permissions. 7743f289c3fSJeff Roberson */ 7753f289c3fSJeff Roberson static void 7763f289c3fSJeff Roberson cpuset_update_domain(struct cpuset *set, struct domainset *domain, 7773f289c3fSJeff Roberson struct domainset *orig, struct domainlist *domains) 7783f289c3fSJeff Roberson { 7793f289c3fSJeff Roberson struct cpuset *nset; 7803f289c3fSJeff Roberson 7813f289c3fSJeff Roberson mtx_assert(&cpuset_lock, MA_OWNED); 7823f289c3fSJeff Roberson /* 7833f289c3fSJeff Roberson * If this domainset has changed from the parent we must calculate 7843f289c3fSJeff Roberson * a new set. Otherwise it simply inherits from the parent. When 7853f289c3fSJeff Roberson * we inherit from the parent we get a new mask and policy. If the 7863f289c3fSJeff Roberson * set is modified from the parent we keep the policy and only 7873f289c3fSJeff Roberson * update the mask. 7883f289c3fSJeff Roberson */ 7893f289c3fSJeff Roberson if (set->cs_domain != orig) { 7903f289c3fSJeff Roberson orig = set->cs_domain; 7913f289c3fSJeff Roberson set->cs_domain = domainset_shadow(domain, orig, domains); 7923f289c3fSJeff Roberson } else 7933f289c3fSJeff Roberson set->cs_domain = domain; 7943f289c3fSJeff Roberson LIST_FOREACH(nset, &set->cs_children, cs_siblings) 7953f289c3fSJeff Roberson cpuset_update_domain(nset, set->cs_domain, orig, domains); 7963f289c3fSJeff Roberson 7973f289c3fSJeff Roberson return; 7983f289c3fSJeff Roberson } 7993f289c3fSJeff Roberson 8003f289c3fSJeff Roberson /* 8013f289c3fSJeff Roberson * Modify the set 'set' to use a copy the domainset provided. Apply this new 8023f289c3fSJeff Roberson * mask to restrict all children in the tree. Checks for validity before 8033f289c3fSJeff Roberson * applying the changes. 8043f289c3fSJeff Roberson */ 8053f289c3fSJeff Roberson static int 8063f289c3fSJeff Roberson cpuset_modify_domain(struct cpuset *set, struct domainset *domain) 8073f289c3fSJeff Roberson { 8083f289c3fSJeff Roberson struct domainlist domains; 8093f289c3fSJeff Roberson struct domainset temp; 8103f289c3fSJeff Roberson struct domainset *dset; 8113f289c3fSJeff Roberson struct cpuset *root; 8123f289c3fSJeff Roberson int ndomains, needed; 8133f289c3fSJeff Roberson int error; 8143f289c3fSJeff Roberson 8153f289c3fSJeff Roberson error = priv_check(curthread, PRIV_SCHED_CPUSET); 8163f289c3fSJeff Roberson if (error) 8173f289c3fSJeff Roberson return (error); 8183f289c3fSJeff Roberson /* 8193f289c3fSJeff Roberson * In case we are called from within the jail 8203f289c3fSJeff Roberson * we do not allow modifying the dedicated root 8213f289c3fSJeff Roberson * cpuset of the jail but may still allow to 8223f289c3fSJeff Roberson * change child sets. 8233f289c3fSJeff Roberson */ 8243f289c3fSJeff Roberson if (jailed(curthread->td_ucred) && 8253f289c3fSJeff Roberson set->cs_flags & CPU_SET_ROOT) 8263f289c3fSJeff Roberson return (EPERM); 8273f289c3fSJeff Roberson domainset_freelist_init(&domains, 0); 8283f289c3fSJeff Roberson domain = domainset_create(domain); 829e07e3fa3SKyle Evans ndomains = 0; 830e07e3fa3SKyle Evans 8313f289c3fSJeff Roberson mtx_lock_spin(&cpuset_lock); 832e07e3fa3SKyle Evans for (;;) { 833e07e3fa3SKyle Evans root = cpuset_getroot(set); 8343f289c3fSJeff Roberson dset = root->cs_domain; 8353f289c3fSJeff Roberson /* 8363f289c3fSJeff Roberson * Verify that we have access to this set of domains. 8373f289c3fSJeff Roberson */ 83845cdd437SMark Johnston if (!domainset_valid(dset, domain)) { 8393f289c3fSJeff Roberson error = EINVAL; 8403f289c3fSJeff Roberson goto out; 8413f289c3fSJeff Roberson } 8423f289c3fSJeff Roberson /* 8433f289c3fSJeff Roberson * If applying prefer we keep the current set as the fallback. 8443f289c3fSJeff Roberson */ 8453f289c3fSJeff Roberson if (domain->ds_policy == DOMAINSET_POLICY_PREFER) 8463f289c3fSJeff Roberson DOMAINSET_COPY(&set->cs_domain->ds_mask, 8473f289c3fSJeff Roberson &domain->ds_mask); 8483f289c3fSJeff Roberson /* 8493f289c3fSJeff Roberson * Determine whether we can apply this set of domains and 8503f289c3fSJeff Roberson * how many new domain structures it will require. 8513f289c3fSJeff Roberson */ 8523f289c3fSJeff Roberson domainset_copy(domain, &temp); 8533f289c3fSJeff Roberson needed = 0; 8543f289c3fSJeff Roberson error = cpuset_testupdate_domain(set, &temp, set->cs_domain, 8553f289c3fSJeff Roberson &needed, 0); 8563f289c3fSJeff Roberson if (error) 8573f289c3fSJeff Roberson goto out; 858e07e3fa3SKyle Evans if (ndomains >= needed) 859e07e3fa3SKyle Evans break; 860e07e3fa3SKyle Evans 861e07e3fa3SKyle Evans /* Dropping the lock; we'll need to re-evaluate again. */ 862e07e3fa3SKyle Evans mtx_unlock_spin(&cpuset_lock); 863e07e3fa3SKyle Evans domainset_freelist_add(&domains, needed - ndomains); 864e07e3fa3SKyle Evans ndomains = needed; 865e07e3fa3SKyle Evans mtx_lock_spin(&cpuset_lock); 866e07e3fa3SKyle Evans } 8673f289c3fSJeff Roberson dset = set->cs_domain; 8683f289c3fSJeff Roberson cpuset_update_domain(set, domain, dset, &domains); 8693f289c3fSJeff Roberson out: 8703f289c3fSJeff Roberson mtx_unlock_spin(&cpuset_lock); 8713f289c3fSJeff Roberson domainset_freelist_free(&domains); 8723f289c3fSJeff Roberson if (error == 0) 8733f289c3fSJeff Roberson domainset_notify(); 8743f289c3fSJeff Roberson 8753f289c3fSJeff Roberson return (error); 8763f289c3fSJeff Roberson } 8773f289c3fSJeff Roberson 8783f289c3fSJeff Roberson /* 879d7f687fcSJeff Roberson * Resolve the 'which' parameter of several cpuset apis. 880d7f687fcSJeff Roberson * 881d7f687fcSJeff Roberson * For WHICH_PID and WHICH_TID return a locked proc and valid proc/tid. Also 882d7f687fcSJeff Roberson * checks for permission via p_cansched(). 883d7f687fcSJeff Roberson * 884d7f687fcSJeff Roberson * For WHICH_SET returns a valid set with a new reference. 885d7f687fcSJeff Roberson * 886d7f687fcSJeff Roberson * -1 may be supplied for any argument to mean the current proc/thread or 887d7f687fcSJeff Roberson * the base set of the current thread. May fail with ESRCH/EPERM. 888d7f687fcSJeff Roberson */ 8895bbb2169SAdrian Chadd int 890d7f687fcSJeff Roberson cpuset_which(cpuwhich_t which, id_t id, struct proc **pp, struct thread **tdp, 891d7f687fcSJeff Roberson struct cpuset **setp) 892d7f687fcSJeff Roberson { 893d7f687fcSJeff Roberson struct cpuset *set; 894d7f687fcSJeff Roberson struct thread *td; 895d7f687fcSJeff Roberson struct proc *p; 896d7f687fcSJeff Roberson int error; 897d7f687fcSJeff Roberson 898d7f687fcSJeff Roberson *pp = p = NULL; 899d7f687fcSJeff Roberson *tdp = td = NULL; 900d7f687fcSJeff Roberson *setp = set = NULL; 901d7f687fcSJeff Roberson switch (which) { 902d7f687fcSJeff Roberson case CPU_WHICH_PID: 903d7f687fcSJeff Roberson if (id == -1) { 904d7f687fcSJeff Roberson PROC_LOCK(curproc); 905d7f687fcSJeff Roberson p = curproc; 906d7f687fcSJeff Roberson break; 907d7f687fcSJeff Roberson } 908d7f687fcSJeff Roberson if ((p = pfind(id)) == NULL) 909d7f687fcSJeff Roberson return (ESRCH); 910d7f687fcSJeff Roberson break; 911d7f687fcSJeff Roberson case CPU_WHICH_TID: 912d7f687fcSJeff Roberson if (id == -1) { 913d7f687fcSJeff Roberson PROC_LOCK(curproc); 914d7f687fcSJeff Roberson p = curproc; 915d7f687fcSJeff Roberson td = curthread; 916d7f687fcSJeff Roberson break; 917d7f687fcSJeff Roberson } 91842fe684cSDavid Xu td = tdfind(id, -1); 919d7f687fcSJeff Roberson if (td == NULL) 920d7f687fcSJeff Roberson return (ESRCH); 92142fe684cSDavid Xu p = td->td_proc; 922d7f687fcSJeff Roberson break; 923d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 924d7f687fcSJeff Roberson if (id == -1) { 925d7f687fcSJeff Roberson thread_lock(curthread); 926a03ee000SJeff Roberson set = cpuset_refbase(curthread->td_cpuset); 927d7f687fcSJeff Roberson thread_unlock(curthread); 928d7f687fcSJeff Roberson } else 929413628a7SBjoern A. Zeeb set = cpuset_lookup(id, curthread); 930d7f687fcSJeff Roberson if (set) { 931d7f687fcSJeff Roberson *setp = set; 932d7f687fcSJeff Roberson return (0); 933d7f687fcSJeff Roberson } 934d7f687fcSJeff Roberson return (ESRCH); 935413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 936413628a7SBjoern A. Zeeb { 937413628a7SBjoern A. Zeeb /* Find `set' for prison with given id. */ 938413628a7SBjoern A. Zeeb struct prison *pr; 939413628a7SBjoern A. Zeeb 940413628a7SBjoern A. Zeeb sx_slock(&allprison_lock); 9410304c731SJamie Gritton pr = prison_find_child(curthread->td_ucred->cr_prison, id); 942413628a7SBjoern A. Zeeb sx_sunlock(&allprison_lock); 943413628a7SBjoern A. Zeeb if (pr == NULL) 944413628a7SBjoern A. Zeeb return (ESRCH); 945413628a7SBjoern A. Zeeb cpuset_ref(pr->pr_cpuset); 9460304c731SJamie Gritton *setp = pr->pr_cpuset; 947413628a7SBjoern A. Zeeb mtx_unlock(&pr->pr_mtx); 948413628a7SBjoern A. Zeeb return (0); 949413628a7SBjoern A. Zeeb } 9509b33b154SJeff Roberson case CPU_WHICH_IRQ: 951c0ae6688SJohn Baldwin case CPU_WHICH_DOMAIN: 9529b33b154SJeff Roberson return (0); 953d7f687fcSJeff Roberson default: 954d7f687fcSJeff Roberson return (EINVAL); 955d7f687fcSJeff Roberson } 956d7f687fcSJeff Roberson error = p_cansched(curthread, p); 957d7f687fcSJeff Roberson if (error) { 958d7f687fcSJeff Roberson PROC_UNLOCK(p); 959d7f687fcSJeff Roberson return (error); 960d7f687fcSJeff Roberson } 961d7f687fcSJeff Roberson if (td == NULL) 962d7f687fcSJeff Roberson td = FIRST_THREAD_IN_PROC(p); 963d7f687fcSJeff Roberson *pp = p; 964d7f687fcSJeff Roberson *tdp = td; 965d7f687fcSJeff Roberson return (0); 966d7f687fcSJeff Roberson } 967d7f687fcSJeff Roberson 968d7f687fcSJeff Roberson static int 9693f289c3fSJeff Roberson cpuset_testshadow(struct cpuset *set, const cpuset_t *mask, 9703f289c3fSJeff Roberson const struct domainset *domain) 971d7f687fcSJeff Roberson { 972d7f687fcSJeff Roberson struct cpuset *parent; 9733f289c3fSJeff Roberson struct domainset *dset; 974d7f687fcSJeff Roberson 9753f289c3fSJeff Roberson parent = cpuset_getbase(set); 9763f289c3fSJeff Roberson /* 9773f289c3fSJeff Roberson * If we are restricting a cpu mask it must be a subset of the 9783f289c3fSJeff Roberson * parent or invalid CPUs have been specified. 9793f289c3fSJeff Roberson */ 9803f289c3fSJeff Roberson if (mask != NULL && !CPU_SUBSET(&parent->cs_mask, mask)) 9813f289c3fSJeff Roberson return (EINVAL); 9823f289c3fSJeff Roberson 9833f289c3fSJeff Roberson /* 9843f289c3fSJeff Roberson * If we are restricting a domain mask it must be a subset of the 9853f289c3fSJeff Roberson * parent or invalid domains have been specified. 9863f289c3fSJeff Roberson */ 9873f289c3fSJeff Roberson dset = parent->cs_domain; 9883f289c3fSJeff Roberson if (domain != NULL && !domainset_valid(dset, domain)) 9893f289c3fSJeff Roberson return (EINVAL); 9903f289c3fSJeff Roberson 9913f289c3fSJeff Roberson return (0); 992d7f687fcSJeff Roberson } 993d7f687fcSJeff Roberson 994d7f687fcSJeff Roberson /* 9953f289c3fSJeff Roberson * Create an anonymous set with the provided mask in the space provided by 9963f289c3fSJeff Roberson * 'nset'. If the passed in set is anonymous we use its parent otherwise 9973f289c3fSJeff Roberson * the new set is a child of 'set'. 9983f289c3fSJeff Roberson */ 9993f289c3fSJeff Roberson static int 10003f289c3fSJeff Roberson cpuset_shadow(struct cpuset *set, struct cpuset **nsetp, 10013f289c3fSJeff Roberson const cpuset_t *mask, const struct domainset *domain, 10023f289c3fSJeff Roberson struct setlist *cpusets, struct domainlist *domains) 10033f289c3fSJeff Roberson { 10043f289c3fSJeff Roberson struct cpuset *parent; 10053f289c3fSJeff Roberson struct cpuset *nset; 10063f289c3fSJeff Roberson struct domainset *dset; 10073f289c3fSJeff Roberson struct domainset *d; 10083f289c3fSJeff Roberson int error; 10093f289c3fSJeff Roberson 10103f289c3fSJeff Roberson error = cpuset_testshadow(set, mask, domain); 10113f289c3fSJeff Roberson if (error) 10123f289c3fSJeff Roberson return (error); 10133f289c3fSJeff Roberson 10143f289c3fSJeff Roberson parent = cpuset_getbase(set); 10153f289c3fSJeff Roberson dset = parent->cs_domain; 10163f289c3fSJeff Roberson if (mask == NULL) 10173f289c3fSJeff Roberson mask = &set->cs_mask; 10183f289c3fSJeff Roberson if (domain != NULL) 10193f289c3fSJeff Roberson d = domainset_shadow(dset, domain, domains); 10203f289c3fSJeff Roberson else 10213f289c3fSJeff Roberson d = set->cs_domain; 10223f289c3fSJeff Roberson nset = LIST_FIRST(cpusets); 102330b7c6f9SKyle Evans error = cpuset_init(nset, parent, mask, d, CPUSET_INVALID); 10243f289c3fSJeff Roberson if (error == 0) { 10253f289c3fSJeff Roberson LIST_REMOVE(nset, cs_link); 10263f289c3fSJeff Roberson *nsetp = nset; 10273f289c3fSJeff Roberson } 10283f289c3fSJeff Roberson return (error); 10293f289c3fSJeff Roberson } 10303f289c3fSJeff Roberson 10313f289c3fSJeff Roberson static struct cpuset * 10323f289c3fSJeff Roberson cpuset_update_thread(struct thread *td, struct cpuset *nset) 10333f289c3fSJeff Roberson { 10343f289c3fSJeff Roberson struct cpuset *tdset; 10353f289c3fSJeff Roberson 10363f289c3fSJeff Roberson tdset = td->td_cpuset; 10373f289c3fSJeff Roberson td->td_cpuset = nset; 10383f289c3fSJeff Roberson td->td_domain.dr_policy = nset->cs_domain; 10393f289c3fSJeff Roberson sched_affinity(td); 10403f289c3fSJeff Roberson 10413f289c3fSJeff Roberson return (tdset); 10423f289c3fSJeff Roberson } 10433f289c3fSJeff Roberson 10443f289c3fSJeff Roberson static int 10453f289c3fSJeff Roberson cpuset_setproc_test_maskthread(struct cpuset *tdset, cpuset_t *mask, 10463f289c3fSJeff Roberson struct domainset *domain) 10473f289c3fSJeff Roberson { 10483f289c3fSJeff Roberson struct cpuset *parent; 10493f289c3fSJeff Roberson 10503f289c3fSJeff Roberson parent = cpuset_getbase(tdset); 10513f289c3fSJeff Roberson if (mask == NULL) 10523f289c3fSJeff Roberson mask = &tdset->cs_mask; 10533f289c3fSJeff Roberson if (domain == NULL) 10543f289c3fSJeff Roberson domain = tdset->cs_domain; 10553f289c3fSJeff Roberson return cpuset_testshadow(parent, mask, domain); 10563f289c3fSJeff Roberson } 10573f289c3fSJeff Roberson 10583f289c3fSJeff Roberson static int 10593f289c3fSJeff Roberson cpuset_setproc_maskthread(struct cpuset *tdset, cpuset_t *mask, 10603f289c3fSJeff Roberson struct domainset *domain, struct cpuset **nsetp, 10613f289c3fSJeff Roberson struct setlist *freelist, struct domainlist *domainlist) 10623f289c3fSJeff Roberson { 10633f289c3fSJeff Roberson struct cpuset *parent; 10643f289c3fSJeff Roberson 10653f289c3fSJeff Roberson parent = cpuset_getbase(tdset); 10663f289c3fSJeff Roberson if (mask == NULL) 10673f289c3fSJeff Roberson mask = &tdset->cs_mask; 10683f289c3fSJeff Roberson if (domain == NULL) 10693f289c3fSJeff Roberson domain = tdset->cs_domain; 10703f289c3fSJeff Roberson return cpuset_shadow(parent, nsetp, mask, domain, freelist, 10713f289c3fSJeff Roberson domainlist); 10723f289c3fSJeff Roberson } 10733f289c3fSJeff Roberson 10743f289c3fSJeff Roberson static int 10753f289c3fSJeff Roberson cpuset_setproc_setthread_mask(struct cpuset *tdset, struct cpuset *set, 10763f289c3fSJeff Roberson cpuset_t *mask, struct domainset *domain) 10773f289c3fSJeff Roberson { 10783f289c3fSJeff Roberson struct cpuset *parent; 10793f289c3fSJeff Roberson 10803f289c3fSJeff Roberson parent = cpuset_getbase(tdset); 10813f289c3fSJeff Roberson 10823f289c3fSJeff Roberson /* 10833f289c3fSJeff Roberson * If the thread restricted its mask then apply that same 10843f289c3fSJeff Roberson * restriction to the new set, otherwise take it wholesale. 10853f289c3fSJeff Roberson */ 10863f289c3fSJeff Roberson if (CPU_CMP(&tdset->cs_mask, &parent->cs_mask) != 0) { 1087e2650af1SStefan Eßer CPU_AND(mask, &tdset->cs_mask, &set->cs_mask); 10883f289c3fSJeff Roberson } else 10893f289c3fSJeff Roberson CPU_COPY(&set->cs_mask, mask); 10903f289c3fSJeff Roberson 10913f289c3fSJeff Roberson /* 10923f289c3fSJeff Roberson * If the thread restricted the domain then we apply the 10933f289c3fSJeff Roberson * restriction to the new set but retain the policy. 10943f289c3fSJeff Roberson */ 10953f289c3fSJeff Roberson if (tdset->cs_domain != parent->cs_domain) { 10963f289c3fSJeff Roberson domainset_copy(tdset->cs_domain, domain); 10973f289c3fSJeff Roberson DOMAINSET_AND(&domain->ds_mask, &set->cs_domain->ds_mask); 10983f289c3fSJeff Roberson } else 10993f289c3fSJeff Roberson domainset_copy(set->cs_domain, domain); 11003f289c3fSJeff Roberson 11013f289c3fSJeff Roberson if (CPU_EMPTY(mask) || DOMAINSET_EMPTY(&domain->ds_mask)) 11023f289c3fSJeff Roberson return (EDEADLK); 11033f289c3fSJeff Roberson 11043f289c3fSJeff Roberson return (0); 11053f289c3fSJeff Roberson } 11063f289c3fSJeff Roberson 11073f289c3fSJeff Roberson static int 11083f289c3fSJeff Roberson cpuset_setproc_test_setthread(struct cpuset *tdset, struct cpuset *set) 11093f289c3fSJeff Roberson { 11103f289c3fSJeff Roberson struct domainset domain; 11113f289c3fSJeff Roberson cpuset_t mask; 11123f289c3fSJeff Roberson 11133f289c3fSJeff Roberson if (tdset->cs_id != CPUSET_INVALID) 11143f289c3fSJeff Roberson return (0); 11153f289c3fSJeff Roberson return cpuset_setproc_setthread_mask(tdset, set, &mask, &domain); 11163f289c3fSJeff Roberson } 11173f289c3fSJeff Roberson 11183f289c3fSJeff Roberson static int 11193f289c3fSJeff Roberson cpuset_setproc_setthread(struct cpuset *tdset, struct cpuset *set, 11203f289c3fSJeff Roberson struct cpuset **nsetp, struct setlist *freelist, 11213f289c3fSJeff Roberson struct domainlist *domainlist) 11223f289c3fSJeff Roberson { 11233f289c3fSJeff Roberson struct domainset domain; 11243f289c3fSJeff Roberson cpuset_t mask; 11253f289c3fSJeff Roberson int error; 11263f289c3fSJeff Roberson 11273f289c3fSJeff Roberson /* 11283f289c3fSJeff Roberson * If we're replacing on a thread that has not constrained the 11293f289c3fSJeff Roberson * original set we can simply accept the new set. 11303f289c3fSJeff Roberson */ 11313f289c3fSJeff Roberson if (tdset->cs_id != CPUSET_INVALID) { 11323f289c3fSJeff Roberson *nsetp = cpuset_ref(set); 11333f289c3fSJeff Roberson return (0); 11343f289c3fSJeff Roberson } 11353f289c3fSJeff Roberson error = cpuset_setproc_setthread_mask(tdset, set, &mask, &domain); 11363f289c3fSJeff Roberson if (error) 11373f289c3fSJeff Roberson return (error); 11383f289c3fSJeff Roberson 1139dac521ebSKyle Evans return cpuset_shadow(set, nsetp, &mask, &domain, freelist, 11403f289c3fSJeff Roberson domainlist); 11413f289c3fSJeff Roberson } 11423f289c3fSJeff Roberson 1143d431dea5SKyle Evans static int 1144d431dea5SKyle Evans cpuset_setproc_newbase(struct thread *td, struct cpuset *set, 1145d431dea5SKyle Evans struct cpuset *nroot, struct cpuset **nsetp, 1146d431dea5SKyle Evans struct setlist *cpusets, struct domainlist *domainlist) 1147d431dea5SKyle Evans { 1148d431dea5SKyle Evans struct domainset ndomain; 1149d431dea5SKyle Evans cpuset_t nmask; 1150d431dea5SKyle Evans struct cpuset *pbase; 1151d431dea5SKyle Evans int error; 1152d431dea5SKyle Evans 1153d431dea5SKyle Evans pbase = cpuset_getbase(td->td_cpuset); 1154d431dea5SKyle Evans 1155d431dea5SKyle Evans /* Copy process mask, then further apply the new root mask. */ 1156e2650af1SStefan Eßer CPU_AND(&nmask, &pbase->cs_mask, &nroot->cs_mask); 1157d431dea5SKyle Evans 1158d431dea5SKyle Evans domainset_copy(pbase->cs_domain, &ndomain); 1159d431dea5SKyle Evans DOMAINSET_AND(&ndomain.ds_mask, &set->cs_domain->ds_mask); 1160d431dea5SKyle Evans 1161d431dea5SKyle Evans /* Policy is too restrictive, will not work. */ 1162d431dea5SKyle Evans if (CPU_EMPTY(&nmask) || DOMAINSET_EMPTY(&ndomain.ds_mask)) 1163d431dea5SKyle Evans return (EDEADLK); 1164d431dea5SKyle Evans 11653f289c3fSJeff Roberson /* 1166d431dea5SKyle Evans * Remove pbase from the freelist in advance, it'll be pushed to 1167d431dea5SKyle Evans * cpuset_ids on success. We assume here that cpuset_create() will not 1168d431dea5SKyle Evans * touch pbase on failure, and we just enqueue it back to the freelist 1169d431dea5SKyle Evans * to remain in a consistent state. 1170d431dea5SKyle Evans */ 1171d431dea5SKyle Evans pbase = LIST_FIRST(cpusets); 1172d431dea5SKyle Evans LIST_REMOVE(pbase, cs_link); 1173d431dea5SKyle Evans error = cpuset_create(&pbase, set, &nmask); 1174d431dea5SKyle Evans if (error != 0) { 1175d431dea5SKyle Evans LIST_INSERT_HEAD(cpusets, pbase, cs_link); 1176d431dea5SKyle Evans return (error); 1177d431dea5SKyle Evans } 1178d431dea5SKyle Evans 1179d431dea5SKyle Evans /* Duplicates some work from above... oh well. */ 1180d431dea5SKyle Evans pbase->cs_domain = domainset_shadow(set->cs_domain, &ndomain, 1181d431dea5SKyle Evans domainlist); 1182d431dea5SKyle Evans *nsetp = pbase; 1183d431dea5SKyle Evans return (0); 1184d431dea5SKyle Evans } 1185d431dea5SKyle Evans 1186d431dea5SKyle Evans /* 1187d431dea5SKyle Evans * Handle four cases for updating an entire process. 1188d7f687fcSJeff Roberson * 1189d431dea5SKyle Evans * 1) Set is non-null and the process is not rebasing onto a new root. This 1190d431dea5SKyle Evans * reparents all anonymous sets to the provided set and replaces all 1191d431dea5SKyle Evans * non-anonymous td_cpusets with the provided set. 1192d431dea5SKyle Evans * 2) Set is non-null and the process is rebasing onto a new root. This 1193d431dea5SKyle Evans * creates a new base set if the process previously had its own base set, 1194d431dea5SKyle Evans * then reparents all anonymous sets either to that set or the provided set 1195d431dea5SKyle Evans * if one was not created. Non-anonymous sets are similarly replaced. 1196d431dea5SKyle Evans * 3) Mask is non-null. This replaces or creates anonymous sets for every 11973f289c3fSJeff Roberson * thread with the existing base as a parent. 1198d431dea5SKyle Evans * 4) domain is non-null. This creates anonymous sets for every thread 11993f289c3fSJeff Roberson * and replaces the domain set. 1200d7f687fcSJeff Roberson * 1201d7f687fcSJeff Roberson * This is overly complicated because we can't allocate while holding a 1202d7f687fcSJeff Roberson * spinlock and spinlocks must be held while changing and examining thread 1203d7f687fcSJeff Roberson * state. 1204d7f687fcSJeff Roberson */ 1205d7f687fcSJeff Roberson static int 12063f289c3fSJeff Roberson cpuset_setproc(pid_t pid, struct cpuset *set, cpuset_t *mask, 1207d431dea5SKyle Evans struct domainset *domain, bool rebase) 1208d7f687fcSJeff Roberson { 1209d7f687fcSJeff Roberson struct setlist freelist; 1210d7f687fcSJeff Roberson struct setlist droplist; 12113f289c3fSJeff Roberson struct domainlist domainlist; 1212d431dea5SKyle Evans struct cpuset *base, *nset, *nroot, *tdroot; 1213d7f687fcSJeff Roberson struct thread *td; 1214d7f687fcSJeff Roberson struct proc *p; 1215d431dea5SKyle Evans int needed; 1216d7f687fcSJeff Roberson int nfree; 1217d7f687fcSJeff Roberson int error; 1218f299c47bSAllan Jude 1219d7f687fcSJeff Roberson /* 1220d7f687fcSJeff Roberson * The algorithm requires two passes due to locking considerations. 1221d7f687fcSJeff Roberson * 1222d7f687fcSJeff Roberson * 1) Lookup the process and acquire the locks in the required order. 1223d7f687fcSJeff Roberson * 2) If enough cpusets have not been allocated release the locks and 1224d7f687fcSJeff Roberson * allocate them. Loop. 1225d7f687fcSJeff Roberson */ 12263f289c3fSJeff Roberson cpuset_freelist_init(&freelist, 1); 12273f289c3fSJeff Roberson domainset_freelist_init(&domainlist, 1); 12283f289c3fSJeff Roberson nfree = 1; 1229d7f687fcSJeff Roberson LIST_INIT(&droplist); 1230d7f687fcSJeff Roberson nfree = 0; 1231d431dea5SKyle Evans base = set; 1232d431dea5SKyle Evans nroot = NULL; 1233d431dea5SKyle Evans if (set != NULL) 1234d431dea5SKyle Evans nroot = cpuset_getroot(set); 1235d7f687fcSJeff Roberson for (;;) { 1236d7f687fcSJeff Roberson error = cpuset_which(CPU_WHICH_PID, pid, &p, &td, &nset); 1237d7f687fcSJeff Roberson if (error) 1238d7f687fcSJeff Roberson goto out; 1239d431dea5SKyle Evans tdroot = cpuset_getroot(td->td_cpuset); 1240d431dea5SKyle Evans needed = p->p_numthreads; 1241d431dea5SKyle Evans if (set != NULL && rebase && tdroot != nroot) 1242d431dea5SKyle Evans needed++; 1243d431dea5SKyle Evans if (nfree >= needed) 1244d7f687fcSJeff Roberson break; 1245d7f687fcSJeff Roberson PROC_UNLOCK(p); 1246d431dea5SKyle Evans if (nfree < needed) { 1247d431dea5SKyle Evans cpuset_freelist_add(&freelist, needed - nfree); 1248d431dea5SKyle Evans domainset_freelist_add(&domainlist, needed - nfree); 1249d431dea5SKyle Evans nfree = needed; 1250d7f687fcSJeff Roberson } 1251d7f687fcSJeff Roberson } 1252d7f687fcSJeff Roberson PROC_LOCK_ASSERT(p, MA_OWNED); 1253d431dea5SKyle Evans 1254d431dea5SKyle Evans /* 1255d431dea5SKyle Evans * If we're changing roots and the root set is what has been specified 1256d431dea5SKyle Evans * as the parent, then we'll check if the process was previously using 1257d431dea5SKyle Evans * the root set and, if it wasn't, create a new base with the process's 1258d431dea5SKyle Evans * mask applied to it. 125960c4ec80SKyle Evans * 126060c4ec80SKyle Evans * If the new root is incompatible with the existing mask, then we allow 126160c4ec80SKyle Evans * the process to take on the new root if and only if they have 126260c4ec80SKyle Evans * privilege to widen their mask anyways. Unprivileged processes get 126360c4ec80SKyle Evans * rejected with EDEADLK. 1264d431dea5SKyle Evans */ 1265d431dea5SKyle Evans if (set != NULL && rebase && nroot != tdroot) { 1266d431dea5SKyle Evans cpusetid_t base_id, root_id; 1267d431dea5SKyle Evans 1268d431dea5SKyle Evans root_id = td->td_ucred->cr_prison->pr_cpuset->cs_id; 1269d431dea5SKyle Evans base_id = cpuset_getbase(td->td_cpuset)->cs_id; 1270d431dea5SKyle Evans 1271d431dea5SKyle Evans if (base_id != root_id) { 1272d431dea5SKyle Evans error = cpuset_setproc_newbase(td, set, nroot, &base, 1273d431dea5SKyle Evans &freelist, &domainlist); 127460c4ec80SKyle Evans if (error == EDEADLK && 127560c4ec80SKyle Evans priv_check(td, PRIV_SCHED_CPUSET) == 0) 127660c4ec80SKyle Evans error = 0; 1277d431dea5SKyle Evans if (error != 0) 1278d431dea5SKyle Evans goto unlock_out; 1279d431dea5SKyle Evans } 1280d431dea5SKyle Evans } 1281d431dea5SKyle Evans 1282d7f687fcSJeff Roberson /* 1283d7f687fcSJeff Roberson * Now that the appropriate locks are held and we have enough cpusets, 128473c40187SJeff Roberson * make sure the operation will succeed before applying changes. The 128573c40187SJeff Roberson * proc lock prevents td_cpuset from changing between calls. 1286d7f687fcSJeff Roberson */ 1287d7f687fcSJeff Roberson error = 0; 1288d7f687fcSJeff Roberson FOREACH_THREAD_IN_PROC(p, td) { 128973c40187SJeff Roberson thread_lock(td); 12903f289c3fSJeff Roberson if (set != NULL) 12913f289c3fSJeff Roberson error = cpuset_setproc_test_setthread(td->td_cpuset, 1292d431dea5SKyle Evans base); 12933f289c3fSJeff Roberson else 12943f289c3fSJeff Roberson error = cpuset_setproc_test_maskthread(td->td_cpuset, 12953f289c3fSJeff Roberson mask, domain); 129673c40187SJeff Roberson thread_unlock(td); 129773c40187SJeff Roberson if (error) 129873c40187SJeff Roberson goto unlock_out; 129973c40187SJeff Roberson } 130073c40187SJeff Roberson /* 130173c40187SJeff Roberson * Replace each thread's cpuset while using deferred release. We 1302374ae2a3SJeff Roberson * must do this because the thread lock must be held while operating 1303374ae2a3SJeff Roberson * on the thread and this limits the type of operations allowed. 130473c40187SJeff Roberson */ 130573c40187SJeff Roberson FOREACH_THREAD_IN_PROC(p, td) { 1306d7f687fcSJeff Roberson thread_lock(td); 13073f289c3fSJeff Roberson if (set != NULL) 1308d431dea5SKyle Evans error = cpuset_setproc_setthread(td->td_cpuset, base, 13093f289c3fSJeff Roberson &nset, &freelist, &domainlist); 1310d7f687fcSJeff Roberson else 13113f289c3fSJeff Roberson error = cpuset_setproc_maskthread(td->td_cpuset, mask, 13123f289c3fSJeff Roberson domain, &nset, &freelist, &domainlist); 1313d7f687fcSJeff Roberson if (error) { 1314d7f687fcSJeff Roberson thread_unlock(td); 1315d7f687fcSJeff Roberson break; 1316d7f687fcSJeff Roberson } 13173f289c3fSJeff Roberson cpuset_rel_defer(&droplist, cpuset_update_thread(td, nset)); 1318d7f687fcSJeff Roberson thread_unlock(td); 1319d7f687fcSJeff Roberson } 132073c40187SJeff Roberson unlock_out: 1321d7f687fcSJeff Roberson PROC_UNLOCK(p); 1322d7f687fcSJeff Roberson out: 1323d431dea5SKyle Evans if (base != NULL && base != set) 1324d431dea5SKyle Evans cpuset_rel(base); 1325d7f687fcSJeff Roberson while ((nset = LIST_FIRST(&droplist)) != NULL) 1326d7f687fcSJeff Roberson cpuset_rel_complete(nset); 13273f289c3fSJeff Roberson cpuset_freelist_free(&freelist); 13283f289c3fSJeff Roberson domainset_freelist_free(&domainlist); 1329d7f687fcSJeff Roberson return (error); 1330d7f687fcSJeff Roberson } 1331d7f687fcSJeff Roberson 1332e5818a53SJeff Roberson static int 1333e5818a53SJeff Roberson bitset_strprint(char *buf, size_t bufsiz, const struct bitset *set, int setlen) 1334e5818a53SJeff Roberson { 1335e5818a53SJeff Roberson size_t bytes; 1336e5818a53SJeff Roberson int i, once; 1337e5818a53SJeff Roberson char *p; 1338e5818a53SJeff Roberson 1339e5818a53SJeff Roberson once = 0; 1340e5818a53SJeff Roberson p = buf; 1341e5818a53SJeff Roberson for (i = 0; i < __bitset_words(setlen); i++) { 1342e5818a53SJeff Roberson if (once != 0) { 1343e5818a53SJeff Roberson if (bufsiz < 1) 1344e5818a53SJeff Roberson return (0); 1345e5818a53SJeff Roberson *p = ','; 1346e5818a53SJeff Roberson p++; 1347e5818a53SJeff Roberson bufsiz--; 1348e5818a53SJeff Roberson } else 1349e5818a53SJeff Roberson once = 1; 1350e5818a53SJeff Roberson if (bufsiz < sizeof(__STRING(ULONG_MAX))) 1351e5818a53SJeff Roberson return (0); 1352e5818a53SJeff Roberson bytes = snprintf(p, bufsiz, "%lx", set->__bits[i]); 1353e5818a53SJeff Roberson p += bytes; 1354e5818a53SJeff Roberson bufsiz -= bytes; 1355e5818a53SJeff Roberson } 1356e5818a53SJeff Roberson return (p - buf); 1357e5818a53SJeff Roberson } 1358e5818a53SJeff Roberson 1359e5818a53SJeff Roberson static int 1360e5818a53SJeff Roberson bitset_strscan(struct bitset *set, int setlen, const char *buf) 1361e5818a53SJeff Roberson { 1362e5818a53SJeff Roberson int i, ret; 1363e5818a53SJeff Roberson const char *p; 1364e5818a53SJeff Roberson 1365e5818a53SJeff Roberson BIT_ZERO(setlen, set); 1366e5818a53SJeff Roberson p = buf; 1367e5818a53SJeff Roberson for (i = 0; i < __bitset_words(setlen); i++) { 1368e5818a53SJeff Roberson if (*p == ',') { 1369e5818a53SJeff Roberson p++; 1370e5818a53SJeff Roberson continue; 1371e5818a53SJeff Roberson } 1372e5818a53SJeff Roberson ret = sscanf(p, "%lx", &set->__bits[i]); 1373e5818a53SJeff Roberson if (ret == 0 || ret == -1) 1374e5818a53SJeff Roberson break; 1375e5818a53SJeff Roberson while (isxdigit(*p)) 1376e5818a53SJeff Roberson p++; 1377e5818a53SJeff Roberson } 1378e5818a53SJeff Roberson return (p - buf); 1379e5818a53SJeff Roberson } 1380e5818a53SJeff Roberson 1381d7f687fcSJeff Roberson /* 138271a19bdcSAttilio Rao * Return a string representing a valid layout for a cpuset_t object. 138371a19bdcSAttilio Rao * It expects an incoming buffer at least sized as CPUSETBUFSIZ. 138471a19bdcSAttilio Rao */ 138571a19bdcSAttilio Rao char * 138671a19bdcSAttilio Rao cpusetobj_strprint(char *buf, const cpuset_t *set) 138771a19bdcSAttilio Rao { 138871a19bdcSAttilio Rao 1389e5818a53SJeff Roberson bitset_strprint(buf, CPUSETBUFSIZ, (const struct bitset *)set, 1390e5818a53SJeff Roberson CPU_SETSIZE); 139171a19bdcSAttilio Rao return (buf); 139271a19bdcSAttilio Rao } 139371a19bdcSAttilio Rao 139471a19bdcSAttilio Rao /* 1395e3709597SAttilio Rao * Build a valid cpuset_t object from a string representation. 1396e3709597SAttilio Rao * It expects an incoming buffer at least sized as CPUSETBUFSIZ. 1397e3709597SAttilio Rao */ 1398e3709597SAttilio Rao int 1399e3709597SAttilio Rao cpusetobj_strscan(cpuset_t *set, const char *buf) 1400e3709597SAttilio Rao { 1401e5818a53SJeff Roberson char p; 1402e3709597SAttilio Rao 1403e3709597SAttilio Rao if (strlen(buf) > CPUSETBUFSIZ - 1) 1404e3709597SAttilio Rao return (-1); 1405e3709597SAttilio Rao 1406e5818a53SJeff Roberson p = buf[bitset_strscan((struct bitset *)set, CPU_SETSIZE, buf)]; 1407e5818a53SJeff Roberson if (p != '\0') 1408e3709597SAttilio Rao return (-1); 1409e3709597SAttilio Rao 1410e3709597SAttilio Rao return (0); 1411e3709597SAttilio Rao } 1412e3709597SAttilio Rao 1413e3709597SAttilio Rao /* 1414e5818a53SJeff Roberson * Handle a domainset specifier in the sysctl tree. A poiner to a pointer to 1415e5818a53SJeff Roberson * a domainset is in arg1. If the user specifies a valid domainset the 1416e5818a53SJeff Roberson * pointer is updated. 1417e5818a53SJeff Roberson * 1418e5818a53SJeff Roberson * Format is: 1419e5818a53SJeff Roberson * hex mask word 0,hex mask word 1,...:decimal policy:decimal preferred 1420e5818a53SJeff Roberson */ 1421e5818a53SJeff Roberson int 1422e5818a53SJeff Roberson sysctl_handle_domainset(SYSCTL_HANDLER_ARGS) 1423e5818a53SJeff Roberson { 1424e5818a53SJeff Roberson char buf[DOMAINSETBUFSIZ]; 1425e5818a53SJeff Roberson struct domainset *dset; 1426e5818a53SJeff Roberson struct domainset key; 1427e5818a53SJeff Roberson int policy, prefer, error; 1428e5818a53SJeff Roberson char *p; 1429e5818a53SJeff Roberson 1430e5818a53SJeff Roberson dset = *(struct domainset **)arg1; 1431e5818a53SJeff Roberson error = 0; 1432e5818a53SJeff Roberson 1433e5818a53SJeff Roberson if (dset != NULL) { 1434e5818a53SJeff Roberson p = buf + bitset_strprint(buf, DOMAINSETBUFSIZ, 1435e5818a53SJeff Roberson (const struct bitset *)&dset->ds_mask, DOMAINSET_SETSIZE); 1436e5818a53SJeff Roberson sprintf(p, ":%d:%d", dset->ds_policy, dset->ds_prefer); 1437e5818a53SJeff Roberson } else 1438e5818a53SJeff Roberson sprintf(buf, "<NULL>"); 1439e5818a53SJeff Roberson error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 1440e5818a53SJeff Roberson if (error != 0 || req->newptr == NULL) 1441e5818a53SJeff Roberson return (error); 1442e5818a53SJeff Roberson 1443e5818a53SJeff Roberson /* 1444e5818a53SJeff Roberson * Read in and validate the string. 1445e5818a53SJeff Roberson */ 1446e5818a53SJeff Roberson memset(&key, 0, sizeof(key)); 1447e5818a53SJeff Roberson p = &buf[bitset_strscan((struct bitset *)&key.ds_mask, 1448e5818a53SJeff Roberson DOMAINSET_SETSIZE, buf)]; 1449e5818a53SJeff Roberson if (p == buf) 1450e5818a53SJeff Roberson return (EINVAL); 1451e5818a53SJeff Roberson if (sscanf(p, ":%d:%d", &policy, &prefer) != 2) 1452e5818a53SJeff Roberson return (EINVAL); 1453e5818a53SJeff Roberson key.ds_policy = policy; 1454e5818a53SJeff Roberson key.ds_prefer = prefer; 1455e5818a53SJeff Roberson 1456e5818a53SJeff Roberson /* Domainset_create() validates the policy.*/ 1457e5818a53SJeff Roberson dset = domainset_create(&key); 1458e5818a53SJeff Roberson if (dset == NULL) 1459e5818a53SJeff Roberson return (EINVAL); 1460e5818a53SJeff Roberson *(struct domainset **)arg1 = dset; 1461e5818a53SJeff Roberson 1462e5818a53SJeff Roberson return (error); 1463e5818a53SJeff Roberson } 1464e5818a53SJeff Roberson 1465e5818a53SJeff Roberson /* 14663f289c3fSJeff Roberson * Apply an anonymous mask or a domain to a single thread. 1467d7f687fcSJeff Roberson */ 14683f289c3fSJeff Roberson static int 14693f289c3fSJeff Roberson _cpuset_setthread(lwpid_t id, cpuset_t *mask, struct domainset *domain) 1470d7f687fcSJeff Roberson { 14713f289c3fSJeff Roberson struct setlist cpusets; 14723f289c3fSJeff Roberson struct domainlist domainlist; 1473d7f687fcSJeff Roberson struct cpuset *nset; 1474d7f687fcSJeff Roberson struct cpuset *set; 1475d7f687fcSJeff Roberson struct thread *td; 1476d7f687fcSJeff Roberson struct proc *p; 1477d7f687fcSJeff Roberson int error; 1478d7f687fcSJeff Roberson 14793f289c3fSJeff Roberson cpuset_freelist_init(&cpusets, 1); 14803f289c3fSJeff Roberson domainset_freelist_init(&domainlist, domain != NULL); 14818bd75bddSJeff Roberson error = cpuset_which(CPU_WHICH_TID, id, &p, &td, &set); 1482d7f687fcSJeff Roberson if (error) 1483d7f687fcSJeff Roberson goto out; 1484a03ee000SJeff Roberson set = NULL; 1485d7f687fcSJeff Roberson thread_lock(td); 14863f289c3fSJeff Roberson error = cpuset_shadow(td->td_cpuset, &nset, mask, domain, 14873f289c3fSJeff Roberson &cpusets, &domainlist); 14883f289c3fSJeff Roberson if (error == 0) 14893f289c3fSJeff Roberson set = cpuset_update_thread(td, nset); 1490d7f687fcSJeff Roberson thread_unlock(td); 1491d7f687fcSJeff Roberson PROC_UNLOCK(p); 1492a03ee000SJeff Roberson if (set) 1493a03ee000SJeff Roberson cpuset_rel(set); 1494d7f687fcSJeff Roberson out: 14953f289c3fSJeff Roberson cpuset_freelist_free(&cpusets); 14963f289c3fSJeff Roberson domainset_freelist_free(&domainlist); 1497d7f687fcSJeff Roberson return (error); 1498d7f687fcSJeff Roberson } 1499d7f687fcSJeff Roberson 1500d7f687fcSJeff Roberson /* 15013f289c3fSJeff Roberson * Apply an anonymous mask to a single thread. 15023f289c3fSJeff Roberson */ 15033f289c3fSJeff Roberson int 15043f289c3fSJeff Roberson cpuset_setthread(lwpid_t id, cpuset_t *mask) 15053f289c3fSJeff Roberson { 15063f289c3fSJeff Roberson 15073f289c3fSJeff Roberson return _cpuset_setthread(id, mask, NULL); 15083f289c3fSJeff Roberson } 15093f289c3fSJeff Roberson 15103f289c3fSJeff Roberson /* 151181198539SAlexander V. Chernikov * Apply new cpumask to the ithread. 151281198539SAlexander V. Chernikov */ 151381198539SAlexander V. Chernikov int 15147f7528fcSAdrian Chadd cpuset_setithread(lwpid_t id, int cpu) 151581198539SAlexander V. Chernikov { 151681198539SAlexander V. Chernikov cpuset_t mask; 151781198539SAlexander V. Chernikov 151881198539SAlexander V. Chernikov CPU_ZERO(&mask); 151981198539SAlexander V. Chernikov if (cpu == NOCPU) 152081198539SAlexander V. Chernikov CPU_COPY(cpuset_root, &mask); 152181198539SAlexander V. Chernikov else 152281198539SAlexander V. Chernikov CPU_SET(cpu, &mask); 1523e5818a53SJeff Roberson return _cpuset_setthread(id, &mask, NULL); 1524e5818a53SJeff Roberson } 1525c1d9ecf2SAlexander V. Chernikov 152681198539SAlexander V. Chernikov /* 1527662e7fa8SMark Johnston * Initialize static domainsets after NUMA information is available. This is 1528920239efSMark Johnston * called before memory allocators are initialized. 1529662e7fa8SMark Johnston */ 1530662e7fa8SMark Johnston void 1531662e7fa8SMark Johnston domainset_init(void) 1532662e7fa8SMark Johnston { 1533662e7fa8SMark Johnston struct domainset *dset; 1534662e7fa8SMark Johnston int i; 1535662e7fa8SMark Johnston 153629bb6c19SMark Johnston dset = &domainset_firsttouch; 153729bb6c19SMark Johnston DOMAINSET_COPY(&all_domains, &dset->ds_mask); 153829bb6c19SMark Johnston dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH; 153929bb6c19SMark Johnston dset->ds_prefer = -1; 154029bb6c19SMark Johnston _domainset_create(dset, NULL); 154129bb6c19SMark Johnston 154229bb6c19SMark Johnston dset = &domainset_interleave; 154329bb6c19SMark Johnston DOMAINSET_COPY(&all_domains, &dset->ds_mask); 154429bb6c19SMark Johnston dset->ds_policy = DOMAINSET_POLICY_INTERLEAVE; 154529bb6c19SMark Johnston dset->ds_prefer = -1; 154629bb6c19SMark Johnston _domainset_create(dset, NULL); 154729bb6c19SMark Johnston 1548662e7fa8SMark Johnston dset = &domainset_roundrobin; 1549662e7fa8SMark Johnston DOMAINSET_COPY(&all_domains, &dset->ds_mask); 1550662e7fa8SMark Johnston dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; 1551662e7fa8SMark Johnston dset->ds_prefer = -1; 1552662e7fa8SMark Johnston _domainset_create(dset, NULL); 1553662e7fa8SMark Johnston 1554662e7fa8SMark Johnston for (i = 0; i < vm_ndomains; i++) { 15559978bd99SMark Johnston dset = &domainset_fixed[i]; 15569978bd99SMark Johnston DOMAINSET_ZERO(&dset->ds_mask); 15579978bd99SMark Johnston DOMAINSET_SET(i, &dset->ds_mask); 15589978bd99SMark Johnston dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; 15599978bd99SMark Johnston _domainset_create(dset, NULL); 15609978bd99SMark Johnston 1561662e7fa8SMark Johnston dset = &domainset_prefer[i]; 1562662e7fa8SMark Johnston DOMAINSET_COPY(&all_domains, &dset->ds_mask); 1563662e7fa8SMark Johnston dset->ds_policy = DOMAINSET_POLICY_PREFER; 1564662e7fa8SMark Johnston dset->ds_prefer = i; 1565662e7fa8SMark Johnston _domainset_create(dset, NULL); 1566662e7fa8SMark Johnston } 1567662e7fa8SMark Johnston } 1568662e7fa8SMark Johnston 1569662e7fa8SMark Johnston /* 157029bb6c19SMark Johnston * Define the domainsets for cpuset 0, 1 and cpuset 2. 157181198539SAlexander V. Chernikov */ 15723f289c3fSJeff Roberson void 15733f289c3fSJeff Roberson domainset_zero(void) 15743f289c3fSJeff Roberson { 1575920239efSMark Johnston struct domainset *dset, *tmp; 15763f289c3fSJeff Roberson 15773f289c3fSJeff Roberson mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE); 15783f289c3fSJeff Roberson 157929bb6c19SMark Johnston domainset0 = &domainset_firsttouch; 158029bb6c19SMark Johnston curthread->td_domain.dr_policy = domainset0; 1581e5818a53SJeff Roberson 158229bb6c19SMark Johnston domainset2 = &domainset_interleave; 158329bb6c19SMark Johnston kernel_object->domain.dr_policy = domainset2; 1584662e7fa8SMark Johnston 1585662e7fa8SMark Johnston /* Remove empty domains from the global policies. */ 1586920239efSMark Johnston LIST_FOREACH_SAFE(dset, &cpuset_domains, ds_link, tmp) 1587920239efSMark Johnston if (domainset_empty_vm(dset)) 1588920239efSMark Johnston LIST_REMOVE(dset, ds_link); 15893f289c3fSJeff Roberson } 159081198539SAlexander V. Chernikov 159181198539SAlexander V. Chernikov /* 1592e5818a53SJeff Roberson * Creates system-wide cpusets and the cpuset for thread0 including three 1593c0ae6688SJohn Baldwin * sets: 1594d7f687fcSJeff Roberson * 1595d7f687fcSJeff Roberson * 0 - The root set which should represent all valid processors in the 159630b7c6f9SKyle Evans * system. This set is immutable. 1597d7f687fcSJeff Roberson * 1 - The default set which all processes are a member of until changed. 1598d7f687fcSJeff Roberson * This allows an administrator to move all threads off of given cpus to 1599d7f687fcSJeff Roberson * dedicate them to high priority tasks or save power etc. 1600e5818a53SJeff Roberson * 2 - The kernel set which allows restriction and policy to be applied only 1601e5818a53SJeff Roberson * to kernel threads and the kernel_object. 1602d7f687fcSJeff Roberson */ 1603d7f687fcSJeff Roberson struct cpuset * 1604d7f687fcSJeff Roberson cpuset_thread0(void) 1605d7f687fcSJeff Roberson { 1606d7f687fcSJeff Roberson struct cpuset *set; 160727a3c9d7SJeff Roberson int i; 1608a6c7423aSMatt Macy int error __unused; 1609d7f687fcSJeff Roberson 1610d7f687fcSJeff Roberson cpuset_zone = uma_zcreate("cpuset", sizeof(struct cpuset), NULL, NULL, 1611e5818a53SJeff Roberson NULL, NULL, UMA_ALIGN_CACHE, 0); 16123f289c3fSJeff Roberson domainset_zone = uma_zcreate("domainset", sizeof(struct domainset), 1613e5818a53SJeff Roberson NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 161481198539SAlexander V. Chernikov 1615d7f687fcSJeff Roberson /* 1616e5818a53SJeff Roberson * Create the root system set (0) for the whole machine. Doesn't use 1617d7f687fcSJeff Roberson * cpuset_create() due to NULL parent. 1618d7f687fcSJeff Roberson */ 1619d7f687fcSJeff Roberson set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 162027a3c9d7SJeff Roberson CPU_COPY(&all_cpus, &set->cs_mask); 1621d7f687fcSJeff Roberson LIST_INIT(&set->cs_children); 1622d7f687fcSJeff Roberson LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); 16231a7bb896SMateusz Guzik refcount_init(&set->cs_ref, 1); 162427a3c9d7SJeff Roberson set->cs_flags = CPU_SET_ROOT | CPU_SET_RDONLY; 162529bb6c19SMark Johnston set->cs_domain = domainset0; 1626d7f687fcSJeff Roberson cpuset_zero = set; 1627a03ee000SJeff Roberson cpuset_root = &set->cs_mask; 162881198539SAlexander V. Chernikov 1629d7f687fcSJeff Roberson /* 1630e5818a53SJeff Roberson * Now derive a default (1), modifiable set from that to give out. 1631d7f687fcSJeff Roberson */ 16323f289c3fSJeff Roberson set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 163330b7c6f9SKyle Evans error = cpuset_init(set, cpuset_zero, NULL, NULL, 1); 1634d7f687fcSJeff Roberson KASSERT(error == 0, ("Error creating default set: %d\n", error)); 163581198539SAlexander V. Chernikov cpuset_default = set; 1636e5818a53SJeff Roberson /* 1637e5818a53SJeff Roberson * Create the kernel set (2). 1638e5818a53SJeff Roberson */ 1639e5818a53SJeff Roberson set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); 164030b7c6f9SKyle Evans error = cpuset_init(set, cpuset_zero, NULL, NULL, 2); 1641e5818a53SJeff Roberson KASSERT(error == 0, ("Error creating kernel set: %d\n", error)); 164229bb6c19SMark Johnston set->cs_domain = domainset2; 1643e5818a53SJeff Roberson cpuset_kernel = set; 164481198539SAlexander V. Chernikov 1645d7f687fcSJeff Roberson /* 1646d7f687fcSJeff Roberson * Initialize the unit allocator. 0 and 1 are allocated above. 1647d7f687fcSJeff Roberson */ 1648d57cd5ccSStephen J. Kiernan cpuset_unr = new_unrhdr(3, INT_MAX, NULL); 1649d7f687fcSJeff Roberson 165027a3c9d7SJeff Roberson /* 165127a3c9d7SJeff Roberson * If MD code has not initialized per-domain cpusets, place all 165227a3c9d7SJeff Roberson * CPUs in domain 0. 165327a3c9d7SJeff Roberson */ 165427a3c9d7SJeff Roberson for (i = 0; i < MAXMEMDOM; i++) 165527a3c9d7SJeff Roberson if (!CPU_EMPTY(&cpuset_domain[i])) 165627a3c9d7SJeff Roberson goto domains_set; 165727a3c9d7SJeff Roberson CPU_COPY(&all_cpus, &cpuset_domain[0]); 165827a3c9d7SJeff Roberson domains_set: 165927a3c9d7SJeff Roberson 1660e5818a53SJeff Roberson return (cpuset_default); 1661e5818a53SJeff Roberson } 1662e5818a53SJeff Roberson 1663e5818a53SJeff Roberson void 1664e5818a53SJeff Roberson cpuset_kernthread(struct thread *td) 1665e5818a53SJeff Roberson { 1666e5818a53SJeff Roberson struct cpuset *set; 1667e5818a53SJeff Roberson 1668e5818a53SJeff Roberson thread_lock(td); 1669e5818a53SJeff Roberson set = td->td_cpuset; 1670e5818a53SJeff Roberson td->td_cpuset = cpuset_ref(cpuset_kernel); 1671e5818a53SJeff Roberson thread_unlock(td); 1672e5818a53SJeff Roberson cpuset_rel(set); 1673d7f687fcSJeff Roberson } 1674d7f687fcSJeff Roberson 1675d7f687fcSJeff Roberson /* 1676413628a7SBjoern A. Zeeb * Create a cpuset, which would be cpuset_create() but 1677413628a7SBjoern A. Zeeb * mark the new 'set' as root. 1678413628a7SBjoern A. Zeeb * 167947479a8cSBjoern A. Zeeb * We are not going to reparent the td to it. Use cpuset_setproc_update_set() 168047479a8cSBjoern A. Zeeb * for that. 1681413628a7SBjoern A. Zeeb * 1682413628a7SBjoern A. Zeeb * In case of no error, returns the set in *setp locked with a reference. 1683413628a7SBjoern A. Zeeb */ 1684413628a7SBjoern A. Zeeb int 16850304c731SJamie Gritton cpuset_create_root(struct prison *pr, struct cpuset **setp) 1686413628a7SBjoern A. Zeeb { 1687413628a7SBjoern A. Zeeb struct cpuset *set; 1688413628a7SBjoern A. Zeeb int error; 1689413628a7SBjoern A. Zeeb 16900304c731SJamie Gritton KASSERT(pr != NULL, ("[%s:%d] invalid pr", __func__, __LINE__)); 1691413628a7SBjoern A. Zeeb KASSERT(setp != NULL, ("[%s:%d] invalid setp", __func__, __LINE__)); 1692413628a7SBjoern A. Zeeb 169329d04ea8SKyle Evans set = NULL; 169429d04ea8SKyle Evans error = cpuset_create(&set, pr->pr_cpuset, &pr->pr_cpuset->cs_mask); 1695413628a7SBjoern A. Zeeb if (error) 1696413628a7SBjoern A. Zeeb return (error); 1697413628a7SBjoern A. Zeeb 169829d04ea8SKyle Evans KASSERT(set != NULL, ("[%s:%d] cpuset_create returned invalid data", 1699413628a7SBjoern A. Zeeb __func__, __LINE__)); 1700413628a7SBjoern A. Zeeb 1701413628a7SBjoern A. Zeeb /* Mark the set as root. */ 1702413628a7SBjoern A. Zeeb set->cs_flags |= CPU_SET_ROOT; 170329d04ea8SKyle Evans *setp = set; 1704413628a7SBjoern A. Zeeb 1705413628a7SBjoern A. Zeeb return (0); 1706413628a7SBjoern A. Zeeb } 1707413628a7SBjoern A. Zeeb 1708413628a7SBjoern A. Zeeb int 1709413628a7SBjoern A. Zeeb cpuset_setproc_update_set(struct proc *p, struct cpuset *set) 1710413628a7SBjoern A. Zeeb { 1711413628a7SBjoern A. Zeeb int error; 1712413628a7SBjoern A. Zeeb 1713413628a7SBjoern A. Zeeb KASSERT(p != NULL, ("[%s:%d] invalid proc", __func__, __LINE__)); 1714413628a7SBjoern A. Zeeb KASSERT(set != NULL, ("[%s:%d] invalid set", __func__, __LINE__)); 1715413628a7SBjoern A. Zeeb 1716413628a7SBjoern A. Zeeb cpuset_ref(set); 1717d431dea5SKyle Evans error = cpuset_setproc(p->p_pid, set, NULL, NULL, true); 1718413628a7SBjoern A. Zeeb if (error) 1719413628a7SBjoern A. Zeeb return (error); 1720413628a7SBjoern A. Zeeb cpuset_rel(set); 1721413628a7SBjoern A. Zeeb return (0); 1722413628a7SBjoern A. Zeeb } 1723413628a7SBjoern A. Zeeb 17249eb997cbSMark Johnston /* 17259eb997cbSMark Johnston * In Capability mode, the only accesses that are permitted are to the current 17269eb997cbSMark Johnston * thread and process' CPU and domain sets. 17279eb997cbSMark Johnston */ 17289eb997cbSMark Johnston static int 17299eb997cbSMark Johnston cpuset_check_capabilities(struct thread *td, cpulevel_t level, cpuwhich_t which, 17309eb997cbSMark Johnston id_t id) 17319eb997cbSMark Johnston { 17329eb997cbSMark Johnston if (IN_CAPABILITY_MODE(td)) { 17339eb997cbSMark Johnston if (level != CPU_LEVEL_WHICH) 17349eb997cbSMark Johnston return (ECAPMODE); 17359eb997cbSMark Johnston if (which != CPU_WHICH_TID && which != CPU_WHICH_PID) 17369eb997cbSMark Johnston return (ECAPMODE); 173769b565d7SMark Johnston if (id != -1 && 173869b565d7SMark Johnston !(which == CPU_WHICH_TID && id == td->td_tid) && 173969b565d7SMark Johnston !(which == CPU_WHICH_PID && id == td->td_proc->p_pid)) 17409eb997cbSMark Johnston return (ECAPMODE); 17419eb997cbSMark Johnston } 17429eb997cbSMark Johnston return (0); 17439eb997cbSMark Johnston } 17449eb997cbSMark Johnston 1745d7f687fcSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 1746d7f687fcSJeff Roberson struct cpuset_args { 1747d7f687fcSJeff Roberson cpusetid_t *setid; 1748d7f687fcSJeff Roberson }; 1749d7f687fcSJeff Roberson #endif 1750d7f687fcSJeff Roberson int 17518451d0ddSKip Macy sys_cpuset(struct thread *td, struct cpuset_args *uap) 1752d7f687fcSJeff Roberson { 1753d7f687fcSJeff Roberson struct cpuset *root; 1754d7f687fcSJeff Roberson struct cpuset *set; 1755d7f687fcSJeff Roberson int error; 1756d7f687fcSJeff Roberson 1757d7f687fcSJeff Roberson thread_lock(td); 1758a03ee000SJeff Roberson root = cpuset_refroot(td->td_cpuset); 1759d7f687fcSJeff Roberson thread_unlock(td); 176029d04ea8SKyle Evans set = NULL; 1761d7f687fcSJeff Roberson error = cpuset_create(&set, root, &root->cs_mask); 1762d7f687fcSJeff Roberson cpuset_rel(root); 1763d7f687fcSJeff Roberson if (error) 1764d7f687fcSJeff Roberson return (error); 1765d7f687fcSJeff Roberson error = copyout(&set->cs_id, uap->setid, sizeof(set->cs_id)); 1766a03ee000SJeff Roberson if (error == 0) 1767d431dea5SKyle Evans error = cpuset_setproc(-1, set, NULL, NULL, false); 1768d7f687fcSJeff Roberson cpuset_rel(set); 1769d7f687fcSJeff Roberson return (error); 1770d7f687fcSJeff Roberson } 1771d7f687fcSJeff Roberson 1772d7f687fcSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 1773d7f687fcSJeff Roberson struct cpuset_setid_args { 1774d7f687fcSJeff Roberson cpuwhich_t which; 1775d7f687fcSJeff Roberson id_t id; 1776d7f687fcSJeff Roberson cpusetid_t setid; 1777d7f687fcSJeff Roberson }; 1778d7f687fcSJeff Roberson #endif 1779d7f687fcSJeff Roberson int 17808451d0ddSKip Macy sys_cpuset_setid(struct thread *td, struct cpuset_setid_args *uap) 1781d7f687fcSJeff Roberson { 1782ea2ebdc1SEdward Tomasz Napierala 1783ea2ebdc1SEdward Tomasz Napierala return (kern_cpuset_setid(td, uap->which, uap->id, uap->setid)); 1784ea2ebdc1SEdward Tomasz Napierala } 1785ea2ebdc1SEdward Tomasz Napierala 1786ea2ebdc1SEdward Tomasz Napierala int 1787ea2ebdc1SEdward Tomasz Napierala kern_cpuset_setid(struct thread *td, cpuwhich_t which, 1788ea2ebdc1SEdward Tomasz Napierala id_t id, cpusetid_t setid) 1789ea2ebdc1SEdward Tomasz Napierala { 1790d7f687fcSJeff Roberson struct cpuset *set; 1791d7f687fcSJeff Roberson int error; 1792d7f687fcSJeff Roberson 1793d7f687fcSJeff Roberson /* 1794d7f687fcSJeff Roberson * Presently we only support per-process sets. 1795d7f687fcSJeff Roberson */ 1796ea2ebdc1SEdward Tomasz Napierala if (which != CPU_WHICH_PID) 1797d7f687fcSJeff Roberson return (EINVAL); 1798ea2ebdc1SEdward Tomasz Napierala set = cpuset_lookup(setid, td); 1799d7f687fcSJeff Roberson if (set == NULL) 1800d7f687fcSJeff Roberson return (ESRCH); 1801d431dea5SKyle Evans error = cpuset_setproc(id, set, NULL, NULL, false); 1802d7f687fcSJeff Roberson cpuset_rel(set); 1803d7f687fcSJeff Roberson return (error); 1804d7f687fcSJeff Roberson } 1805d7f687fcSJeff Roberson 1806d7f687fcSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 1807d7f687fcSJeff Roberson struct cpuset_getid_args { 1808d7f687fcSJeff Roberson cpulevel_t level; 1809d7f687fcSJeff Roberson cpuwhich_t which; 1810d7f687fcSJeff Roberson id_t id; 1811d7f687fcSJeff Roberson cpusetid_t *setid; 18122b69bb1fSKevin Lo }; 1813d7f687fcSJeff Roberson #endif 1814d7f687fcSJeff Roberson int 18158451d0ddSKip Macy sys_cpuset_getid(struct thread *td, struct cpuset_getid_args *uap) 1816d7f687fcSJeff Roberson { 1817ea2ebdc1SEdward Tomasz Napierala 1818ea2ebdc1SEdward Tomasz Napierala return (kern_cpuset_getid(td, uap->level, uap->which, uap->id, 1819ea2ebdc1SEdward Tomasz Napierala uap->setid)); 1820ea2ebdc1SEdward Tomasz Napierala } 1821ea2ebdc1SEdward Tomasz Napierala 1822ea2ebdc1SEdward Tomasz Napierala int 1823ea2ebdc1SEdward Tomasz Napierala kern_cpuset_getid(struct thread *td, cpulevel_t level, cpuwhich_t which, 1824ea2ebdc1SEdward Tomasz Napierala id_t id, cpusetid_t *setid) 1825ea2ebdc1SEdward Tomasz Napierala { 1826d7f687fcSJeff Roberson struct cpuset *nset; 1827d7f687fcSJeff Roberson struct cpuset *set; 1828d7f687fcSJeff Roberson struct thread *ttd; 1829d7f687fcSJeff Roberson struct proc *p; 1830ea2ebdc1SEdward Tomasz Napierala cpusetid_t tmpid; 1831d7f687fcSJeff Roberson int error; 1832d7f687fcSJeff Roberson 1833ea2ebdc1SEdward Tomasz Napierala if (level == CPU_LEVEL_WHICH && which != CPU_WHICH_CPUSET) 1834d7f687fcSJeff Roberson return (EINVAL); 1835ea2ebdc1SEdward Tomasz Napierala error = cpuset_which(which, id, &p, &ttd, &set); 1836d7f687fcSJeff Roberson if (error) 1837d7f687fcSJeff Roberson return (error); 1838ea2ebdc1SEdward Tomasz Napierala switch (which) { 1839d7f687fcSJeff Roberson case CPU_WHICH_TID: 1840d7f687fcSJeff Roberson case CPU_WHICH_PID: 1841d7f687fcSJeff Roberson thread_lock(ttd); 1842a03ee000SJeff Roberson set = cpuset_refbase(ttd->td_cpuset); 1843d7f687fcSJeff Roberson thread_unlock(ttd); 1844d7f687fcSJeff Roberson PROC_UNLOCK(p); 1845d7f687fcSJeff Roberson break; 1846d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 1847413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 1848d7f687fcSJeff Roberson break; 18499b33b154SJeff Roberson case CPU_WHICH_IRQ: 1850c0ae6688SJohn Baldwin case CPU_WHICH_DOMAIN: 18519b33b154SJeff Roberson return (EINVAL); 1852d7f687fcSJeff Roberson } 1853ea2ebdc1SEdward Tomasz Napierala switch (level) { 1854d7f687fcSJeff Roberson case CPU_LEVEL_ROOT: 1855a03ee000SJeff Roberson nset = cpuset_refroot(set); 1856d7f687fcSJeff Roberson cpuset_rel(set); 1857d7f687fcSJeff Roberson set = nset; 1858d7f687fcSJeff Roberson break; 1859d7f687fcSJeff Roberson case CPU_LEVEL_CPUSET: 1860d7f687fcSJeff Roberson break; 1861d7f687fcSJeff Roberson case CPU_LEVEL_WHICH: 1862d7f687fcSJeff Roberson break; 1863d7f687fcSJeff Roberson } 1864ea2ebdc1SEdward Tomasz Napierala tmpid = set->cs_id; 1865d7f687fcSJeff Roberson cpuset_rel(set); 1866d7f687fcSJeff Roberson if (error == 0) 1867a1d0659cSJung-uk Kim error = copyout(&tmpid, setid, sizeof(tmpid)); 1868d7f687fcSJeff Roberson 1869d7f687fcSJeff Roberson return (error); 1870d7f687fcSJeff Roberson } 1871d7f687fcSJeff Roberson 1872d7f687fcSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 1873d7f687fcSJeff Roberson struct cpuset_getaffinity_args { 1874d7f687fcSJeff Roberson cpulevel_t level; 1875d7f687fcSJeff Roberson cpuwhich_t which; 18767f64829aSRuslan Ermilov id_t id; 18777f64829aSRuslan Ermilov size_t cpusetsize; 18787f64829aSRuslan Ermilov cpuset_t *mask; 1879d7f687fcSJeff Roberson }; 1880d7f687fcSJeff Roberson #endif 1881d7f687fcSJeff Roberson int 18828451d0ddSKip Macy sys_cpuset_getaffinity(struct thread *td, struct cpuset_getaffinity_args *uap) 1883d7f687fcSJeff Roberson { 188496ee4310SEdward Tomasz Napierala 188596ee4310SEdward Tomasz Napierala return (kern_cpuset_getaffinity(td, uap->level, uap->which, 188696ee4310SEdward Tomasz Napierala uap->id, uap->cpusetsize, uap->mask)); 188796ee4310SEdward Tomasz Napierala } 188896ee4310SEdward Tomasz Napierala 188996ee4310SEdward Tomasz Napierala int 189096ee4310SEdward Tomasz Napierala kern_cpuset_getaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, 189196ee4310SEdward Tomasz Napierala id_t id, size_t cpusetsize, cpuset_t *maskp) 189296ee4310SEdward Tomasz Napierala { 1893d7f687fcSJeff Roberson struct thread *ttd; 1894d7f687fcSJeff Roberson struct cpuset *nset; 1895d7f687fcSJeff Roberson struct cpuset *set; 1896d7f687fcSJeff Roberson struct proc *p; 1897d7f687fcSJeff Roberson cpuset_t *mask; 1898d7f687fcSJeff Roberson int error; 18997f64829aSRuslan Ermilov size_t size; 1900d7f687fcSJeff Roberson 19019eb997cbSMark Johnston error = cpuset_check_capabilities(td, level, which, id); 19029eb997cbSMark Johnston if (error != 0) 19039eb997cbSMark Johnston return (error); 1904f35093f8SDmitry Chagin mask = malloc(sizeof(cpuset_t), M_TEMP, M_WAITOK | M_ZERO); 190596ee4310SEdward Tomasz Napierala error = cpuset_which(which, id, &p, &ttd, &set); 1906d7f687fcSJeff Roberson if (error) 1907d7f687fcSJeff Roberson goto out; 190896ee4310SEdward Tomasz Napierala switch (level) { 1909d7f687fcSJeff Roberson case CPU_LEVEL_ROOT: 1910d7f687fcSJeff Roberson case CPU_LEVEL_CPUSET: 191196ee4310SEdward Tomasz Napierala switch (which) { 1912d7f687fcSJeff Roberson case CPU_WHICH_TID: 1913d7f687fcSJeff Roberson case CPU_WHICH_PID: 1914d7f687fcSJeff Roberson thread_lock(ttd); 1915d7f687fcSJeff Roberson set = cpuset_ref(ttd->td_cpuset); 1916d7f687fcSJeff Roberson thread_unlock(ttd); 1917d7f687fcSJeff Roberson break; 1918d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 1919413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 1920d7f687fcSJeff Roberson break; 19219b33b154SJeff Roberson case CPU_WHICH_IRQ: 192229dfb631SConrad Meyer case CPU_WHICH_INTRHANDLER: 192329dfb631SConrad Meyer case CPU_WHICH_ITHREAD: 1924c0ae6688SJohn Baldwin case CPU_WHICH_DOMAIN: 19259b33b154SJeff Roberson error = EINVAL; 19269b33b154SJeff Roberson goto out; 1927d7f687fcSJeff Roberson } 192896ee4310SEdward Tomasz Napierala if (level == CPU_LEVEL_ROOT) 1929a03ee000SJeff Roberson nset = cpuset_refroot(set); 1930d7f687fcSJeff Roberson else 1931a03ee000SJeff Roberson nset = cpuset_refbase(set); 1932d7f687fcSJeff Roberson CPU_COPY(&nset->cs_mask, mask); 1933d7f687fcSJeff Roberson cpuset_rel(nset); 1934d7f687fcSJeff Roberson break; 1935d7f687fcSJeff Roberson case CPU_LEVEL_WHICH: 193696ee4310SEdward Tomasz Napierala switch (which) { 1937d7f687fcSJeff Roberson case CPU_WHICH_TID: 1938d7f687fcSJeff Roberson thread_lock(ttd); 1939d7f687fcSJeff Roberson CPU_COPY(&ttd->td_cpuset->cs_mask, mask); 1940d7f687fcSJeff Roberson thread_unlock(ttd); 1941d7f687fcSJeff Roberson break; 1942d7f687fcSJeff Roberson case CPU_WHICH_PID: 1943d7f687fcSJeff Roberson FOREACH_THREAD_IN_PROC(p, ttd) { 1944d7f687fcSJeff Roberson thread_lock(ttd); 1945e2650af1SStefan Eßer CPU_OR(mask, mask, &ttd->td_cpuset->cs_mask); 1946d7f687fcSJeff Roberson thread_unlock(ttd); 1947d7f687fcSJeff Roberson } 1948d7f687fcSJeff Roberson break; 1949d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 1950413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 1951d7f687fcSJeff Roberson CPU_COPY(&set->cs_mask, mask); 1952d7f687fcSJeff Roberson break; 19539b33b154SJeff Roberson case CPU_WHICH_IRQ: 195429dfb631SConrad Meyer case CPU_WHICH_INTRHANDLER: 195529dfb631SConrad Meyer case CPU_WHICH_ITHREAD: 195629dfb631SConrad Meyer error = intr_getaffinity(id, which, mask); 19579b33b154SJeff Roberson break; 1958c0ae6688SJohn Baldwin case CPU_WHICH_DOMAIN: 195996ee4310SEdward Tomasz Napierala if (id < 0 || id >= MAXMEMDOM) 1960c0ae6688SJohn Baldwin error = ESRCH; 1961c0ae6688SJohn Baldwin else 196296ee4310SEdward Tomasz Napierala CPU_COPY(&cpuset_domain[id], mask); 1963c0ae6688SJohn Baldwin break; 1964d7f687fcSJeff Roberson } 1965d7f687fcSJeff Roberson break; 1966d7f687fcSJeff Roberson default: 1967d7f687fcSJeff Roberson error = EINVAL; 1968d7f687fcSJeff Roberson break; 1969d7f687fcSJeff Roberson } 1970d7f687fcSJeff Roberson if (set) 1971d7f687fcSJeff Roberson cpuset_rel(set); 1972d7f687fcSJeff Roberson if (p) 1973d7f687fcSJeff Roberson PROC_UNLOCK(p); 1974f35093f8SDmitry Chagin if (error == 0) { 1975f35093f8SDmitry Chagin if (cpusetsize < howmany(CPU_FLS(mask), NBBY)) { 1976f35093f8SDmitry Chagin error = ERANGE; 1977f35093f8SDmitry Chagin goto out; 1978f35093f8SDmitry Chagin } 1979f35093f8SDmitry Chagin size = min(cpusetsize, sizeof(cpuset_t)); 198096ee4310SEdward Tomasz Napierala error = copyout(mask, maskp, size); 1981f35093f8SDmitry Chagin if (error != 0) 1982f35093f8SDmitry Chagin goto out; 1983f35093f8SDmitry Chagin if (cpusetsize > size) { 1984f35093f8SDmitry Chagin char *end; 1985f35093f8SDmitry Chagin char *cp; 1986f35093f8SDmitry Chagin int rv; 1987f35093f8SDmitry Chagin 1988f35093f8SDmitry Chagin end = cp = (char *)&maskp->__bits; 1989f35093f8SDmitry Chagin end += cpusetsize; 1990f35093f8SDmitry Chagin cp += size; 1991f35093f8SDmitry Chagin while (cp != end) { 1992f35093f8SDmitry Chagin rv = subyte(cp, 0); 1993f35093f8SDmitry Chagin if (rv == -1) { 1994f35093f8SDmitry Chagin error = EFAULT; 1995f35093f8SDmitry Chagin goto out; 1996f35093f8SDmitry Chagin } 1997f35093f8SDmitry Chagin cp++; 1998f35093f8SDmitry Chagin } 1999f35093f8SDmitry Chagin } 2000586ed321SDmitry Chagin #ifdef KTRACE 2001586ed321SDmitry Chagin if ( KTRPOINT(td, KTR_STRUCT)) 2002586ed321SDmitry Chagin ktrcpuset(mask, size); 2003586ed321SDmitry Chagin #endif 2004f35093f8SDmitry Chagin } 2005d7f687fcSJeff Roberson out: 2006d7f687fcSJeff Roberson free(mask, M_TEMP); 2007d7f687fcSJeff Roberson return (error); 2008d7f687fcSJeff Roberson } 2009d7f687fcSJeff Roberson 2010d7f687fcSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 2011d7f687fcSJeff Roberson struct cpuset_setaffinity_args { 2012d7f687fcSJeff Roberson cpulevel_t level; 2013d7f687fcSJeff Roberson cpuwhich_t which; 20147f64829aSRuslan Ermilov id_t id; 20157f64829aSRuslan Ermilov size_t cpusetsize; 20167f64829aSRuslan Ermilov const cpuset_t *mask; 2017d7f687fcSJeff Roberson }; 2018d7f687fcSJeff Roberson #endif 2019d7f687fcSJeff Roberson int 20208451d0ddSKip Macy sys_cpuset_setaffinity(struct thread *td, struct cpuset_setaffinity_args *uap) 2021d7f687fcSJeff Roberson { 202296ee4310SEdward Tomasz Napierala 2023f35093f8SDmitry Chagin return (user_cpuset_setaffinity(td, uap->level, uap->which, 202496ee4310SEdward Tomasz Napierala uap->id, uap->cpusetsize, uap->mask)); 202596ee4310SEdward Tomasz Napierala } 202696ee4310SEdward Tomasz Napierala 202796ee4310SEdward Tomasz Napierala int 202896ee4310SEdward Tomasz Napierala kern_cpuset_setaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, 2029f35093f8SDmitry Chagin id_t id, cpuset_t *mask) 203096ee4310SEdward Tomasz Napierala { 2031d7f687fcSJeff Roberson struct cpuset *nset; 2032d7f687fcSJeff Roberson struct cpuset *set; 2033d7f687fcSJeff Roberson struct thread *ttd; 2034d7f687fcSJeff Roberson struct proc *p; 2035d7f687fcSJeff Roberson int error; 2036d7f687fcSJeff Roberson 2037586ed321SDmitry Chagin #ifdef KTRACE 2038586ed321SDmitry Chagin if (KTRPOINT(td, KTR_STRUCT)) 2039586ed321SDmitry Chagin ktrcpuset(mask, sizeof(cpuset_t)); 2040586ed321SDmitry Chagin #endif 20419eb997cbSMark Johnston error = cpuset_check_capabilities(td, level, which, id); 20429eb997cbSMark Johnston if (error != 0) 20439eb997cbSMark Johnston return (error); 2044f35093f8SDmitry Chagin if (CPU_EMPTY(mask)) 2045f35093f8SDmitry Chagin return (EDEADLK); 204696ee4310SEdward Tomasz Napierala switch (level) { 2047d7f687fcSJeff Roberson case CPU_LEVEL_ROOT: 2048d7f687fcSJeff Roberson case CPU_LEVEL_CPUSET: 204996ee4310SEdward Tomasz Napierala error = cpuset_which(which, id, &p, &ttd, &set); 2050d7f687fcSJeff Roberson if (error) 2051d7f687fcSJeff Roberson break; 205296ee4310SEdward Tomasz Napierala switch (which) { 2053d7f687fcSJeff Roberson case CPU_WHICH_TID: 2054d7f687fcSJeff Roberson case CPU_WHICH_PID: 2055d7f687fcSJeff Roberson thread_lock(ttd); 2056d7f687fcSJeff Roberson set = cpuset_ref(ttd->td_cpuset); 2057d7f687fcSJeff Roberson thread_unlock(ttd); 2058c6440f72SJeff Roberson PROC_UNLOCK(p); 2059d7f687fcSJeff Roberson break; 2060d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 2061413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 2062d7f687fcSJeff Roberson break; 20639b33b154SJeff Roberson case CPU_WHICH_IRQ: 206429dfb631SConrad Meyer case CPU_WHICH_INTRHANDLER: 206529dfb631SConrad Meyer case CPU_WHICH_ITHREAD: 2066c0ae6688SJohn Baldwin case CPU_WHICH_DOMAIN: 2067f35093f8SDmitry Chagin return (EINVAL); 2068d7f687fcSJeff Roberson } 206996ee4310SEdward Tomasz Napierala if (level == CPU_LEVEL_ROOT) 2070a03ee000SJeff Roberson nset = cpuset_refroot(set); 2071d7f687fcSJeff Roberson else 2072a03ee000SJeff Roberson nset = cpuset_refbase(set); 2073d7f687fcSJeff Roberson error = cpuset_modify(nset, mask); 2074d7f687fcSJeff Roberson cpuset_rel(nset); 2075d7f687fcSJeff Roberson cpuset_rel(set); 2076d7f687fcSJeff Roberson break; 2077d7f687fcSJeff Roberson case CPU_LEVEL_WHICH: 207896ee4310SEdward Tomasz Napierala switch (which) { 2079d7f687fcSJeff Roberson case CPU_WHICH_TID: 208096ee4310SEdward Tomasz Napierala error = cpuset_setthread(id, mask); 2081d7f687fcSJeff Roberson break; 2082d7f687fcSJeff Roberson case CPU_WHICH_PID: 2083d431dea5SKyle Evans error = cpuset_setproc(id, NULL, mask, NULL, false); 2084d7f687fcSJeff Roberson break; 2085d7f687fcSJeff Roberson case CPU_WHICH_CPUSET: 2086413628a7SBjoern A. Zeeb case CPU_WHICH_JAIL: 208796ee4310SEdward Tomasz Napierala error = cpuset_which(which, id, &p, &ttd, &set); 2088d7f687fcSJeff Roberson if (error == 0) { 2089d7f687fcSJeff Roberson error = cpuset_modify(set, mask); 2090d7f687fcSJeff Roberson cpuset_rel(set); 2091d7f687fcSJeff Roberson } 2092d7f687fcSJeff Roberson break; 20939b33b154SJeff Roberson case CPU_WHICH_IRQ: 209429dfb631SConrad Meyer case CPU_WHICH_INTRHANDLER: 209529dfb631SConrad Meyer case CPU_WHICH_ITHREAD: 209629dfb631SConrad Meyer error = intr_setaffinity(id, which, mask); 20979b33b154SJeff Roberson break; 2098d7f687fcSJeff Roberson default: 2099d7f687fcSJeff Roberson error = EINVAL; 2100d7f687fcSJeff Roberson break; 2101d7f687fcSJeff Roberson } 2102d7f687fcSJeff Roberson break; 2103d7f687fcSJeff Roberson default: 2104d7f687fcSJeff Roberson error = EINVAL; 2105d7f687fcSJeff Roberson break; 2106d7f687fcSJeff Roberson } 2107f35093f8SDmitry Chagin return (error); 2108f35093f8SDmitry Chagin } 2109f35093f8SDmitry Chagin 2110f35093f8SDmitry Chagin int 2111f35093f8SDmitry Chagin user_cpuset_setaffinity(struct thread *td, cpulevel_t level, cpuwhich_t which, 2112f35093f8SDmitry Chagin id_t id, size_t cpusetsize, const cpuset_t *maskp) 2113f35093f8SDmitry Chagin { 2114f35093f8SDmitry Chagin cpuset_t *mask; 2115f35093f8SDmitry Chagin int error; 2116f35093f8SDmitry Chagin size_t size; 2117f35093f8SDmitry Chagin 2118f35093f8SDmitry Chagin size = min(cpusetsize, sizeof(cpuset_t)); 2119f35093f8SDmitry Chagin mask = malloc(sizeof(cpuset_t), M_TEMP, M_WAITOK | M_ZERO); 2120f35093f8SDmitry Chagin error = copyin(maskp, mask, size); 2121f35093f8SDmitry Chagin if (error) 2122f35093f8SDmitry Chagin goto out; 2123f35093f8SDmitry Chagin /* 2124f35093f8SDmitry Chagin * Verify that no high bits are set. 2125f35093f8SDmitry Chagin */ 2126f35093f8SDmitry Chagin if (cpusetsize > sizeof(cpuset_t)) { 2127f35093f8SDmitry Chagin const char *end, *cp; 2128f35093f8SDmitry Chagin int val; 2129f35093f8SDmitry Chagin end = cp = (const char *)&maskp->__bits; 2130f35093f8SDmitry Chagin end += cpusetsize; 2131f35093f8SDmitry Chagin cp += sizeof(cpuset_t); 2132f35093f8SDmitry Chagin 2133f35093f8SDmitry Chagin while (cp != end) { 2134f35093f8SDmitry Chagin val = fubyte(cp); 2135f35093f8SDmitry Chagin if (val == -1) { 2136f35093f8SDmitry Chagin error = EFAULT; 2137f35093f8SDmitry Chagin goto out; 2138f35093f8SDmitry Chagin } 2139f35093f8SDmitry Chagin if (val != 0) { 2140f35093f8SDmitry Chagin error = EINVAL; 2141f35093f8SDmitry Chagin goto out; 2142f35093f8SDmitry Chagin } 2143f35093f8SDmitry Chagin cp++; 2144f35093f8SDmitry Chagin } 2145f35093f8SDmitry Chagin } 2146f35093f8SDmitry Chagin error = kern_cpuset_setaffinity(td, level, which, id, mask); 2147f35093f8SDmitry Chagin 2148d7f687fcSJeff Roberson out: 2149d7f687fcSJeff Roberson free(mask, M_TEMP); 2150d7f687fcSJeff Roberson return (error); 2151d7f687fcSJeff Roberson } 2152dea0ed66SBjoern A. Zeeb 21533f289c3fSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 21543f289c3fSJeff Roberson struct cpuset_getdomain_args { 21553f289c3fSJeff Roberson cpulevel_t level; 21563f289c3fSJeff Roberson cpuwhich_t which; 21573f289c3fSJeff Roberson id_t id; 21583f289c3fSJeff Roberson size_t domainsetsize; 21593f289c3fSJeff Roberson domainset_t *mask; 21603f289c3fSJeff Roberson int *policy; 21613f289c3fSJeff Roberson }; 21623f289c3fSJeff Roberson #endif 21633f289c3fSJeff Roberson int 21643f289c3fSJeff Roberson sys_cpuset_getdomain(struct thread *td, struct cpuset_getdomain_args *uap) 2165dea0ed66SBjoern A. Zeeb { 2166dea0ed66SBjoern A. Zeeb 21673f289c3fSJeff Roberson return (kern_cpuset_getdomain(td, uap->level, uap->which, 21683f289c3fSJeff Roberson uap->id, uap->domainsetsize, uap->mask, uap->policy)); 21693f289c3fSJeff Roberson } 21703f289c3fSJeff Roberson 21713f289c3fSJeff Roberson int 21723f289c3fSJeff Roberson kern_cpuset_getdomain(struct thread *td, cpulevel_t level, cpuwhich_t which, 21733f289c3fSJeff Roberson id_t id, size_t domainsetsize, domainset_t *maskp, int *policyp) 21743f289c3fSJeff Roberson { 21753f289c3fSJeff Roberson struct domainset outset; 21763f289c3fSJeff Roberson struct thread *ttd; 21773f289c3fSJeff Roberson struct cpuset *nset; 21783f289c3fSJeff Roberson struct cpuset *set; 21793f289c3fSJeff Roberson struct domainset *dset; 21803f289c3fSJeff Roberson struct proc *p; 21813f289c3fSJeff Roberson domainset_t *mask; 21823f289c3fSJeff Roberson int error; 21833f289c3fSJeff Roberson 21843f289c3fSJeff Roberson if (domainsetsize < sizeof(domainset_t) || 21853f289c3fSJeff Roberson domainsetsize > DOMAINSET_MAXSIZE / NBBY) 21863f289c3fSJeff Roberson return (ERANGE); 21879eb997cbSMark Johnston error = cpuset_check_capabilities(td, level, which, id); 21889eb997cbSMark Johnston if (error != 0) 21899eb997cbSMark Johnston return (error); 21903f289c3fSJeff Roberson mask = malloc(domainsetsize, M_TEMP, M_WAITOK | M_ZERO); 21913f289c3fSJeff Roberson bzero(&outset, sizeof(outset)); 21923f289c3fSJeff Roberson error = cpuset_which(which, id, &p, &ttd, &set); 21933f289c3fSJeff Roberson if (error) 21943f289c3fSJeff Roberson goto out; 21953f289c3fSJeff Roberson switch (level) { 21963f289c3fSJeff Roberson case CPU_LEVEL_ROOT: 21973f289c3fSJeff Roberson case CPU_LEVEL_CPUSET: 21983f289c3fSJeff Roberson switch (which) { 21993f289c3fSJeff Roberson case CPU_WHICH_TID: 22003f289c3fSJeff Roberson case CPU_WHICH_PID: 22013f289c3fSJeff Roberson thread_lock(ttd); 22023f289c3fSJeff Roberson set = cpuset_ref(ttd->td_cpuset); 22033f289c3fSJeff Roberson thread_unlock(ttd); 22043f289c3fSJeff Roberson break; 22053f289c3fSJeff Roberson case CPU_WHICH_CPUSET: 22063f289c3fSJeff Roberson case CPU_WHICH_JAIL: 22073f289c3fSJeff Roberson break; 22083f289c3fSJeff Roberson case CPU_WHICH_IRQ: 22093f289c3fSJeff Roberson case CPU_WHICH_INTRHANDLER: 22103f289c3fSJeff Roberson case CPU_WHICH_ITHREAD: 22113f289c3fSJeff Roberson case CPU_WHICH_DOMAIN: 22123f289c3fSJeff Roberson error = EINVAL; 22133f289c3fSJeff Roberson goto out; 22143f289c3fSJeff Roberson } 22153f289c3fSJeff Roberson if (level == CPU_LEVEL_ROOT) 22163f289c3fSJeff Roberson nset = cpuset_refroot(set); 22173f289c3fSJeff Roberson else 22183f289c3fSJeff Roberson nset = cpuset_refbase(set); 22193f289c3fSJeff Roberson domainset_copy(nset->cs_domain, &outset); 22203f289c3fSJeff Roberson cpuset_rel(nset); 22213f289c3fSJeff Roberson break; 22223f289c3fSJeff Roberson case CPU_LEVEL_WHICH: 22233f289c3fSJeff Roberson switch (which) { 22243f289c3fSJeff Roberson case CPU_WHICH_TID: 22253f289c3fSJeff Roberson thread_lock(ttd); 22263f289c3fSJeff Roberson domainset_copy(ttd->td_cpuset->cs_domain, &outset); 22273f289c3fSJeff Roberson thread_unlock(ttd); 22283f289c3fSJeff Roberson break; 22293f289c3fSJeff Roberson case CPU_WHICH_PID: 22303f289c3fSJeff Roberson FOREACH_THREAD_IN_PROC(p, ttd) { 22313f289c3fSJeff Roberson thread_lock(ttd); 22323f289c3fSJeff Roberson dset = ttd->td_cpuset->cs_domain; 22333f289c3fSJeff Roberson /* Show all domains in the proc. */ 22343f289c3fSJeff Roberson DOMAINSET_OR(&outset.ds_mask, &dset->ds_mask); 22353f289c3fSJeff Roberson /* Last policy wins. */ 22363f289c3fSJeff Roberson outset.ds_policy = dset->ds_policy; 22373f289c3fSJeff Roberson outset.ds_prefer = dset->ds_prefer; 22383f289c3fSJeff Roberson thread_unlock(ttd); 22393f289c3fSJeff Roberson } 22403f289c3fSJeff Roberson break; 22413f289c3fSJeff Roberson case CPU_WHICH_CPUSET: 22423f289c3fSJeff Roberson case CPU_WHICH_JAIL: 22433f289c3fSJeff Roberson domainset_copy(set->cs_domain, &outset); 22443f289c3fSJeff Roberson break; 22453f289c3fSJeff Roberson case CPU_WHICH_IRQ: 22463f289c3fSJeff Roberson case CPU_WHICH_INTRHANDLER: 22473f289c3fSJeff Roberson case CPU_WHICH_ITHREAD: 22483f289c3fSJeff Roberson case CPU_WHICH_DOMAIN: 22493f289c3fSJeff Roberson error = EINVAL; 22503f289c3fSJeff Roberson break; 22513f289c3fSJeff Roberson } 22523f289c3fSJeff Roberson break; 22533f289c3fSJeff Roberson default: 22543f289c3fSJeff Roberson error = EINVAL; 22553f289c3fSJeff Roberson break; 22563f289c3fSJeff Roberson } 22573f289c3fSJeff Roberson if (set) 22583f289c3fSJeff Roberson cpuset_rel(set); 22593f289c3fSJeff Roberson if (p) 22603f289c3fSJeff Roberson PROC_UNLOCK(p); 22613f289c3fSJeff Roberson /* 22623f289c3fSJeff Roberson * Translate prefer into a set containing only the preferred domain, 22633f289c3fSJeff Roberson * not the entire fallback set. 22643f289c3fSJeff Roberson */ 22653f289c3fSJeff Roberson if (outset.ds_policy == DOMAINSET_POLICY_PREFER) { 22663f289c3fSJeff Roberson DOMAINSET_ZERO(&outset.ds_mask); 22673f289c3fSJeff Roberson DOMAINSET_SET(outset.ds_prefer, &outset.ds_mask); 22683f289c3fSJeff Roberson } 22693f289c3fSJeff Roberson DOMAINSET_COPY(&outset.ds_mask, mask); 22703f289c3fSJeff Roberson if (error == 0) 22713f289c3fSJeff Roberson error = copyout(mask, maskp, domainsetsize); 22723f289c3fSJeff Roberson if (error == 0) 2273dd51fec3SBrooks Davis if (suword32(policyp, outset.ds_policy) != 0) 2274dd51fec3SBrooks Davis error = EFAULT; 22753f289c3fSJeff Roberson out: 22763f289c3fSJeff Roberson free(mask, M_TEMP); 22773f289c3fSJeff Roberson return (error); 22783f289c3fSJeff Roberson } 22793f289c3fSJeff Roberson 22803f289c3fSJeff Roberson #ifndef _SYS_SYSPROTO_H_ 22813f289c3fSJeff Roberson struct cpuset_setdomain_args { 22823f289c3fSJeff Roberson cpulevel_t level; 22833f289c3fSJeff Roberson cpuwhich_t which; 22843f289c3fSJeff Roberson id_t id; 22853f289c3fSJeff Roberson size_t domainsetsize; 22863f289c3fSJeff Roberson domainset_t *mask; 22873f289c3fSJeff Roberson int policy; 22883f289c3fSJeff Roberson }; 22893f289c3fSJeff Roberson #endif 22903f289c3fSJeff Roberson int 22913f289c3fSJeff Roberson sys_cpuset_setdomain(struct thread *td, struct cpuset_setdomain_args *uap) 22923f289c3fSJeff Roberson { 22933f289c3fSJeff Roberson 22943f289c3fSJeff Roberson return (kern_cpuset_setdomain(td, uap->level, uap->which, 22953f289c3fSJeff Roberson uap->id, uap->domainsetsize, uap->mask, uap->policy)); 22963f289c3fSJeff Roberson } 22973f289c3fSJeff Roberson 22983f289c3fSJeff Roberson int 22993f289c3fSJeff Roberson kern_cpuset_setdomain(struct thread *td, cpulevel_t level, cpuwhich_t which, 23003f289c3fSJeff Roberson id_t id, size_t domainsetsize, const domainset_t *maskp, int policy) 23013f289c3fSJeff Roberson { 23023f289c3fSJeff Roberson struct cpuset *nset; 23033f289c3fSJeff Roberson struct cpuset *set; 23043f289c3fSJeff Roberson struct thread *ttd; 23053f289c3fSJeff Roberson struct proc *p; 23063f289c3fSJeff Roberson struct domainset domain; 23073f289c3fSJeff Roberson domainset_t *mask; 23083f289c3fSJeff Roberson int error; 23093f289c3fSJeff Roberson 23103f289c3fSJeff Roberson if (domainsetsize < sizeof(domainset_t) || 23113f289c3fSJeff Roberson domainsetsize > DOMAINSET_MAXSIZE / NBBY) 23123f289c3fSJeff Roberson return (ERANGE); 231370d66bcfSEric van Gyzen if (policy <= DOMAINSET_POLICY_INVALID || 231470d66bcfSEric van Gyzen policy > DOMAINSET_POLICY_MAX) 231570d66bcfSEric van Gyzen return (EINVAL); 23169eb997cbSMark Johnston error = cpuset_check_capabilities(td, level, which, id); 23179eb997cbSMark Johnston if (error != 0) 23189eb997cbSMark Johnston return (error); 23193f289c3fSJeff Roberson memset(&domain, 0, sizeof(domain)); 23203f289c3fSJeff Roberson mask = malloc(domainsetsize, M_TEMP, M_WAITOK | M_ZERO); 23213f289c3fSJeff Roberson error = copyin(maskp, mask, domainsetsize); 23223f289c3fSJeff Roberson if (error) 23233f289c3fSJeff Roberson goto out; 23243f289c3fSJeff Roberson /* 23253f289c3fSJeff Roberson * Verify that no high bits are set. 23263f289c3fSJeff Roberson */ 23273f289c3fSJeff Roberson if (domainsetsize > sizeof(domainset_t)) { 23283f289c3fSJeff Roberson char *end; 23293f289c3fSJeff Roberson char *cp; 23303f289c3fSJeff Roberson 23313f289c3fSJeff Roberson end = cp = (char *)&mask->__bits; 23323f289c3fSJeff Roberson end += domainsetsize; 23333f289c3fSJeff Roberson cp += sizeof(domainset_t); 23343f289c3fSJeff Roberson while (cp != end) 23353f289c3fSJeff Roberson if (*cp++ != 0) { 23363f289c3fSJeff Roberson error = EINVAL; 23373f289c3fSJeff Roberson goto out; 23383f289c3fSJeff Roberson } 23393f289c3fSJeff Roberson } 2340f1b18a66SKyle Evans if (DOMAINSET_EMPTY(mask)) { 2341f1b18a66SKyle Evans error = EDEADLK; 2342f1b18a66SKyle Evans goto out; 2343f1b18a66SKyle Evans } 23443f289c3fSJeff Roberson DOMAINSET_COPY(mask, &domain.ds_mask); 23453f289c3fSJeff Roberson domain.ds_policy = policy; 23463f289c3fSJeff Roberson 234763cdd18eSMark Johnston /* 234863cdd18eSMark Johnston * Sanitize the provided mask. 234963cdd18eSMark Johnston */ 235063cdd18eSMark Johnston if (!DOMAINSET_SUBSET(&all_domains, &domain.ds_mask)) { 235163cdd18eSMark Johnston error = EINVAL; 235263cdd18eSMark Johnston goto out; 235363cdd18eSMark Johnston } 235463cdd18eSMark Johnston 23553f289c3fSJeff Roberson /* Translate preferred policy into a mask and fallback. */ 23563f289c3fSJeff Roberson if (policy == DOMAINSET_POLICY_PREFER) { 23573f289c3fSJeff Roberson /* Only support a single preferred domain. */ 235870d66bcfSEric van Gyzen if (DOMAINSET_COUNT(&domain.ds_mask) != 1) { 235970d66bcfSEric van Gyzen error = EINVAL; 236070d66bcfSEric van Gyzen goto out; 236170d66bcfSEric van Gyzen } 23623f289c3fSJeff Roberson domain.ds_prefer = DOMAINSET_FFS(&domain.ds_mask) - 1; 23633f289c3fSJeff Roberson /* This will be constrained by domainset_shadow(). */ 236463cdd18eSMark Johnston DOMAINSET_COPY(&all_domains, &domain.ds_mask); 23653f289c3fSJeff Roberson } 23663f289c3fSJeff Roberson 236730c5525bSAndrew Gallatin /* 236830c5525bSAndrew Gallatin * When given an impossible policy, fall back to interleaving 236963cdd18eSMark Johnston * across all domains. 237030c5525bSAndrew Gallatin */ 237130c5525bSAndrew Gallatin if (domainset_empty_vm(&domain)) 237229bb6c19SMark Johnston domainset_copy(domainset2, &domain); 237330c5525bSAndrew Gallatin 23743f289c3fSJeff Roberson switch (level) { 23753f289c3fSJeff Roberson case CPU_LEVEL_ROOT: 23763f289c3fSJeff Roberson case CPU_LEVEL_CPUSET: 23773f289c3fSJeff Roberson error = cpuset_which(which, id, &p, &ttd, &set); 23783f289c3fSJeff Roberson if (error) 23793f289c3fSJeff Roberson break; 23803f289c3fSJeff Roberson switch (which) { 23813f289c3fSJeff Roberson case CPU_WHICH_TID: 23823f289c3fSJeff Roberson case CPU_WHICH_PID: 23833f289c3fSJeff Roberson thread_lock(ttd); 23843f289c3fSJeff Roberson set = cpuset_ref(ttd->td_cpuset); 23853f289c3fSJeff Roberson thread_unlock(ttd); 23863f289c3fSJeff Roberson PROC_UNLOCK(p); 23873f289c3fSJeff Roberson break; 23883f289c3fSJeff Roberson case CPU_WHICH_CPUSET: 23893f289c3fSJeff Roberson case CPU_WHICH_JAIL: 23903f289c3fSJeff Roberson break; 23913f289c3fSJeff Roberson case CPU_WHICH_IRQ: 23923f289c3fSJeff Roberson case CPU_WHICH_INTRHANDLER: 23933f289c3fSJeff Roberson case CPU_WHICH_ITHREAD: 23943f289c3fSJeff Roberson case CPU_WHICH_DOMAIN: 23953f289c3fSJeff Roberson error = EINVAL; 23963f289c3fSJeff Roberson goto out; 23973f289c3fSJeff Roberson } 23983f289c3fSJeff Roberson if (level == CPU_LEVEL_ROOT) 23993f289c3fSJeff Roberson nset = cpuset_refroot(set); 24003f289c3fSJeff Roberson else 24013f289c3fSJeff Roberson nset = cpuset_refbase(set); 24023f289c3fSJeff Roberson error = cpuset_modify_domain(nset, &domain); 24033f289c3fSJeff Roberson cpuset_rel(nset); 24043f289c3fSJeff Roberson cpuset_rel(set); 24053f289c3fSJeff Roberson break; 24063f289c3fSJeff Roberson case CPU_LEVEL_WHICH: 24073f289c3fSJeff Roberson switch (which) { 24083f289c3fSJeff Roberson case CPU_WHICH_TID: 24093f289c3fSJeff Roberson error = _cpuset_setthread(id, NULL, &domain); 24103f289c3fSJeff Roberson break; 24113f289c3fSJeff Roberson case CPU_WHICH_PID: 2412d431dea5SKyle Evans error = cpuset_setproc(id, NULL, NULL, &domain, false); 24133f289c3fSJeff Roberson break; 24143f289c3fSJeff Roberson case CPU_WHICH_CPUSET: 24153f289c3fSJeff Roberson case CPU_WHICH_JAIL: 24163f289c3fSJeff Roberson error = cpuset_which(which, id, &p, &ttd, &set); 24173f289c3fSJeff Roberson if (error == 0) { 24183f289c3fSJeff Roberson error = cpuset_modify_domain(set, &domain); 24193f289c3fSJeff Roberson cpuset_rel(set); 24203f289c3fSJeff Roberson } 24213f289c3fSJeff Roberson break; 24223f289c3fSJeff Roberson case CPU_WHICH_IRQ: 24233f289c3fSJeff Roberson case CPU_WHICH_INTRHANDLER: 24243f289c3fSJeff Roberson case CPU_WHICH_ITHREAD: 24253f289c3fSJeff Roberson default: 24263f289c3fSJeff Roberson error = EINVAL; 24273f289c3fSJeff Roberson break; 24283f289c3fSJeff Roberson } 24293f289c3fSJeff Roberson break; 24303f289c3fSJeff Roberson default: 24313f289c3fSJeff Roberson error = EINVAL; 24323f289c3fSJeff Roberson break; 24333f289c3fSJeff Roberson } 24343f289c3fSJeff Roberson out: 24353f289c3fSJeff Roberson free(mask, M_TEMP); 24363f289c3fSJeff Roberson return (error); 24373f289c3fSJeff Roberson } 24383f289c3fSJeff Roberson 24393f289c3fSJeff Roberson #ifdef DDB 2440e5818a53SJeff Roberson 24413f289c3fSJeff Roberson static void 24423f289c3fSJeff Roberson ddb_display_bitset(const struct bitset *set, int size) 24433f289c3fSJeff Roberson { 24443f289c3fSJeff Roberson int bit, once; 24453f289c3fSJeff Roberson 24463f289c3fSJeff Roberson for (once = 0, bit = 0; bit < size; bit++) { 24473f289c3fSJeff Roberson if (CPU_ISSET(bit, set)) { 2448dea0ed66SBjoern A. Zeeb if (once == 0) { 24493f289c3fSJeff Roberson db_printf("%d", bit); 2450dea0ed66SBjoern A. Zeeb once = 1; 2451dea0ed66SBjoern A. Zeeb } else 24523f289c3fSJeff Roberson db_printf(",%d", bit); 2453dea0ed66SBjoern A. Zeeb } 2454dea0ed66SBjoern A. Zeeb } 2455cd32bd7aSJohn Baldwin if (once == 0) 2456cd32bd7aSJohn Baldwin db_printf("<none>"); 2457cd32bd7aSJohn Baldwin } 2458cd32bd7aSJohn Baldwin 24593f289c3fSJeff Roberson void 24603f289c3fSJeff Roberson ddb_display_cpuset(const cpuset_t *set) 24613f289c3fSJeff Roberson { 24623f289c3fSJeff Roberson ddb_display_bitset((const struct bitset *)set, CPU_SETSIZE); 24633f289c3fSJeff Roberson } 24643f289c3fSJeff Roberson 24653f289c3fSJeff Roberson static void 24663f289c3fSJeff Roberson ddb_display_domainset(const domainset_t *set) 24673f289c3fSJeff Roberson { 24683f289c3fSJeff Roberson ddb_display_bitset((const struct bitset *)set, DOMAINSET_SETSIZE); 24693f289c3fSJeff Roberson } 24703f289c3fSJeff Roberson 2471cd32bd7aSJohn Baldwin DB_SHOW_COMMAND(cpusets, db_show_cpusets) 2472cd32bd7aSJohn Baldwin { 2473cd32bd7aSJohn Baldwin struct cpuset *set; 2474cd32bd7aSJohn Baldwin 2475cd32bd7aSJohn Baldwin LIST_FOREACH(set, &cpuset_ids, cs_link) { 2476cd32bd7aSJohn Baldwin db_printf("set=%p id=%-6u ref=%-6d flags=0x%04x parent id=%d\n", 24771a7bb896SMateusz Guzik set, set->cs_id, refcount_load(&set->cs_ref), set->cs_flags, 2478cd32bd7aSJohn Baldwin (set->cs_parent != NULL) ? set->cs_parent->cs_id : 0); 24793f289c3fSJeff Roberson db_printf(" cpu mask="); 2480cd32bd7aSJohn Baldwin ddb_display_cpuset(&set->cs_mask); 2481dea0ed66SBjoern A. Zeeb db_printf("\n"); 24823f289c3fSJeff Roberson db_printf(" domain policy %d prefer %d mask=", 24833f289c3fSJeff Roberson set->cs_domain->ds_policy, set->cs_domain->ds_prefer); 24843f289c3fSJeff Roberson ddb_display_domainset(&set->cs_domain->ds_mask); 24853f289c3fSJeff Roberson db_printf("\n"); 2486dea0ed66SBjoern A. Zeeb if (db_pager_quit) 2487dea0ed66SBjoern A. Zeeb break; 2488dea0ed66SBjoern A. Zeeb } 2489dea0ed66SBjoern A. Zeeb } 24903f289c3fSJeff Roberson 24913f289c3fSJeff Roberson DB_SHOW_COMMAND(domainsets, db_show_domainsets) 24923f289c3fSJeff Roberson { 24933f289c3fSJeff Roberson struct domainset *set; 24943f289c3fSJeff Roberson 24953f289c3fSJeff Roberson LIST_FOREACH(set, &cpuset_domains, ds_link) { 2496e5818a53SJeff Roberson db_printf("set=%p policy %d prefer %d cnt %d\n", 2497e5818a53SJeff Roberson set, set->ds_policy, set->ds_prefer, set->ds_cnt); 24983f289c3fSJeff Roberson db_printf(" mask ="); 24993f289c3fSJeff Roberson ddb_display_domainset(&set->ds_mask); 25003f289c3fSJeff Roberson db_printf("\n"); 25013f289c3fSJeff Roberson } 25023f289c3fSJeff Roberson } 2503dea0ed66SBjoern A. Zeeb #endif /* DDB */ 2504