1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_CPUMASK_H 3 #define __LINUX_CPUMASK_H 4 5 /* 6 * Cpumasks provide a bitmap suitable for representing the 7 * set of CPU's in a system, one bit position per CPU number. In general, 8 * only nr_cpu_ids (<= NR_CPUS) bits are valid. 9 */ 10 #include <linux/kernel.h> 11 #include <linux/threads.h> 12 #include <linux/bitmap.h> 13 #include <linux/atomic.h> 14 #include <linux/bug.h> 15 16 /* Don't assign or return these: may not be this big! */ 17 typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t; 18 19 /** 20 * cpumask_bits - get the bits in a cpumask 21 * @maskp: the struct cpumask * 22 * 23 * You should only assume nr_cpu_ids bits of this mask are valid. This is 24 * a macro so it's const-correct. 25 */ 26 #define cpumask_bits(maskp) ((maskp)->bits) 27 28 /** 29 * cpumask_pr_args - printf args to output a cpumask 30 * @maskp: cpumask to be printed 31 * 32 * Can be used to provide arguments for '%*pb[l]' when printing a cpumask. 33 */ 34 #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp) 35 36 #if NR_CPUS == 1 37 #define nr_cpu_ids 1U 38 #else 39 extern unsigned int nr_cpu_ids; 40 #endif 41 42 #ifdef CONFIG_CPUMASK_OFFSTACK 43 /* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also, 44 * not all bits may be allocated. */ 45 #define nr_cpumask_bits nr_cpu_ids 46 #else 47 #define nr_cpumask_bits ((unsigned int)NR_CPUS) 48 #endif 49 50 /* 51 * The following particular system cpumasks and operations manage 52 * possible, present, active and online cpus. 53 * 54 * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable 55 * cpu_present_mask - has bit 'cpu' set iff cpu is populated 56 * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler 57 * cpu_active_mask - has bit 'cpu' set iff cpu available to migration 58 * 59 * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online. 60 * 61 * The cpu_possible_mask is fixed at boot time, as the set of CPU id's 62 * that it is possible might ever be plugged in at anytime during the 63 * life of that system boot. The cpu_present_mask is dynamic(*), 64 * representing which CPUs are currently plugged in. And 65 * cpu_online_mask is the dynamic subset of cpu_present_mask, 66 * indicating those CPUs available for scheduling. 67 * 68 * If HOTPLUG is enabled, then cpu_possible_mask is forced to have 69 * all NR_CPUS bits set, otherwise it is just the set of CPUs that 70 * ACPI reports present at boot. 71 * 72 * If HOTPLUG is enabled, then cpu_present_mask varies dynamically, 73 * depending on what ACPI reports as currently plugged in, otherwise 74 * cpu_present_mask is just a copy of cpu_possible_mask. 75 * 76 * (*) Well, cpu_present_mask is dynamic in the hotplug case. If not 77 * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot. 78 * 79 * Subtleties: 80 * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode 81 * assumption that their single CPU is online. The UP 82 * cpu_{online,possible,present}_masks are placebos. Changing them 83 * will have no useful affect on the following num_*_cpus() 84 * and cpu_*() macros in the UP case. This ugliness is a UP 85 * optimization - don't waste any instructions or memory references 86 * asking if you're online or how many CPUs there are if there is 87 * only one CPU. 88 */ 89 90 extern struct cpumask __cpu_possible_mask; 91 extern struct cpumask __cpu_online_mask; 92 extern struct cpumask __cpu_present_mask; 93 extern struct cpumask __cpu_active_mask; 94 #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask) 95 #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask) 96 #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) 97 #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) 98 99 extern atomic_t __num_online_cpus; 100 101 #if NR_CPUS > 1 102 /** 103 * num_online_cpus() - Read the number of online CPUs 104 * 105 * Despite the fact that __num_online_cpus is of type atomic_t, this 106 * interface gives only a momentary snapshot and is not protected against 107 * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held 108 * region. 109 */ 110 static inline unsigned int num_online_cpus(void) 111 { 112 return atomic_read(&__num_online_cpus); 113 } 114 #define num_possible_cpus() cpumask_weight(cpu_possible_mask) 115 #define num_present_cpus() cpumask_weight(cpu_present_mask) 116 #define num_active_cpus() cpumask_weight(cpu_active_mask) 117 #define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask) 118 #define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask) 119 #define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask) 120 #define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask) 121 #else 122 #define num_online_cpus() 1U 123 #define num_possible_cpus() 1U 124 #define num_present_cpus() 1U 125 #define num_active_cpus() 1U 126 #define cpu_online(cpu) ((cpu) == 0) 127 #define cpu_possible(cpu) ((cpu) == 0) 128 #define cpu_present(cpu) ((cpu) == 0) 129 #define cpu_active(cpu) ((cpu) == 0) 130 #endif 131 132 extern cpumask_t cpus_booted_once_mask; 133 134 static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits) 135 { 136 #ifdef CONFIG_DEBUG_PER_CPU_MAPS 137 WARN_ON_ONCE(cpu >= bits); 138 #endif /* CONFIG_DEBUG_PER_CPU_MAPS */ 139 } 140 141 /* verify cpu argument to cpumask_* operators */ 142 static inline unsigned int cpumask_check(unsigned int cpu) 143 { 144 cpu_max_bits_warn(cpu, nr_cpumask_bits); 145 return cpu; 146 } 147 148 #if NR_CPUS == 1 149 /* Uniprocessor. Assume all masks are "1". */ 150 static inline unsigned int cpumask_first(const struct cpumask *srcp) 151 { 152 return 0; 153 } 154 155 static inline unsigned int cpumask_last(const struct cpumask *srcp) 156 { 157 return 0; 158 } 159 160 /* Valid inputs for n are -1 and 0. */ 161 static inline unsigned int cpumask_next(int n, const struct cpumask *srcp) 162 { 163 return n+1; 164 } 165 166 static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) 167 { 168 return n+1; 169 } 170 171 static inline unsigned int cpumask_next_and(int n, 172 const struct cpumask *srcp, 173 const struct cpumask *andp) 174 { 175 return n+1; 176 } 177 178 static inline unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, 179 int start, bool wrap) 180 { 181 /* cpu0 unless stop condition, wrap and at cpu0, then nr_cpumask_bits */ 182 return (wrap && n == 0); 183 } 184 185 /* cpu must be a valid cpu, ie 0, so there's no other choice. */ 186 static inline unsigned int cpumask_any_but(const struct cpumask *mask, 187 unsigned int cpu) 188 { 189 return 1; 190 } 191 192 static inline unsigned int cpumask_local_spread(unsigned int i, int node) 193 { 194 return 0; 195 } 196 197 #define for_each_cpu(cpu, mask) \ 198 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) 199 #define for_each_cpu_not(cpu, mask) \ 200 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) 201 #define for_each_cpu_wrap(cpu, mask, start) \ 202 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)(start)) 203 #define for_each_cpu_and(cpu, mask1, mask2) \ 204 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask1, (void)mask2) 205 #else 206 /** 207 * cpumask_first - get the first cpu in a cpumask 208 * @srcp: the cpumask pointer 209 * 210 * Returns >= nr_cpu_ids if no cpus set. 211 */ 212 static inline unsigned int cpumask_first(const struct cpumask *srcp) 213 { 214 return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits); 215 } 216 217 /** 218 * cpumask_last - get the last CPU in a cpumask 219 * @srcp: - the cpumask pointer 220 * 221 * Returns >= nr_cpumask_bits if no CPUs set. 222 */ 223 static inline unsigned int cpumask_last(const struct cpumask *srcp) 224 { 225 return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits); 226 } 227 228 unsigned int cpumask_next(int n, const struct cpumask *srcp); 229 230 /** 231 * cpumask_next_zero - get the next unset cpu in a cpumask 232 * @n: the cpu prior to the place to search (ie. return will be > @n) 233 * @srcp: the cpumask pointer 234 * 235 * Returns >= nr_cpu_ids if no further cpus unset. 236 */ 237 static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) 238 { 239 /* -1 is a legal arg here. */ 240 if (n != -1) 241 cpumask_check(n); 242 return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1); 243 } 244 245 int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *); 246 int cpumask_any_but(const struct cpumask *mask, unsigned int cpu); 247 unsigned int cpumask_local_spread(unsigned int i, int node); 248 249 /** 250 * for_each_cpu - iterate over every cpu in a mask 251 * @cpu: the (optionally unsigned) integer iterator 252 * @mask: the cpumask pointer 253 * 254 * After the loop, cpu is >= nr_cpu_ids. 255 */ 256 #define for_each_cpu(cpu, mask) \ 257 for ((cpu) = -1; \ 258 (cpu) = cpumask_next((cpu), (mask)), \ 259 (cpu) < nr_cpu_ids;) 260 261 /** 262 * for_each_cpu_not - iterate over every cpu in a complemented mask 263 * @cpu: the (optionally unsigned) integer iterator 264 * @mask: the cpumask pointer 265 * 266 * After the loop, cpu is >= nr_cpu_ids. 267 */ 268 #define for_each_cpu_not(cpu, mask) \ 269 for ((cpu) = -1; \ 270 (cpu) = cpumask_next_zero((cpu), (mask)), \ 271 (cpu) < nr_cpu_ids;) 272 273 extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap); 274 275 /** 276 * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location 277 * @cpu: the (optionally unsigned) integer iterator 278 * @mask: the cpumask poiter 279 * @start: the start location 280 * 281 * The implementation does not assume any bit in @mask is set (including @start). 282 * 283 * After the loop, cpu is >= nr_cpu_ids. 284 */ 285 #define for_each_cpu_wrap(cpu, mask, start) \ 286 for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \ 287 (cpu) < nr_cpumask_bits; \ 288 (cpu) = cpumask_next_wrap((cpu), (mask), (start), true)) 289 290 /** 291 * for_each_cpu_and - iterate over every cpu in both masks 292 * @cpu: the (optionally unsigned) integer iterator 293 * @mask1: the first cpumask pointer 294 * @mask2: the second cpumask pointer 295 * 296 * This saves a temporary CPU mask in many places. It is equivalent to: 297 * struct cpumask tmp; 298 * cpumask_and(&tmp, &mask1, &mask2); 299 * for_each_cpu(cpu, &tmp) 300 * ... 301 * 302 * After the loop, cpu is >= nr_cpu_ids. 303 */ 304 #define for_each_cpu_and(cpu, mask1, mask2) \ 305 for ((cpu) = -1; \ 306 (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \ 307 (cpu) < nr_cpu_ids;) 308 #endif /* SMP */ 309 310 #define CPU_BITS_NONE \ 311 { \ 312 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \ 313 } 314 315 #define CPU_BITS_CPU0 \ 316 { \ 317 [0] = 1UL \ 318 } 319 320 /** 321 * cpumask_set_cpu - set a cpu in a cpumask 322 * @cpu: cpu number (< nr_cpu_ids) 323 * @dstp: the cpumask pointer 324 */ 325 static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) 326 { 327 set_bit(cpumask_check(cpu), cpumask_bits(dstp)); 328 } 329 330 static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) 331 { 332 __set_bit(cpumask_check(cpu), cpumask_bits(dstp)); 333 } 334 335 336 /** 337 * cpumask_clear_cpu - clear a cpu in a cpumask 338 * @cpu: cpu number (< nr_cpu_ids) 339 * @dstp: the cpumask pointer 340 */ 341 static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp) 342 { 343 clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); 344 } 345 346 static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) 347 { 348 __clear_bit(cpumask_check(cpu), cpumask_bits(dstp)); 349 } 350 351 /** 352 * cpumask_test_cpu - test for a cpu in a cpumask 353 * @cpu: cpu number (< nr_cpu_ids) 354 * @cpumask: the cpumask pointer 355 * 356 * Returns 1 if @cpu is set in @cpumask, else returns 0 357 */ 358 static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask) 359 { 360 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); 361 } 362 363 /** 364 * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask 365 * @cpu: cpu number (< nr_cpu_ids) 366 * @cpumask: the cpumask pointer 367 * 368 * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0 369 * 370 * test_and_set_bit wrapper for cpumasks. 371 */ 372 static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) 373 { 374 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask)); 375 } 376 377 /** 378 * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask 379 * @cpu: cpu number (< nr_cpu_ids) 380 * @cpumask: the cpumask pointer 381 * 382 * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0 383 * 384 * test_and_clear_bit wrapper for cpumasks. 385 */ 386 static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) 387 { 388 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask)); 389 } 390 391 /** 392 * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask 393 * @dstp: the cpumask pointer 394 */ 395 static inline void cpumask_setall(struct cpumask *dstp) 396 { 397 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits); 398 } 399 400 /** 401 * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask 402 * @dstp: the cpumask pointer 403 */ 404 static inline void cpumask_clear(struct cpumask *dstp) 405 { 406 bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits); 407 } 408 409 /** 410 * cpumask_and - *dstp = *src1p & *src2p 411 * @dstp: the cpumask result 412 * @src1p: the first input 413 * @src2p: the second input 414 * 415 * If *@dstp is empty, returns 0, else returns 1 416 */ 417 static inline int cpumask_and(struct cpumask *dstp, 418 const struct cpumask *src1p, 419 const struct cpumask *src2p) 420 { 421 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p), 422 cpumask_bits(src2p), nr_cpumask_bits); 423 } 424 425 /** 426 * cpumask_or - *dstp = *src1p | *src2p 427 * @dstp: the cpumask result 428 * @src1p: the first input 429 * @src2p: the second input 430 */ 431 static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, 432 const struct cpumask *src2p) 433 { 434 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p), 435 cpumask_bits(src2p), nr_cpumask_bits); 436 } 437 438 /** 439 * cpumask_xor - *dstp = *src1p ^ *src2p 440 * @dstp: the cpumask result 441 * @src1p: the first input 442 * @src2p: the second input 443 */ 444 static inline void cpumask_xor(struct cpumask *dstp, 445 const struct cpumask *src1p, 446 const struct cpumask *src2p) 447 { 448 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p), 449 cpumask_bits(src2p), nr_cpumask_bits); 450 } 451 452 /** 453 * cpumask_andnot - *dstp = *src1p & ~*src2p 454 * @dstp: the cpumask result 455 * @src1p: the first input 456 * @src2p: the second input 457 * 458 * If *@dstp is empty, returns 0, else returns 1 459 */ 460 static inline int cpumask_andnot(struct cpumask *dstp, 461 const struct cpumask *src1p, 462 const struct cpumask *src2p) 463 { 464 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p), 465 cpumask_bits(src2p), nr_cpumask_bits); 466 } 467 468 /** 469 * cpumask_complement - *dstp = ~*srcp 470 * @dstp: the cpumask result 471 * @srcp: the input to invert 472 */ 473 static inline void cpumask_complement(struct cpumask *dstp, 474 const struct cpumask *srcp) 475 { 476 bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp), 477 nr_cpumask_bits); 478 } 479 480 /** 481 * cpumask_equal - *src1p == *src2p 482 * @src1p: the first input 483 * @src2p: the second input 484 */ 485 static inline bool cpumask_equal(const struct cpumask *src1p, 486 const struct cpumask *src2p) 487 { 488 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p), 489 nr_cpumask_bits); 490 } 491 492 /** 493 * cpumask_or_equal - *src1p | *src2p == *src3p 494 * @src1p: the first input 495 * @src2p: the second input 496 * @src3p: the third input 497 */ 498 static inline bool cpumask_or_equal(const struct cpumask *src1p, 499 const struct cpumask *src2p, 500 const struct cpumask *src3p) 501 { 502 return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p), 503 cpumask_bits(src3p), nr_cpumask_bits); 504 } 505 506 /** 507 * cpumask_intersects - (*src1p & *src2p) != 0 508 * @src1p: the first input 509 * @src2p: the second input 510 */ 511 static inline bool cpumask_intersects(const struct cpumask *src1p, 512 const struct cpumask *src2p) 513 { 514 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p), 515 nr_cpumask_bits); 516 } 517 518 /** 519 * cpumask_subset - (*src1p & ~*src2p) == 0 520 * @src1p: the first input 521 * @src2p: the second input 522 * 523 * Returns 1 if *@src1p is a subset of *@src2p, else returns 0 524 */ 525 static inline int cpumask_subset(const struct cpumask *src1p, 526 const struct cpumask *src2p) 527 { 528 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p), 529 nr_cpumask_bits); 530 } 531 532 /** 533 * cpumask_empty - *srcp == 0 534 * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear. 535 */ 536 static inline bool cpumask_empty(const struct cpumask *srcp) 537 { 538 return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits); 539 } 540 541 /** 542 * cpumask_full - *srcp == 0xFFFFFFFF... 543 * @srcp: the cpumask to that all cpus < nr_cpu_ids are set. 544 */ 545 static inline bool cpumask_full(const struct cpumask *srcp) 546 { 547 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits); 548 } 549 550 /** 551 * cpumask_weight - Count of bits in *srcp 552 * @srcp: the cpumask to count bits (< nr_cpu_ids) in. 553 */ 554 static inline unsigned int cpumask_weight(const struct cpumask *srcp) 555 { 556 return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits); 557 } 558 559 /** 560 * cpumask_shift_right - *dstp = *srcp >> n 561 * @dstp: the cpumask result 562 * @srcp: the input to shift 563 * @n: the number of bits to shift by 564 */ 565 static inline void cpumask_shift_right(struct cpumask *dstp, 566 const struct cpumask *srcp, int n) 567 { 568 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n, 569 nr_cpumask_bits); 570 } 571 572 /** 573 * cpumask_shift_left - *dstp = *srcp << n 574 * @dstp: the cpumask result 575 * @srcp: the input to shift 576 * @n: the number of bits to shift by 577 */ 578 static inline void cpumask_shift_left(struct cpumask *dstp, 579 const struct cpumask *srcp, int n) 580 { 581 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n, 582 nr_cpumask_bits); 583 } 584 585 /** 586 * cpumask_copy - *dstp = *srcp 587 * @dstp: the result 588 * @srcp: the input cpumask 589 */ 590 static inline void cpumask_copy(struct cpumask *dstp, 591 const struct cpumask *srcp) 592 { 593 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits); 594 } 595 596 /** 597 * cpumask_any - pick a "random" cpu from *srcp 598 * @srcp: the input cpumask 599 * 600 * Returns >= nr_cpu_ids if no cpus set. 601 */ 602 #define cpumask_any(srcp) cpumask_first(srcp) 603 604 /** 605 * cpumask_first_and - return the first cpu from *srcp1 & *srcp2 606 * @src1p: the first input 607 * @src2p: the second input 608 * 609 * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and(). 610 */ 611 #define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p)) 612 613 /** 614 * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2 615 * @mask1: the first input cpumask 616 * @mask2: the second input cpumask 617 * 618 * Returns >= nr_cpu_ids if no cpus set. 619 */ 620 #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2)) 621 622 /** 623 * cpumask_of - the cpumask containing just a given cpu 624 * @cpu: the cpu (<= nr_cpu_ids) 625 */ 626 #define cpumask_of(cpu) (get_cpu_mask(cpu)) 627 628 /** 629 * cpumask_parse_user - extract a cpumask from a user string 630 * @buf: the buffer to extract from 631 * @len: the length of the buffer 632 * @dstp: the cpumask to set. 633 * 634 * Returns -errno, or 0 for success. 635 */ 636 static inline int cpumask_parse_user(const char __user *buf, int len, 637 struct cpumask *dstp) 638 { 639 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits); 640 } 641 642 /** 643 * cpumask_parselist_user - extract a cpumask from a user string 644 * @buf: the buffer to extract from 645 * @len: the length of the buffer 646 * @dstp: the cpumask to set. 647 * 648 * Returns -errno, or 0 for success. 649 */ 650 static inline int cpumask_parselist_user(const char __user *buf, int len, 651 struct cpumask *dstp) 652 { 653 return bitmap_parselist_user(buf, len, cpumask_bits(dstp), 654 nr_cpumask_bits); 655 } 656 657 /** 658 * cpumask_parse - extract a cpumask from a string 659 * @buf: the buffer to extract from 660 * @dstp: the cpumask to set. 661 * 662 * Returns -errno, or 0 for success. 663 */ 664 static inline int cpumask_parse(const char *buf, struct cpumask *dstp) 665 { 666 return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits); 667 } 668 669 /** 670 * cpulist_parse - extract a cpumask from a user string of ranges 671 * @buf: the buffer to extract from 672 * @dstp: the cpumask to set. 673 * 674 * Returns -errno, or 0 for success. 675 */ 676 static inline int cpulist_parse(const char *buf, struct cpumask *dstp) 677 { 678 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits); 679 } 680 681 /** 682 * cpumask_size - size to allocate for a 'struct cpumask' in bytes 683 */ 684 static inline unsigned int cpumask_size(void) 685 { 686 return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long); 687 } 688 689 /* 690 * cpumask_var_t: struct cpumask for stack usage. 691 * 692 * Oh, the wicked games we play! In order to make kernel coding a 693 * little more difficult, we typedef cpumask_var_t to an array or a 694 * pointer: doing &mask on an array is a noop, so it still works. 695 * 696 * ie. 697 * cpumask_var_t tmpmask; 698 * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL)) 699 * return -ENOMEM; 700 * 701 * ... use 'tmpmask' like a normal struct cpumask * ... 702 * 703 * free_cpumask_var(tmpmask); 704 * 705 * 706 * However, one notable exception is there. alloc_cpumask_var() allocates 707 * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has 708 * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t. 709 * 710 * cpumask_var_t tmpmask; 711 * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL)) 712 * return -ENOMEM; 713 * 714 * var = *tmpmask; 715 * 716 * This code makes NR_CPUS length memcopy and brings to a memory corruption. 717 * cpumask_copy() provide safe copy functionality. 718 * 719 * Note that there is another evil here: If you define a cpumask_var_t 720 * as a percpu variable then the way to obtain the address of the cpumask 721 * structure differently influences what this_cpu_* operation needs to be 722 * used. Please use this_cpu_cpumask_var_t in those cases. The direct use 723 * of this_cpu_ptr() or this_cpu_read() will lead to failures when the 724 * other type of cpumask_var_t implementation is configured. 725 * 726 * Please also note that __cpumask_var_read_mostly can be used to declare 727 * a cpumask_var_t variable itself (not its content) as read mostly. 728 */ 729 #ifdef CONFIG_CPUMASK_OFFSTACK 730 typedef struct cpumask *cpumask_var_t; 731 732 #define this_cpu_cpumask_var_ptr(x) this_cpu_read(x) 733 #define __cpumask_var_read_mostly __read_mostly 734 735 bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); 736 bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags); 737 bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); 738 bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags); 739 void alloc_bootmem_cpumask_var(cpumask_var_t *mask); 740 void free_cpumask_var(cpumask_var_t mask); 741 void free_bootmem_cpumask_var(cpumask_var_t mask); 742 743 static inline bool cpumask_available(cpumask_var_t mask) 744 { 745 return mask != NULL; 746 } 747 748 #else 749 typedef struct cpumask cpumask_var_t[1]; 750 751 #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x) 752 #define __cpumask_var_read_mostly 753 754 static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 755 { 756 return true; 757 } 758 759 static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, 760 int node) 761 { 762 return true; 763 } 764 765 static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) 766 { 767 cpumask_clear(*mask); 768 return true; 769 } 770 771 static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, 772 int node) 773 { 774 cpumask_clear(*mask); 775 return true; 776 } 777 778 static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask) 779 { 780 } 781 782 static inline void free_cpumask_var(cpumask_var_t mask) 783 { 784 } 785 786 static inline void free_bootmem_cpumask_var(cpumask_var_t mask) 787 { 788 } 789 790 static inline bool cpumask_available(cpumask_var_t mask) 791 { 792 return true; 793 } 794 #endif /* CONFIG_CPUMASK_OFFSTACK */ 795 796 /* It's common to want to use cpu_all_mask in struct member initializers, 797 * so it has to refer to an address rather than a pointer. */ 798 extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS); 799 #define cpu_all_mask to_cpumask(cpu_all_bits) 800 801 /* First bits of cpu_bit_bitmap are in fact unset. */ 802 #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0]) 803 804 #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask) 805 #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask) 806 #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask) 807 808 /* Wrappers for arch boot code to manipulate normally-constant masks */ 809 void init_cpu_present(const struct cpumask *src); 810 void init_cpu_possible(const struct cpumask *src); 811 void init_cpu_online(const struct cpumask *src); 812 813 static inline void reset_cpu_possible_mask(void) 814 { 815 bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS); 816 } 817 818 static inline void 819 set_cpu_possible(unsigned int cpu, bool possible) 820 { 821 if (possible) 822 cpumask_set_cpu(cpu, &__cpu_possible_mask); 823 else 824 cpumask_clear_cpu(cpu, &__cpu_possible_mask); 825 } 826 827 static inline void 828 set_cpu_present(unsigned int cpu, bool present) 829 { 830 if (present) 831 cpumask_set_cpu(cpu, &__cpu_present_mask); 832 else 833 cpumask_clear_cpu(cpu, &__cpu_present_mask); 834 } 835 836 void set_cpu_online(unsigned int cpu, bool online); 837 838 static inline void 839 set_cpu_active(unsigned int cpu, bool active) 840 { 841 if (active) 842 cpumask_set_cpu(cpu, &__cpu_active_mask); 843 else 844 cpumask_clear_cpu(cpu, &__cpu_active_mask); 845 } 846 847 848 /** 849 * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask * 850 * @bitmap: the bitmap 851 * 852 * There are a few places where cpumask_var_t isn't appropriate and 853 * static cpumasks must be used (eg. very early boot), yet we don't 854 * expose the definition of 'struct cpumask'. 855 * 856 * This does the conversion, and can be used as a constant initializer. 857 */ 858 #define to_cpumask(bitmap) \ 859 ((struct cpumask *)(1 ? (bitmap) \ 860 : (void *)sizeof(__check_is_bitmap(bitmap)))) 861 862 static inline int __check_is_bitmap(const unsigned long *bitmap) 863 { 864 return 1; 865 } 866 867 /* 868 * Special-case data structure for "single bit set only" constant CPU masks. 869 * 870 * We pre-generate all the 64 (or 32) possible bit positions, with enough 871 * padding to the left and the right, and return the constant pointer 872 * appropriately offset. 873 */ 874 extern const unsigned long 875 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)]; 876 877 static inline const struct cpumask *get_cpu_mask(unsigned int cpu) 878 { 879 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG]; 880 p -= cpu / BITS_PER_LONG; 881 return to_cpumask(p); 882 } 883 884 #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu)) 885 886 #if NR_CPUS <= BITS_PER_LONG 887 #define CPU_BITS_ALL \ 888 { \ 889 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 890 } 891 892 #else /* NR_CPUS > BITS_PER_LONG */ 893 894 #define CPU_BITS_ALL \ 895 { \ 896 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \ 897 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 898 } 899 #endif /* NR_CPUS > BITS_PER_LONG */ 900 901 /** 902 * cpumap_print_to_pagebuf - copies the cpumask into the buffer either 903 * as comma-separated list of cpus or hex values of cpumask 904 * @list: indicates whether the cpumap must be list 905 * @mask: the cpumask to copy 906 * @buf: the buffer to copy into 907 * 908 * Returns the length of the (null-terminated) @buf string, zero if 909 * nothing is copied. 910 */ 911 static inline ssize_t 912 cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask) 913 { 914 return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask), 915 nr_cpu_ids); 916 } 917 918 #if NR_CPUS <= BITS_PER_LONG 919 #define CPU_MASK_ALL \ 920 (cpumask_t) { { \ 921 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 922 } } 923 #else 924 #define CPU_MASK_ALL \ 925 (cpumask_t) { { \ 926 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \ 927 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \ 928 } } 929 #endif /* NR_CPUS > BITS_PER_LONG */ 930 931 #define CPU_MASK_NONE \ 932 (cpumask_t) { { \ 933 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \ 934 } } 935 936 #define CPU_MASK_CPU0 \ 937 (cpumask_t) { { \ 938 [0] = 1UL \ 939 } } 940 941 #endif /* __LINUX_CPUMASK_H */ 942