xref: /freebsd/share/man/man9/atomic.9 (revision dad64f0e)
1.\" Copyright (c) 2000-2001 John H. Baldwin <jhb@FreeBSD.org>
2.\"
3.\" Redistribution and use in source and binary forms, with or without
4.\" modification, are permitted provided that the following conditions
5.\" are met:
6.\" 1. Redistributions of source code must retain the above copyright
7.\"    notice, this list of conditions and the following disclaimer.
8.\" 2. Redistributions in binary form must reproduce the above copyright
9.\"    notice, this list of conditions and the following disclaimer in the
10.\"    documentation and/or other materials provided with the distribution.
11.\"
12.\" THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY EXPRESS OR
13.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
14.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
15.\" IN NO EVENT SHALL THE DEVELOPERS BE LIABLE FOR ANY DIRECT, INDIRECT,
16.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
17.\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
18.\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
19.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
20.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
21.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
22.\"
23.\" $FreeBSD$
24.\"
25.Dd January 16, 2023
26.Dt ATOMIC 9
27.Os
28.Sh NAME
29.Nm atomic_add ,
30.Nm atomic_clear ,
31.Nm atomic_cmpset ,
32.Nm atomic_fcmpset ,
33.Nm atomic_fetchadd ,
34.Nm atomic_interrupt_fence ,
35.Nm atomic_load ,
36.Nm atomic_readandclear ,
37.Nm atomic_set ,
38.Nm atomic_subtract ,
39.Nm atomic_store ,
40.Nm atomic_thread_fence
41.Nd atomic operations
42.Sh SYNOPSIS
43.In machine/atomic.h
44.Ft void
45.Fn atomic_add_[acq_|rel_]<type> "volatile <type> *p" "<type> v"
46.Ft void
47.Fn atomic_clear_[acq_|rel_]<type> "volatile <type> *p" "<type> v"
48.Ft int
49.Fo atomic_cmpset_[acq_|rel_]<type>
50.Fa "volatile <type> *dst"
51.Fa "<type> old"
52.Fa "<type> new"
53.Fc
54.Ft int
55.Fo atomic_fcmpset_[acq_|rel_]<type>
56.Fa "volatile <type> *dst"
57.Fa "<type> *old"
58.Fa "<type> new"
59.Fc
60.Ft <type>
61.Fn atomic_fetchadd_<type> "volatile <type> *p" "<type> v"
62.Ft void
63.Fn atomic_interrupt_fence "void"
64.Ft <type>
65.Fn atomic_load_[acq_]<type> "volatile <type> *p"
66.Ft <type>
67.Fn atomic_readandclear_<type> "volatile <type> *p"
68.Ft void
69.Fn atomic_set_[acq_|rel_]<type> "volatile <type> *p" "<type> v"
70.Ft void
71.Fn atomic_subtract_[acq_|rel_]<type> "volatile <type> *p" "<type> v"
72.Ft void
73.Fn atomic_store_[rel_]<type> "volatile <type> *p" "<type> v"
74.Ft <type>
75.Fn atomic_swap_<type> "volatile <type> *p" "<type> v"
76.Ft int
77.Fn atomic_testandclear_<type> "volatile <type> *p" "u_int v"
78.Ft int
79.Fn atomic_testandset_<type> "volatile <type> *p" "u_int v"
80.Ft void
81.Fn atomic_thread_fence_[acq|acq_rel|rel|seq_cst] "void"
82.Sh DESCRIPTION
83Atomic operations are commonly used to implement reference counts and as
84building blocks for synchronization primitives, such as mutexes.
85.Pp
86All of these operations are performed
87.Em atomically
88across multiple threads and in the presence of interrupts, meaning that they
89are performed in an indivisible manner from the perspective of concurrently
90running threads and interrupt handlers.
91.Pp
92On all architectures supported by
93.Fx ,
94ordinary loads and stores of integers in cache-coherent memory are
95inherently atomic if the integer is naturally aligned and its size does not
96exceed the processor's word size.
97However, such loads and stores may be elided from the program by
98the compiler, whereas atomic operations are always performed.
99.Pp
100When atomic operations are performed on cache-coherent memory, all
101operations on the same location are totally ordered.
102.Pp
103When an atomic load is performed on a location in cache-coherent memory,
104it reads the entire value that was defined by the last atomic store to
105each byte of the location.
106An atomic load will never return a value out of thin air.
107When an atomic store is performed on a location, no other thread or
108interrupt handler will observe a
109.Em torn write ,
110or partial modification of the location.
111.Pp
112Except as noted below, the semantics of these operations are almost
113identical to the semantics of similarly named C11 atomic operations.
114.Ss Types
115Most atomic operations act upon a specific
116.Fa type .
117That type is indicated in the function name.
118In contrast to C11 atomic operations,
119.Fx Ns 's
120atomic operations are performed on ordinary integer types.
121The available types are:
122.Pp
123.Bl -tag -offset indent -width short -compact
124.It Li int
125unsigned integer
126.It Li long
127unsigned long integer
128.It Li ptr
129unsigned integer the size of a pointer
130.It Li 32
131unsigned 32-bit integer
132.It Li 64
133unsigned 64-bit integer
134.El
135.Pp
136For example, the function to atomically add two integers is called
137.Fn atomic_add_int .
138.Pp
139Certain architectures also provide operations for types smaller than
140.Dq Li int .
141.Pp
142.Bl -tag -offset indent -width short -compact
143.It Li char
144unsigned character
145.It Li short
146unsigned short integer
147.It Li 8
148unsigned 8-bit integer
149.It Li 16
150unsigned 16-bit integer
151.El
152.Pp
153These types must not be used in machine-independent code.
154.Ss Acquire and Release Operations
155By default, a thread's accesses to different memory locations might not be
156performed in
157.Em program order ,
158that is, the order in which the accesses appear in the source code.
159To optimize the program's execution, both the compiler and processor might
160reorder the thread's accesses.
161However, both ensure that their reordering of the accesses is not visible to
162the thread.
163Otherwise, the traditional memory model that is expected by single-threaded
164programs would be violated.
165Nonetheless, other threads in a multithreaded program, such as the
166.Fx
167kernel, might observe the reordering.
168Moreover, in some cases, such as the implementation of synchronization between
169threads, arbitrary reordering might result in the incorrect execution of the
170program.
171To constrain the reordering that both the compiler and processor might perform
172on a thread's accesses, a programmer can use atomic operations with
173.Em acquire
174and
175.Em release
176semantics.
177.Pp
178Atomic operations on memory have up to three variants.
179The first, or
180.Em relaxed
181variant, performs the operation without imposing any ordering constraints on
182accesses to other memory locations.
183This variant is the default.
184The second variant has acquire semantics, and the third variant has release
185semantics.
186.Pp
187When an atomic operation has acquire semantics, the operation must have
188completed before any subsequent load or store (by program order) is
189performed.
190Conversely, acquire semantics do not require that prior loads or stores have
191completed before the atomic operation is performed.
192An atomic operation can only have acquire semantics if it performs a load
193from memory.
194To denote acquire semantics, the suffix
195.Dq Li _acq
196is inserted into the function name immediately prior to the
197.Dq Li _ Ns Aq Fa type
198suffix.
199For example, to subtract two integers ensuring that the subtraction is
200completed before any subsequent loads and stores are performed, use
201.Fn atomic_subtract_acq_int .
202.Pp
203When an atomic operation has release semantics, all prior loads or stores
204(by program order) must have completed before the operation is performed.
205Conversely, release semantics do not require that the atomic operation must
206have completed before any subsequent load or store is performed.
207An atomic operation can only have release semantics if it performs a store
208to memory.
209To denote release semantics, the suffix
210.Dq Li _rel
211is inserted into the function name immediately prior to the
212.Dq Li _ Ns Aq Fa type
213suffix.
214For example, to add two long integers ensuring that all prior loads and
215stores are completed before the addition is performed, use
216.Fn atomic_add_rel_long .
217.Pp
218When a release operation by one thread
219.Em synchronizes with
220an acquire operation by another thread, usually meaning that the acquire
221operation reads the value written by the release operation, then the effects
222of all prior stores by the releasing thread must become visible to
223subsequent loads by the acquiring thread.
224Moreover, the effects of all stores (by other threads) that were visible to
225the releasing thread must also become visible to the acquiring thread.
226These rules only apply to the synchronizing threads.
227Other threads might observe these stores in a different order.
228.Pp
229In effect, atomic operations with acquire and release semantics establish
230one-way barriers to reordering that enable the implementations of
231synchronization primitives to express their ordering requirements without
232also imposing unnecessary ordering.
233For example, for a critical section guarded by a mutex, an acquire operation
234when the mutex is locked and a release operation when the mutex is unlocked
235will prevent any loads or stores from moving outside of the critical
236section.
237However, they will not prevent the compiler or processor from moving loads
238or stores into the critical section, which does not violate the semantics of
239a mutex.
240.Ss Thread Fence Operations
241Alternatively, a programmer can use atomic thread fence operations to
242constrain the reordering of accesses.
243In contrast to other atomic operations, fences do not, themselves, access
244memory.
245.Pp
246When a fence has acquire semantics, all prior loads (by program order) must
247have completed before any subsequent load or store is performed.
248Thus, an acquire fence is a two-way barrier for load operations.
249To denote acquire semantics, the suffix
250.Dq Li _acq
251is appended to the function name, for example,
252.Fn atomic_thread_fence_acq .
253.Pp
254When a fence has release semantics, all prior loads or stores (by program
255order) must have completed before any subsequent store operation is
256performed.
257Thus, a release fence is a two-way barrier for store operations.
258To denote release semantics, the suffix
259.Dq Li _rel
260is appended to the function name, for example,
261.Fn atomic_thread_fence_rel .
262.Pp
263Although
264.Fn atomic_thread_fence_acq_rel
265implements both acquire and release semantics, it is not a full barrier.
266For example, a store prior to the fence (in program order) may be completed
267after a load subsequent to the fence.
268In contrast,
269.Fn atomic_thread_fence_seq_cst
270implements a full barrier.
271Neither loads nor stores may cross this barrier in either direction.
272.Pp
273In C11, a release fence by one thread synchronizes with an acquire fence by
274another thread when an atomic load that is prior to the acquire fence (by
275program order) reads the value written by an atomic store that is subsequent
276to the release fence.
277In constrast, in
278.Fx ,
279because of the atomicity of ordinary, naturally
280aligned loads and stores, fences can also be synchronized by ordinary loads
281and stores.
282This simplifies the implementation and use of some synchronization
283primitives in
284.Fx .
285.Pp
286Since neither a compiler nor a processor can foresee which (atomic) load
287will read the value written by an (atomic) store, the ordering constraints
288imposed by fences must be more restrictive than acquire loads and release
289stores.
290Essentially, this is why fences are two-way barriers.
291.Pp
292Although fences impose more restrictive ordering than acquire loads and
293release stores, by separating access from ordering, they can sometimes
294facilitate more efficient implementations of synchronization primitives.
295For example, they can be used to avoid executing a memory barrier until a
296memory access shows that some condition is satisfied.
297.Ss Interrupt Fence Operations
298The
299.Fn atomic_interrupt_fence
300function establishes ordering between its call location and any interrupt
301handler executing on the same CPU.
302It is modeled after the similar C11 function
303.Fn atomic_signal_fence ,
304and adapted for the kernel environment.
305.Ss Multiple Processors
306In multiprocessor systems, the atomicity of the atomic operations on memory
307depends on support for cache coherence in the underlying architecture.
308In general, cache coherence on the default memory type,
309.Dv VM_MEMATTR_DEFAULT ,
310is guaranteed by all architectures that are supported by
311.Fx .
312For example, cache coherence is guaranteed on write-back memory by the
313.Tn amd64
314and
315.Tn i386
316architectures.
317However, on some architectures, cache coherence might not be enabled on all
318memory types.
319To determine if cache coherence is enabled for a non-default memory type,
320consult the architecture's documentation.
321.Ss Semantics
322This section describes the semantics of each operation using a C like notation.
323.Bl -hang
324.It Fn atomic_add p v
325.Bd -literal -compact
326*p += v;
327.Ed
328.It Fn atomic_clear p v
329.Bd -literal -compact
330*p &= ~v;
331.Ed
332.It Fn atomic_cmpset dst old new
333.Bd -literal -compact
334if (*dst == old) {
335	*dst = new;
336	return (1);
337} else
338	return (0);
339.Ed
340.El
341.Pp
342Some architectures do not implement the
343.Fn atomic_cmpset
344functions for the types
345.Dq Li char ,
346.Dq Li short ,
347.Dq Li 8 ,
348and
349.Dq Li 16 .
350.Bl -hang
351.It Fn atomic_fcmpset dst *old new
352.El
353.Pp
354On architectures implementing
355.Em Compare And Swap
356operation in hardware, the functionality can be described as
357.Bd -literal -offset indent -compact
358if (*dst == *old) {
359	*dst = new;
360	return (1);
361} else {
362	*old = *dst;
363	return (0);
364}
365.Ed
366On architectures which provide
367.Em Load Linked/Store Conditional
368primitive, the write to
369.Dv *dst
370might also fail for several reasons, most important of which
371is a parallel write to
372.Dv *dst
373cache line by other CPU.
374In this case
375.Fn atomic_fcmpset
376function also returns
377.Dv false ,
378despite
379.Dl *old == *dst .
380.Pp
381Some architectures do not implement the
382.Fn atomic_fcmpset
383functions for the types
384.Dq Li char ,
385.Dq Li short ,
386.Dq Li 8 ,
387and
388.Dq Li 16 .
389.Bl -hang
390.It Fn atomic_fetchadd p v
391.Bd -literal -compact
392tmp = *p;
393*p += v;
394return (tmp);
395.Ed
396.El
397.Pp
398The
399.Fn atomic_fetchadd
400functions are only implemented for the types
401.Dq Li int ,
402.Dq Li long
403and
404.Dq Li 32
405and do not have any variants with memory barriers at this time.
406.Bl -hang
407.It Fn atomic_load p
408.Bd -literal -compact
409return (*p);
410.Ed
411.It Fn atomic_readandclear p
412.Bd -literal -compact
413tmp = *p;
414*p = 0;
415return (tmp);
416.Ed
417.El
418.Pp
419The
420.Fn atomic_readandclear
421functions are not implemented for the types
422.Dq Li char ,
423.Dq Li short ,
424.Dq Li ptr ,
425.Dq Li 8 ,
426and
427.Dq Li 16
428and do not have any variants with memory barriers at this time.
429.Bl -hang
430.It Fn atomic_set p v
431.Bd -literal -compact
432*p |= v;
433.Ed
434.It Fn atomic_subtract p v
435.Bd -literal -compact
436*p -= v;
437.Ed
438.It Fn atomic_store p v
439.Bd -literal -compact
440*p = v;
441.Ed
442.It Fn atomic_swap p v
443.Bd -literal -compact
444tmp = *p;
445*p = v;
446return (tmp);
447.Ed
448.El
449.Pp
450The
451.Fn atomic_swap
452functions are not implemented for the types
453.Dq Li char ,
454.Dq Li short ,
455.Dq Li ptr ,
456.Dq Li 8 ,
457and
458.Dq Li 16
459and do not have any variants with memory barriers at this time.
460.Bl -hang
461.It Fn atomic_testandclear p v
462.Bd -literal -compact
463bit = 1 << (v % (sizeof(*p) * NBBY));
464tmp = (*p & bit) != 0;
465*p &= ~bit;
466return (tmp);
467.Ed
468.El
469.Bl -hang
470.It Fn atomic_testandset p v
471.Bd -literal -compact
472bit = 1 << (v % (sizeof(*p) * NBBY));
473tmp = (*p & bit) != 0;
474*p |= bit;
475return (tmp);
476.Ed
477.El
478.Pp
479The
480.Fn atomic_testandset
481and
482.Fn atomic_testandclear
483functions are only implemented for the types
484.Dq Li int ,
485.Dq Li long
486and
487.Dq Li 32
488and do not have any variants with memory barriers at this time.
489.Pp
490The type
491.Dq Li 64
492is currently not implemented for some of the atomic operations on the
493.Tn arm ,
494.Tn i386 ,
495and
496.Tn powerpc
497architectures.
498.Sh RETURN VALUES
499The
500.Fn atomic_cmpset
501function returns the result of the compare operation.
502The
503.Fn atomic_fcmpset
504function returns
505.Dv true
506if the operation succeeded.
507Otherwise it returns
508.Dv false
509and sets
510.Va *old
511to the found value.
512The
513.Fn atomic_fetchadd ,
514.Fn atomic_load ,
515.Fn atomic_readandclear ,
516and
517.Fn atomic_swap
518functions return the value at the specified address.
519The
520.Fn atomic_testandset
521and
522.Fn atomic_testandclear
523function returns the result of the test operation.
524.Sh EXAMPLES
525This example uses the
526.Fn atomic_cmpset_acq_ptr
527and
528.Fn atomic_set_ptr
529functions to obtain a sleep mutex and handle recursion.
530Since the
531.Va mtx_lock
532member of a
533.Vt "struct mtx"
534is a pointer, the
535.Dq Li ptr
536type is used.
537.Bd -literal
538/* Try to obtain mtx_lock once. */
539#define _obtain_lock(mp, tid)						\\
540	atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid))
541
542/* Get a sleep lock, deal with recursion inline. */
543#define _get_sleep_lock(mp, tid, opts, file, line) do {			\\
544	uintptr_t _tid = (uintptr_t)(tid);				\\
545									\\
546	if (!_obtain_lock(mp, tid)) {					\\
547		if (((mp)->mtx_lock & MTX_FLAGMASK) != _tid)		\\
548			_mtx_lock_sleep((mp), _tid, (opts), (file), (line));\\
549		else {							\\
550			atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSE);	\\
551			(mp)->mtx_recurse++;				\\
552		}							\\
553	}								\\
554} while (0)
555.Ed
556.Sh HISTORY
557The
558.Fn atomic_add ,
559.Fn atomic_clear ,
560.Fn atomic_set ,
561and
562.Fn atomic_subtract
563operations were introduced in
564.Fx 3.0 .
565Initially, these operations were defined on the types
566.Dq Li char ,
567.Dq Li short ,
568.Dq Li int ,
569and
570.Dq Li long .
571.Pp
572The
573.Fn atomic_cmpset ,
574.Fn atomic_load_acq ,
575.Fn atomic_readandclear ,
576and
577.Fn atomic_store_rel
578operations were added in
579.Fx 5.0 .
580Simultaneously, the acquire and release variants were introduced, and
581support was added for operation on the types
582.Dq Li 8 ,
583.Dq Li 16 ,
584.Dq Li 32 ,
585.Dq Li 64 ,
586and
587.Dq Li ptr .
588.Pp
589The
590.Fn atomic_fetchadd
591operation was added in
592.Fx 6.0 .
593.Pp
594The
595.Fn atomic_swap
596and
597.Fn atomic_testandset
598operations were added in
599.Fx 10.0 .
600.Pp
601The
602.Fn atomic_testandclear
603and
604.Fn atomic_thread_fence
605operations were added in
606.Fx 11.0 .
607.Pp
608The relaxed variants of
609.Fn atomic_load
610and
611.Fn atomic_store
612were added in
613.Fx 12.0 .
614.Pp
615The
616.Fn atomic_interrupt_fence
617operation was added in
618.Fx 13.0 .
619