18b33cb83SVladimir Kondratyev /*-
24d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause
38b33cb83SVladimir Kondratyev *
48b33cb83SVladimir Kondratyev * Copyright (c) 2021 Vladimir Kondratyev <wulf@FreeBSD.org>
58b33cb83SVladimir Kondratyev *
68b33cb83SVladimir Kondratyev * Redistribution and use in source and binary forms, with or without
78b33cb83SVladimir Kondratyev * modification, are permitted provided that the following conditions are
88b33cb83SVladimir Kondratyev * met:
98b33cb83SVladimir Kondratyev * 1. Redistributions of source code must retain the above copyright
108b33cb83SVladimir Kondratyev * notice, this list of conditions and the following disclaimer.
118b33cb83SVladimir Kondratyev * 2. Redistributions in binary form must reproduce the above copyright
128b33cb83SVladimir Kondratyev * notice, this list of conditions and the following disclaimer in
138b33cb83SVladimir Kondratyev * the documentation and/or other materials provided with the
148b33cb83SVladimir Kondratyev * distribution.
158b33cb83SVladimir Kondratyev *
168b33cb83SVladimir Kondratyev * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
178b33cb83SVladimir Kondratyev * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
188b33cb83SVladimir Kondratyev * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
198b33cb83SVladimir Kondratyev * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
208b33cb83SVladimir Kondratyev * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
218b33cb83SVladimir Kondratyev * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
228b33cb83SVladimir Kondratyev * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
238b33cb83SVladimir Kondratyev * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
248b33cb83SVladimir Kondratyev * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
258b33cb83SVladimir Kondratyev * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
268b33cb83SVladimir Kondratyev * SUCH DAMAGE.
278b33cb83SVladimir Kondratyev */
288b33cb83SVladimir Kondratyev
29307f78f3SVladimir Kondratyev #ifndef _LINUXKPI_LINUX_SEQLOCK_H__
30307f78f3SVladimir Kondratyev #define _LINUXKPI_LINUX_SEQLOCK_H__
318b33cb83SVladimir Kondratyev
328b33cb83SVladimir Kondratyev #include <sys/param.h>
33efb8f0b8SMark Johnston #include <sys/systm.h>
3464e30cbaSVladimir Kondratyev #include <sys/cdefs.h>
358b33cb83SVladimir Kondratyev #include <sys/lock.h>
368b33cb83SVladimir Kondratyev #include <sys/mutex.h>
3718e41123SJean-Sébastien Pédron #include <sys/rwlock.h>
388b33cb83SVladimir Kondratyev #include <sys/seqc.h>
398b33cb83SVladimir Kondratyev
408b33cb83SVladimir Kondratyev struct lock_class_key;
418b33cb83SVladimir Kondratyev
428b33cb83SVladimir Kondratyev struct seqcount {
438b33cb83SVladimir Kondratyev seqc_t seqc;
448b33cb83SVladimir Kondratyev };
458b33cb83SVladimir Kondratyev typedef struct seqcount seqcount_t;
468b33cb83SVladimir Kondratyev
478b33cb83SVladimir Kondratyev struct seqlock {
488b33cb83SVladimir Kondratyev struct mtx seql_lock;
498b33cb83SVladimir Kondratyev struct seqcount seql_count;
508b33cb83SVladimir Kondratyev };
518b33cb83SVladimir Kondratyev typedef struct seqlock seqlock_t;
528b33cb83SVladimir Kondratyev
5318e41123SJean-Sébastien Pédron struct seqcount_mutex {
5468f08e26SVladimir Kondratyev seqc_t seqc;
5518e41123SJean-Sébastien Pédron };
5618e41123SJean-Sébastien Pédron typedef struct seqcount_mutex seqcount_mutex_t;
57b34cd672SJean-Sébastien Pédron typedef struct seqcount_mutex seqcount_ww_mutex_t;
5818e41123SJean-Sébastien Pédron
598b33cb83SVladimir Kondratyev static inline void
__seqcount_init(struct seqcount * seqcount,const char * name __unused,struct lock_class_key * key __unused)608b33cb83SVladimir Kondratyev __seqcount_init(struct seqcount *seqcount, const char *name __unused,
618b33cb83SVladimir Kondratyev struct lock_class_key *key __unused)
628b33cb83SVladimir Kondratyev {
638b33cb83SVladimir Kondratyev seqcount->seqc = 0;
648b33cb83SVladimir Kondratyev }
658b33cb83SVladimir Kondratyev #define seqcount_init(seqcount) __seqcount_init(seqcount, NULL, NULL)
668b33cb83SVladimir Kondratyev
678b33cb83SVladimir Kondratyev static inline void
seqcount_mutex_init(struct seqcount_mutex * seqcount,void * mutex __unused)6868f08e26SVladimir Kondratyev seqcount_mutex_init(struct seqcount_mutex *seqcount, void *mutex __unused)
6918e41123SJean-Sébastien Pédron {
7068f08e26SVladimir Kondratyev seqcount->seqc = 0;
7118e41123SJean-Sébastien Pédron }
7218e41123SJean-Sébastien Pédron
73b34cd672SJean-Sébastien Pédron #define seqcount_ww_mutex_init(seqcount, ww_mutex) \
74b34cd672SJean-Sébastien Pédron seqcount_mutex_init((seqcount), (ww_mutex))
75b34cd672SJean-Sébastien Pédron
7618e41123SJean-Sébastien Pédron #define write_seqcount_begin(s) \
7718e41123SJean-Sébastien Pédron _Generic(*(s), \
7868f08e26SVladimir Kondratyev struct seqcount: seqc_sleepable_write_begin, \
7968f08e26SVladimir Kondratyev struct seqcount_mutex: seqc_write_begin \
8068f08e26SVladimir Kondratyev )(&(s)->seqc)
8118e41123SJean-Sébastien Pédron
8218e41123SJean-Sébastien Pédron #define write_seqcount_end(s) \
8318e41123SJean-Sébastien Pédron _Generic(*(s), \
8468f08e26SVladimir Kondratyev struct seqcount: seqc_sleepable_write_end, \
8568f08e26SVladimir Kondratyev struct seqcount_mutex: seqc_write_end \
8668f08e26SVladimir Kondratyev )(&(s)->seqc)
8718e41123SJean-Sébastien Pédron
8864e30cbaSVladimir Kondratyev static inline void
lkpi_write_seqcount_invalidate(seqc_t * seqcp)8964e30cbaSVladimir Kondratyev lkpi_write_seqcount_invalidate(seqc_t *seqcp)
9064e30cbaSVladimir Kondratyev {
9164e30cbaSVladimir Kondratyev atomic_thread_fence_rel();
9264e30cbaSVladimir Kondratyev *seqcp += SEQC_MOD * 2;
9364e30cbaSVladimir Kondratyev }
9464e30cbaSVladimir Kondratyev #define write_seqcount_invalidate(s) lkpi_write_seqcount_invalidate(&(s)->seqc)
9564e30cbaSVladimir Kondratyev
9668f08e26SVladimir Kondratyev #define read_seqcount_begin(s) seqc_read(&(s)->seqc)
9768f08e26SVladimir Kondratyev #define raw_read_seqcount(s) seqc_read_any(&(s)->seqc)
9818e41123SJean-Sébastien Pédron
9964e30cbaSVladimir Kondratyev static inline seqc_t
lkpi_seqprop_sequence(const seqc_t * seqcp)10064e30cbaSVladimir Kondratyev lkpi_seqprop_sequence(const seqc_t *seqcp)
10164e30cbaSVladimir Kondratyev {
10264e30cbaSVladimir Kondratyev return (atomic_load_int(__DECONST(seqc_t *, seqcp)));
10364e30cbaSVladimir Kondratyev }
10464e30cbaSVladimir Kondratyev #define seqprop_sequence(s) lkpi_seqprop_sequence(&(s)->seqc)
10564e30cbaSVladimir Kondratyev
1068b33cb83SVladimir Kondratyev /*
1078b33cb83SVladimir Kondratyev * XXX: Are predicts from inline functions still not honored by clang?
1088b33cb83SVladimir Kondratyev */
1098b33cb83SVladimir Kondratyev #define __read_seqcount_retry(seqcount, gen) \
110c9a99599SMateusz Guzik (!seqc_consistent_no_fence(&(seqcount)->seqc, gen))
11168f08e26SVladimir Kondratyev #define read_seqcount_retry(seqcount, gen) \
11268f08e26SVladimir Kondratyev (!seqc_consistent(&(seqcount)->seqc, gen))
1138b33cb83SVladimir Kondratyev
1148b33cb83SVladimir Kondratyev static inline void
seqlock_init(struct seqlock * seqlock)1158b33cb83SVladimir Kondratyev seqlock_init(struct seqlock *seqlock)
1168b33cb83SVladimir Kondratyev {
1178b33cb83SVladimir Kondratyev /*
1188b33cb83SVladimir Kondratyev * Don't enroll to witness(4) to avoid orphaned references after struct
1198b33cb83SVladimir Kondratyev * seqlock has been freed. There is no seqlock destructor exists so we
1208b33cb83SVladimir Kondratyev * can't expect automatic mtx_destroy() execution before free().
1218b33cb83SVladimir Kondratyev */
1228b33cb83SVladimir Kondratyev mtx_init(&seqlock->seql_lock, "seqlock", NULL, MTX_DEF|MTX_NOWITNESS);
1238b33cb83SVladimir Kondratyev seqcount_init(&seqlock->seql_count);
1248b33cb83SVladimir Kondratyev }
1258b33cb83SVladimir Kondratyev
1268b33cb83SVladimir Kondratyev static inline void
lkpi_write_seqlock(struct seqlock * seqlock,const bool irqsave)127efb8f0b8SMark Johnston lkpi_write_seqlock(struct seqlock *seqlock, const bool irqsave)
1288b33cb83SVladimir Kondratyev {
1298b33cb83SVladimir Kondratyev mtx_lock(&seqlock->seql_lock);
130efb8f0b8SMark Johnston if (irqsave)
131efb8f0b8SMark Johnston critical_enter();
1328b33cb83SVladimir Kondratyev write_seqcount_begin(&seqlock->seql_count);
1338b33cb83SVladimir Kondratyev }
1348b33cb83SVladimir Kondratyev
1358b33cb83SVladimir Kondratyev static inline void
write_seqlock(struct seqlock * seqlock)136efb8f0b8SMark Johnston write_seqlock(struct seqlock *seqlock)
137efb8f0b8SMark Johnston {
138efb8f0b8SMark Johnston lkpi_write_seqlock(seqlock, false);
139efb8f0b8SMark Johnston }
140efb8f0b8SMark Johnston
141efb8f0b8SMark Johnston static inline void
lkpi_write_sequnlock(struct seqlock * seqlock,const bool irqsave)142efb8f0b8SMark Johnston lkpi_write_sequnlock(struct seqlock *seqlock, const bool irqsave)
143efb8f0b8SMark Johnston {
144efb8f0b8SMark Johnston write_seqcount_end(&seqlock->seql_count);
145efb8f0b8SMark Johnston if (irqsave)
146efb8f0b8SMark Johnston critical_exit();
147efb8f0b8SMark Johnston mtx_unlock(&seqlock->seql_lock);
148efb8f0b8SMark Johnston }
149efb8f0b8SMark Johnston
150efb8f0b8SMark Johnston static inline void
write_sequnlock(struct seqlock * seqlock)1518b33cb83SVladimir Kondratyev write_sequnlock(struct seqlock *seqlock)
1528b33cb83SVladimir Kondratyev {
153efb8f0b8SMark Johnston lkpi_write_sequnlock(seqlock, false);
1548b33cb83SVladimir Kondratyev }
1558b33cb83SVladimir Kondratyev
156efb8f0b8SMark Johnston /*
157efb8f0b8SMark Johnston * Disable preemption when the consumer wants to disable interrupts. This
158efb8f0b8SMark Johnston * ensures that the caller won't be starved if it is preempted by a
159efb8f0b8SMark Johnston * higher-priority reader, but assumes that the caller won't perform any
160efb8f0b8SMark Johnston * blocking operations while holding the write lock; probably a safe
161efb8f0b8SMark Johnston * assumption.
162efb8f0b8SMark Johnston */
1638b33cb83SVladimir Kondratyev #define write_seqlock_irqsave(seqlock, flags) do { \
1648b33cb83SVladimir Kondratyev (flags) = 0; \
165efb8f0b8SMark Johnston lkpi_write_seqlock(seqlock, true); \
1668b33cb83SVladimir Kondratyev } while (0)
1678b33cb83SVladimir Kondratyev
1688b33cb83SVladimir Kondratyev static inline void
write_sequnlock_irqrestore(struct seqlock * seqlock,unsigned long flags __unused)1698b33cb83SVladimir Kondratyev write_sequnlock_irqrestore(struct seqlock *seqlock,
1708b33cb83SVladimir Kondratyev unsigned long flags __unused)
1718b33cb83SVladimir Kondratyev {
172efb8f0b8SMark Johnston lkpi_write_sequnlock(seqlock, true);
1738b33cb83SVladimir Kondratyev }
1748b33cb83SVladimir Kondratyev
1758b33cb83SVladimir Kondratyev static inline unsigned
read_seqbegin(const struct seqlock * seqlock)1768b33cb83SVladimir Kondratyev read_seqbegin(const struct seqlock *seqlock)
1778b33cb83SVladimir Kondratyev {
1788b33cb83SVladimir Kondratyev return (read_seqcount_begin(&seqlock->seql_count));
1798b33cb83SVladimir Kondratyev }
1808b33cb83SVladimir Kondratyev
1818b33cb83SVladimir Kondratyev #define read_seqretry(seqlock, gen) \
1828b33cb83SVladimir Kondratyev read_seqcount_retry(&(seqlock)->seql_count, gen)
1838b33cb83SVladimir Kondratyev
184307f78f3SVladimir Kondratyev #endif /* _LINUXKPI_LINUX_SEQLOCK_H__ */
185