155fda581SMateusz Guzik /*-
255fda581SMateusz Guzik * Copyright (c) 2014 Mateusz Guzik <mjg@FreeBSD.org>
355fda581SMateusz Guzik *
455fda581SMateusz Guzik * Redistribution and use in source and binary forms, with or without
555fda581SMateusz Guzik * modification, are permitted provided that the following conditions
655fda581SMateusz Guzik * are met:
755fda581SMateusz Guzik * 1. Redistributions of source code must retain the above copyright
855fda581SMateusz Guzik * notice, this list of conditions and the following disclaimer.
955fda581SMateusz Guzik * 2. Redistributions in binary form must reproduce the above copyright
1055fda581SMateusz Guzik * notice, this list of conditions and the following disclaimer in the
1155fda581SMateusz Guzik * documentation and/or other materials provided with the distribution.
1255fda581SMateusz Guzik *
1355fda581SMateusz Guzik * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
1455fda581SMateusz Guzik * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1555fda581SMateusz Guzik * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
1655fda581SMateusz Guzik * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
1755fda581SMateusz Guzik * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
1855fda581SMateusz Guzik * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
1955fda581SMateusz Guzik * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
2055fda581SMateusz Guzik * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2155fda581SMateusz Guzik * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2255fda581SMateusz Guzik * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
2355fda581SMateusz Guzik * SUCH DAMAGE.
2455fda581SMateusz Guzik */
2555fda581SMateusz Guzik
2655fda581SMateusz Guzik #ifndef _SYS_SEQC_H_
2755fda581SMateusz Guzik #define _SYS_SEQC_H_
2855fda581SMateusz Guzik
2955fda581SMateusz Guzik #ifdef _KERNEL
3055fda581SMateusz Guzik #include <sys/systm.h>
3155fda581SMateusz Guzik #endif
3255fda581SMateusz Guzik #include <sys/types.h>
3355fda581SMateusz Guzik
3455fda581SMateusz Guzik /*
3555fda581SMateusz Guzik * seqc_t may be included in structs visible to userspace
3655fda581SMateusz Guzik */
3782dc8122SMateusz Guzik #include <sys/_seqc.h>
3855fda581SMateusz Guzik
3955fda581SMateusz Guzik #ifdef _KERNEL
4055fda581SMateusz Guzik
4155fda581SMateusz Guzik /* A hack to get MPASS macro */
4255fda581SMateusz Guzik #include <sys/lock.h>
4355fda581SMateusz Guzik
4455fda581SMateusz Guzik #include <machine/cpu.h>
4555fda581SMateusz Guzik
4671faea93SKonstantin Belousov #define SEQC_MOD 1
4771faea93SKonstantin Belousov
4882dc8122SMateusz Guzik /*
4982dc8122SMateusz Guzik * Predicts from inline functions are not honored by clang.
5082dc8122SMateusz Guzik */
5182dc8122SMateusz Guzik #define seqc_in_modify(seqc) ({ \
5282dc8122SMateusz Guzik seqc_t __seqc = (seqc); \
5382dc8122SMateusz Guzik \
5471faea93SKonstantin Belousov __predict_false(__seqc & SEQC_MOD); \
5582dc8122SMateusz Guzik })
5655fda581SMateusz Guzik
5755fda581SMateusz Guzik static __inline void
seqc_write_begin(seqc_t * seqcp)5855fda581SMateusz Guzik seqc_write_begin(seqc_t *seqcp)
5955fda581SMateusz Guzik {
6055fda581SMateusz Guzik
6155fda581SMateusz Guzik critical_enter();
6255fda581SMateusz Guzik MPASS(!seqc_in_modify(*seqcp));
6371faea93SKonstantin Belousov *seqcp += SEQC_MOD;
6455fda581SMateusz Guzik atomic_thread_fence_rel();
6555fda581SMateusz Guzik }
6655fda581SMateusz Guzik
6755fda581SMateusz Guzik static __inline void
seqc_write_end(seqc_t * seqcp)6855fda581SMateusz Guzik seqc_write_end(seqc_t *seqcp)
6955fda581SMateusz Guzik {
7055fda581SMateusz Guzik
71a6924f5dSMateusz Guzik atomic_thread_fence_rel();
7271faea93SKonstantin Belousov *seqcp += SEQC_MOD;
7355fda581SMateusz Guzik MPASS(!seqc_in_modify(*seqcp));
7455fda581SMateusz Guzik critical_exit();
7555fda581SMateusz Guzik }
7655fda581SMateusz Guzik
7755fda581SMateusz Guzik static __inline seqc_t
seqc_read_any(const seqc_t * seqcp)784846218dSMateusz Guzik seqc_read_any(const seqc_t *seqcp)
794846218dSMateusz Guzik {
804846218dSMateusz Guzik
814846218dSMateusz Guzik return (atomic_load_acq_int(__DECONST(seqc_t *, seqcp)));
824846218dSMateusz Guzik }
834846218dSMateusz Guzik
844846218dSMateusz Guzik static __inline seqc_t
seqc_read_notmodify(const seqc_t * seqcp)8565f77515SMateusz Guzik seqc_read_notmodify(const seqc_t *seqcp)
8665f77515SMateusz Guzik {
8765f77515SMateusz Guzik
8871faea93SKonstantin Belousov return (atomic_load_acq_int(__DECONST(seqc_t *, seqcp)) & ~SEQC_MOD);
8965f77515SMateusz Guzik }
9065f77515SMateusz Guzik
9165f77515SMateusz Guzik static __inline seqc_t
seqc_read(const seqc_t * seqcp)9255fda581SMateusz Guzik seqc_read(const seqc_t *seqcp)
9355fda581SMateusz Guzik {
9455fda581SMateusz Guzik seqc_t ret;
9555fda581SMateusz Guzik
9655fda581SMateusz Guzik for (;;) {
97a6924f5dSMateusz Guzik ret = seqc_read_any(seqcp);
9882dc8122SMateusz Guzik if (seqc_in_modify(ret)) {
9955fda581SMateusz Guzik cpu_spinwait();
10055fda581SMateusz Guzik continue;
10155fda581SMateusz Guzik }
10255fda581SMateusz Guzik break;
10355fda581SMateusz Guzik }
10455fda581SMateusz Guzik
10555fda581SMateusz Guzik return (ret);
10655fda581SMateusz Guzik }
10755fda581SMateusz Guzik
108c9a99599SMateusz Guzik #define seqc_consistent_no_fence(seqcp, oldseqc)({ \
10982dc8122SMateusz Guzik const seqc_t *__seqcp = (seqcp); \
11082dc8122SMateusz Guzik seqc_t __oldseqc = (oldseqc); \
11182dc8122SMateusz Guzik \
11282dc8122SMateusz Guzik MPASS(!(seqc_in_modify(__oldseqc))); \
11382dc8122SMateusz Guzik __predict_true(*__seqcp == __oldseqc); \
11482dc8122SMateusz Guzik })
11582dc8122SMateusz Guzik
11682dc8122SMateusz Guzik #define seqc_consistent(seqcp, oldseqc) ({ \
11782dc8122SMateusz Guzik atomic_thread_fence_acq(); \
118c9a99599SMateusz Guzik seqc_consistent_no_fence(seqcp, oldseqc); \
11982dc8122SMateusz Guzik })
12082dc8122SMateusz Guzik
12182dc8122SMateusz Guzik /*
12282dc8122SMateusz Guzik * Variant which does not critical enter/exit.
12382dc8122SMateusz Guzik */
12482dc8122SMateusz Guzik static __inline void
seqc_sleepable_write_begin(seqc_t * seqcp)12582dc8122SMateusz Guzik seqc_sleepable_write_begin(seqc_t *seqcp)
12655fda581SMateusz Guzik {
12755fda581SMateusz Guzik
12882dc8122SMateusz Guzik MPASS(!seqc_in_modify(*seqcp));
12971faea93SKonstantin Belousov *seqcp += SEQC_MOD;
13082dc8122SMateusz Guzik atomic_thread_fence_rel();
13155fda581SMateusz Guzik }
13255fda581SMateusz Guzik
13382dc8122SMateusz Guzik static __inline void
seqc_sleepable_write_end(seqc_t * seqcp)13482dc8122SMateusz Guzik seqc_sleepable_write_end(seqc_t *seqcp)
13555fda581SMateusz Guzik {
13655fda581SMateusz Guzik
13782dc8122SMateusz Guzik atomic_thread_fence_rel();
13871faea93SKonstantin Belousov *seqcp += SEQC_MOD;
13982dc8122SMateusz Guzik MPASS(!seqc_in_modify(*seqcp));
14055fda581SMateusz Guzik }
14155fda581SMateusz Guzik
14255fda581SMateusz Guzik #endif /* _KERNEL */
14355fda581SMateusz Guzik #endif /* _SYS_SEQC_H_ */
144