1 /*
2 * Copyright (c) 2017-2020 François Tigeot <ftigeot@wolfpond.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #ifndef _LINUX_SEQLOCK_H_
28 #define _LINUX_SEQLOCK_H_
29
30 #include <linux/spinlock.h>
31 #include <linux/preempt.h>
32 #include <linux/lockdep.h>
33 #include <linux/compiler.h>
34 #include <asm/processor.h>
35
36 /*
37 * Seqlock definition from Wikipedia
38 *
39 * A seqlock consists of storage for saving a sequence number in addition to a lock.
40 * The lock is to support synchronization between two writers and the counter is for
41 * indicating consistency in readers. In addition to updating the shared data, the
42 * writer increments the sequence number, both after acquiring the lock and before
43 * releasing the lock. Readers read the sequence number before and after reading the
44 * shared data. If the sequence number is odd on either occasion, a writer had taken
45 * the lock while the data was being read and it may have changed. If the sequence
46 * numbers are different, a writer has changed the data while it was being read. In
47 * either case readers simply retry (using a loop) until they read the same even
48 * sequence number before and after.
49 *
50 */
51
52 typedef struct {
53 unsigned sequence;
54 struct lock lock;
55 } seqlock_t;
56
57 static inline void
seqlock_init(seqlock_t * sl)58 seqlock_init(seqlock_t *sl)
59 {
60 sl->sequence = 0;
61 lockinit(&sl->lock, "lsql", 0, 0);
62 }
63
64 /*
65 * Writers always use a spinlock. We still use store barriers
66 * in order to quickly update the state of the sequence variable
67 * for readers.
68 */
69 static inline void
write_seqlock(seqlock_t * sl)70 write_seqlock(seqlock_t *sl)
71 {
72 lockmgr(&sl->lock, LK_EXCLUSIVE);
73 sl->sequence++;
74 cpu_sfence();
75 }
76
77 static inline void
write_sequnlock(seqlock_t * sl)78 write_sequnlock(seqlock_t *sl)
79 {
80 sl->sequence--;
81 lockmgr(&sl->lock, LK_RELEASE);
82 cpu_sfence();
83 }
84
85 /*
86 * Read functions are fully unlocked.
87 * We use load barriers to obtain a reasonably up-to-date state
88 * for the sequence number.
89 */
90 static inline unsigned
read_seqbegin(const seqlock_t * sl)91 read_seqbegin(const seqlock_t *sl)
92 {
93 return READ_ONCE(sl->sequence);
94 }
95
96 static inline unsigned
read_seqretry(const seqlock_t * sl,unsigned start)97 read_seqretry(const seqlock_t *sl, unsigned start)
98 {
99 cpu_lfence();
100 return (sl->sequence != start);
101 }
102
103 typedef struct seqcount {
104 unsigned sequence;
105 } seqcount_t;
106
107 static inline void
__seqcount_init(seqcount_t * s,const char * name,struct lock_class_key * key)108 __seqcount_init(seqcount_t *s, const char *name, struct lock_class_key *key)
109 {
110 s->sequence = 0;
111 }
112
113 static inline unsigned int
__read_seqcount_begin(const seqcount_t * s)114 __read_seqcount_begin(const seqcount_t *s)
115 {
116 unsigned int ret;
117
118 do {
119 ret = READ_ONCE(s->sequence);
120 /* If the sequence number is odd, a writer has taken the lock */
121 if ((ret & 1) == 0)
122 break;
123 cpu_pause();
124 } while (1);
125
126 return ret;
127 }
128
129 static inline unsigned int
read_seqcount_begin(const seqcount_t * s)130 read_seqcount_begin(const seqcount_t *s)
131 {
132 unsigned int ret = __read_seqcount_begin(s);
133
134 cpu_lfence();
135
136 return ret;
137 }
138
139 static inline int
__read_seqcount_retry(const seqcount_t * s,unsigned start)140 __read_seqcount_retry(const seqcount_t *s, unsigned start)
141 {
142 return (s->sequence != start);
143 }
144
145 static inline int
read_seqcount_retry(const seqcount_t * s,unsigned start)146 read_seqcount_retry(const seqcount_t *s, unsigned start)
147 {
148 cpu_lfence();
149 return __read_seqcount_retry(s, start);
150 }
151
write_seqcount_begin(seqcount_t * s)152 static inline void write_seqcount_begin(seqcount_t *s)
153 {
154 s->sequence++;
155 cpu_ccfence();
156 }
157
write_seqcount_end(seqcount_t * s)158 static inline void write_seqcount_end(seqcount_t *s)
159 {
160 cpu_ccfence();
161 s->sequence++;
162 }
163
164 static inline unsigned int
raw_read_seqcount(const seqcount_t * s)165 raw_read_seqcount(const seqcount_t *s)
166 {
167 unsigned int value = READ_ONCE(s->sequence);
168
169 cpu_ccfence();
170
171 return value;
172 }
173
174 #endif /* _LINUX_SEQLOCK_H_ */
175