1 /*
2 * %CopyrightBegin%
3 *
4 * Copyright Ericsson AB 2018. All Rights Reserved.
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * %CopyrightEnd%
19 */
20
21 /*
22 * Purpose: High performance atomics.
23 */
24
25 #ifdef HAVE_CONFIG_H
26 # include "config.h"
27 #endif
28
29 #include <stddef.h> /* offsetof */
30
31 #include "sys.h"
32 #include "export.h"
33 #include "bif.h"
34 #include "erl_threads.h"
35 #include "big.h"
36 #include "erl_binary.h"
37 #include "erl_bif_unique.h"
38 #include "erl_map.h"
39
40 typedef struct
41 {
42 int is_signed;
43 UWord vlen;
44 erts_atomic64_t v[1];
45 }AtomicsRef;
46
atomics_destructor(Binary * unused)47 static int atomics_destructor(Binary *unused)
48 {
49 return 1;
50 }
51
52 #define OPT_SIGNED (1 << 0)
53
erts_internal_atomics_new_2(BIF_ALIST_2)54 BIF_RETTYPE erts_internal_atomics_new_2(BIF_ALIST_2)
55 {
56 AtomicsRef* p;
57 Binary* mbin;
58 UWord i, cnt, opts;
59 Uint bytes;
60 Eterm* hp;
61
62 if (!term_to_UWord(BIF_ARG_1, &cnt)
63 || cnt == 0
64 || !term_to_UWord(BIF_ARG_2, &opts)) {
65
66 BIF_ERROR(BIF_P, BADARG);
67 }
68
69 if (cnt > (ERTS_UWORD_MAX / sizeof(p->v[0])))
70 BIF_ERROR(BIF_P, SYSTEM_LIMIT);
71
72 bytes = offsetof(AtomicsRef, v) + cnt*sizeof(p->v[0]);
73 mbin = erts_create_magic_binary_x(bytes,
74 atomics_destructor,
75 ERTS_ALC_T_ATOMICS,
76 0);
77 p = ERTS_MAGIC_BIN_DATA(mbin);
78 p->is_signed = opts & OPT_SIGNED;
79 p->vlen = cnt;
80 for (i=0; i < cnt; i++)
81 erts_atomic64_init_nob(&p->v[i], 0);
82 hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
83 return erts_mk_magic_ref(&hp, &MSO(BIF_P), mbin);
84 }
85
get_ref(Eterm ref,AtomicsRef ** pp)86 static ERTS_INLINE int get_ref(Eterm ref, AtomicsRef** pp)
87 {
88 Binary* mbin;
89 if (!is_internal_magic_ref(ref))
90 return 0;
91
92 mbin = erts_magic_ref2bin(ref);
93 if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != atomics_destructor)
94 return 0;
95 *pp = ERTS_MAGIC_BIN_DATA(mbin);
96 return 1;
97 }
98
get_ref_ix(Eterm ref,Eterm ix,AtomicsRef ** pp,UWord * ixp)99 static ERTS_INLINE int get_ref_ix(Eterm ref, Eterm ix,
100 AtomicsRef** pp, UWord* ixp)
101 {
102 return (get_ref(ref, pp)
103 && term_to_UWord(ix, ixp)
104 && --(*ixp) < (*pp)->vlen);
105 }
106
get_value(AtomicsRef * p,Eterm term,erts_aint64_t * valp)107 static ERTS_INLINE int get_value(AtomicsRef* p, Eterm term, erts_aint64_t *valp)
108 {
109 return (p->is_signed ?
110 term_to_Sint64(term, (Sint64*)valp) :
111 term_to_Uint64(term, (Uint64*)valp));
112 }
113
get_incr(AtomicsRef * p,Eterm term,erts_aint64_t * valp)114 static ERTS_INLINE int get_incr(AtomicsRef* p, Eterm term, erts_aint64_t *valp)
115 {
116 return (term_to_Sint64(term, (Sint64*)valp)
117 || term_to_Uint64(term, (Uint64*)valp));
118 }
119
bld_atomic(Process * proc,AtomicsRef * p,erts_aint64_t val)120 static ERTS_INLINE Eterm bld_atomic(Process* proc, AtomicsRef* p,
121 erts_aint64_t val)
122 {
123 if (p->is_signed) {
124 if (IS_SSMALL(val))
125 return make_small((Sint) val);
126 else {
127 Uint hsz = ERTS_SINT64_HEAP_SIZE(val);
128 Eterm* hp = HAlloc(proc, hsz);
129 return erts_sint64_to_big(val, &hp);
130 }
131 }
132 else {
133 if ((Uint64)val <= MAX_SMALL)
134 return make_small((Sint) val);
135 else {
136 Uint hsz = ERTS_UINT64_HEAP_SIZE((Uint64)val);
137 Eterm* hp = HAlloc(proc, hsz);
138 return erts_uint64_to_big(val, &hp);
139 }
140 }
141 }
142
atomics_put_3(BIF_ALIST_3)143 BIF_RETTYPE atomics_put_3(BIF_ALIST_3)
144 {
145 AtomicsRef* p;
146 UWord ix;
147 erts_aint64_t val;
148
149 if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)
150 || !get_value(p, BIF_ARG_3, &val)) {
151 BIF_ERROR(BIF_P, BADARG);
152 }
153 erts_atomic64_set_mb(&p->v[ix], val);
154 return am_ok;
155 }
156
atomics_get_2(BIF_ALIST_2)157 BIF_RETTYPE atomics_get_2(BIF_ALIST_2)
158 {
159 AtomicsRef* p;
160 UWord ix;
161
162 if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)) {
163 BIF_ERROR(BIF_P, BADARG);
164 }
165 return bld_atomic(BIF_P, p, erts_atomic64_read_mb(&p->v[ix]));
166 }
167
atomics_add_3(BIF_ALIST_3)168 BIF_RETTYPE atomics_add_3(BIF_ALIST_3)
169 {
170 AtomicsRef* p;
171 UWord ix;
172 erts_aint64_t incr;
173
174 if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)
175 || !get_incr(p, BIF_ARG_3, &incr)) {
176 BIF_ERROR(BIF_P, BADARG);
177 }
178 erts_atomic64_add_mb(&p->v[ix], incr);
179 return am_ok;
180 }
181
atomics_add_get_3(BIF_ALIST_3)182 BIF_RETTYPE atomics_add_get_3(BIF_ALIST_3)
183 {
184 AtomicsRef* p;
185 UWord ix;
186 erts_aint64_t incr;
187
188 if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)
189 || !get_incr(p, BIF_ARG_3, &incr)) {
190 BIF_ERROR(BIF_P, BADARG);
191 }
192 return bld_atomic(BIF_P, p, erts_atomic64_add_read_mb(&p->v[ix], incr));
193 }
194
atomics_exchange_3(BIF_ALIST_3)195 BIF_RETTYPE atomics_exchange_3(BIF_ALIST_3)
196 {
197 AtomicsRef* p;
198 UWord ix;
199 erts_aint64_t desired, was;
200
201 if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)
202 || !get_value(p, BIF_ARG_3, &desired)) {
203 BIF_ERROR(BIF_P, BADARG);
204 }
205 was = erts_atomic64_xchg_mb(&p->v[ix], desired);
206 return bld_atomic(BIF_P, p, was);
207 }
208
atomics_compare_exchange_4(BIF_ALIST_4)209 BIF_RETTYPE atomics_compare_exchange_4(BIF_ALIST_4)
210 {
211 AtomicsRef* p;
212 UWord ix;
213 erts_aint64_t expected, desired, was;
214
215 if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)
216 || !get_value(p, BIF_ARG_3, &expected)
217 || !get_value(p, BIF_ARG_4, &desired)) {
218 BIF_ERROR(BIF_P, BADARG);
219 }
220 was = erts_atomic64_cmpxchg_mb(&p->v[ix], desired, expected);
221 return was == expected ? am_ok : bld_atomic(BIF_P, p, was);
222 }
223
atomics_info_1(BIF_ALIST_1)224 BIF_RETTYPE atomics_info_1(BIF_ALIST_1)
225 {
226 AtomicsRef* p;
227 Uint hsz = MAP4_SZ;
228 Eterm *hp;
229 Uint64 max;
230 Sint64 min;
231 UWord memory;
232 Eterm max_val, min_val, sz_val, mem_val;
233
234 if (!get_ref(BIF_ARG_1, &p))
235 BIF_ERROR(BIF_P, BADARG);
236
237 max = p->is_signed ? ERTS_SINT64_MAX : ERTS_UINT64_MAX;
238 min = p->is_signed ? ERTS_SINT64_MIN : 0;
239 memory = erts_magic_ref2bin(BIF_ARG_1)->orig_size;
240
241 erts_bld_uint64(NULL, &hsz, max);
242 erts_bld_sint64(NULL, &hsz, min);
243 erts_bld_uword(NULL, &hsz, p->vlen);
244 erts_bld_uword(NULL, &hsz, memory);
245
246 hp = HAlloc(BIF_P, hsz);
247 max_val = erts_bld_uint64(&hp, NULL, max);
248 min_val = erts_bld_sint64(&hp, NULL, min);
249 sz_val = erts_bld_uword(&hp, NULL, p->vlen);
250 mem_val = erts_bld_uword(&hp, NULL, memory);
251
252 return MAP4(hp, am_max, max_val,
253 am_memory, mem_val,
254 am_min, min_val,
255 am_size, sz_val);
256 }
257