1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 /*
3 * See COPYRIGHT in top-level directory.
4 */
5
6 #include <stdlib.h>
7 #include <hwloc.h>
8 #include "lock/zm_mcs.h"
9
10 struct zm_mcs {
11 zm_atomic_ptr_t lock;
12 struct zm_mcs_qnode *local_nodes;
13 hwloc_topology_t topo;
14 };
15
16 static zm_thread_local int tid = -1;
17
18 /* Check the actual affinity mask assigned to the thread */
check_affinity(hwloc_topology_t topo)19 static inline void check_affinity(hwloc_topology_t topo) {
20 hwloc_cpuset_t cpuset = hwloc_bitmap_alloc();
21 int set_length;
22 hwloc_get_cpubind(topo, cpuset, HWLOC_CPUBIND_THREAD);
23 set_length = hwloc_get_nbobjs_inside_cpuset_by_type(topo, cpuset, HWLOC_OBJ_PU);
24 hwloc_bitmap_free(cpuset);
25
26 if(set_length != 1) {
27 printf("IZEM:HMCS:ERROR: thread bound to more than one HW thread!\n");
28 exit(EXIT_FAILURE);
29 }
30 }
31
get_hwthread_id(hwloc_topology_t topo)32 static inline int get_hwthread_id(hwloc_topology_t topo){
33 hwloc_cpuset_t cpuset = hwloc_bitmap_alloc();
34 hwloc_obj_t obj;
35 hwloc_get_cpubind(topo, cpuset, HWLOC_CPUBIND_THREAD);
36 obj = hwloc_get_obj_inside_cpuset_by_type(topo, cpuset, HWLOC_OBJ_PU, 0);
37 hwloc_bitmap_free(cpuset);
38 return obj->logical_index;
39 }
40
new_lock()41 static void* new_lock() {
42 int max_threads;
43 struct zm_mcs_qnode *qnodes;
44
45
46 struct zm_mcs *L;
47 posix_memalign((void **) &L, ZM_CACHELINE_SIZE, sizeof(struct zm_mcs));
48
49 hwloc_topology_init(&L->topo);
50 hwloc_topology_load(L->topo);
51
52 max_threads = hwloc_get_nbobjs_by_type(L->topo, HWLOC_OBJ_PU);
53
54 posix_memalign((void **) &qnodes, ZM_CACHELINE_SIZE, sizeof(struct zm_mcs_qnode) * max_threads);
55
56 zm_atomic_store(&L->lock, (zm_ptr_t)ZM_NULL, zm_memord_release);
57 L->local_nodes = qnodes;
58
59 return L;
60 }
61
62 /* Main routines */
acquire_c(struct zm_mcs * L,zm_mcs_qnode_t * I)63 static inline int acquire_c(struct zm_mcs *L, zm_mcs_qnode_t* I) {
64 zm_atomic_store(&I->next, ZM_NULL, zm_memord_release);
65 zm_mcs_qnode_t* pred = (zm_mcs_qnode_t*)zm_atomic_exchange_ptr(&L->lock, (zm_ptr_t)I, zm_memord_acq_rel);
66 if((zm_ptr_t)pred != ZM_NULL) {
67 zm_atomic_store(&I->status, ZM_LOCKED, zm_memord_release);
68 zm_atomic_store(&pred->next, (zm_ptr_t)I, zm_memord_release);
69 while(zm_atomic_load(&I->status, zm_memord_acquire) != ZM_UNLOCKED)
70 ; /* SPIN */
71 }
72 return 0;
73 }
74
tryacq_c(struct zm_mcs * L,zm_mcs_qnode_t * I,int * success)75 static inline int tryacq_c(struct zm_mcs *L, zm_mcs_qnode_t* I, int *success) {
76 int acquired = 0;
77 zm_atomic_store(&I->next, ZM_NULL, zm_memord_release);
78 zm_ptr_t expected = ZM_NULL;
79 if(zm_atomic_compare_exchange_strong(&L->lock,
80 &expected,
81 (zm_ptr_t)I,
82 zm_memord_acq_rel,
83 zm_memord_acquire))
84 acquired = 1;
85 *success = acquired;
86 return 0;
87 }
88
89 /* Release the lock */
release_c(struct zm_mcs * L,zm_mcs_qnode_t * I)90 static inline int release_c(struct zm_mcs *L, zm_mcs_qnode_t *I) {
91 if (zm_atomic_load(&I->next, zm_memord_acquire) == ZM_NULL) {
92 zm_mcs_qnode_t *tmp = I;
93 if(zm_atomic_compare_exchange_strong(&L->lock,
94 (zm_ptr_t*)&tmp,
95 ZM_NULL,
96 zm_memord_acq_rel,
97 zm_memord_acquire))
98 return 0;
99 while(zm_atomic_load(&I->next, zm_memord_acquire) == ZM_NULL)
100 ; /* SPIN */
101 }
102 zm_atomic_store(&((zm_mcs_qnode_t*)zm_atomic_load(&I->next, zm_memord_acquire))->status, ZM_UNLOCKED, zm_memord_release);
103 return 0;
104 }
105
nowaiters_c(struct zm_mcs * L,zm_mcs_qnode_t * I)106 static inline int nowaiters_c(struct zm_mcs *L, zm_mcs_qnode_t *I) {
107 return (zm_atomic_load(&I->next, zm_memord_acquire) == ZM_NULL);
108 }
109
110 /* Context-less API */
mcs_acquire(struct zm_mcs * L)111 static inline int mcs_acquire(struct zm_mcs *L) {
112 if (zm_unlikely(tid == -1)) {
113 check_affinity(L->topo);
114 tid = get_hwthread_id(L->topo);
115 }
116 acquire_c(L, &L->local_nodes[tid]);
117 return 0;
118 }
119
mcs_tryacq(struct zm_mcs * L,int * success)120 static inline int mcs_tryacq(struct zm_mcs *L, int *success) {
121 if (zm_unlikely(tid == -1)) {
122 check_affinity(L->topo);
123 tid = get_hwthread_id(L->topo);
124 }
125 return tryacq_c(L, &L->local_nodes[tid], success);
126 }
127
mcs_release(struct zm_mcs * L)128 static inline int mcs_release(struct zm_mcs *L) {
129 assert(tid >= 0);
130 return release_c(L, &L->local_nodes[tid]);
131 }
132
mcs_nowaiters(struct zm_mcs * L)133 static inline int mcs_nowaiters(struct zm_mcs *L) {
134 assert(tid >= 0);
135 return nowaiters_c(L, &L->local_nodes[tid]);
136 }
137
138 /* Context-full API */
mcs_acquire_c(struct zm_mcs * L,zm_mcs_qnode_t * I)139 static inline int mcs_acquire_c(struct zm_mcs *L, zm_mcs_qnode_t* I) {
140 return acquire_c(L, I);
141 }
142
mcs_tryacq_c(struct zm_mcs * L,zm_mcs_qnode_t * I,int * success)143 int mcs_tryacq_c(struct zm_mcs *L, zm_mcs_qnode_t* I, int *success) {
144 return tryacq_c(L, I, success);
145 }
146
147
mcs_release_c(struct zm_mcs * L,zm_mcs_qnode_t * I)148 static inline int mcs_release_c(struct zm_mcs *L, zm_mcs_qnode_t *I) {
149 return release_c(L, I);
150 }
151
mcs_nowaiters_c(struct zm_mcs * L,zm_mcs_qnode_t * I)152 static inline int mcs_nowaiters_c(struct zm_mcs *L, zm_mcs_qnode_t *I) {
153 return nowaiters_c(L, I);
154 }
155
free_lock(struct zm_mcs * L)156 static inline int free_lock(struct zm_mcs *L)
157 {
158 free(L->local_nodes);
159 hwloc_topology_destroy(L->topo);
160 return 0;
161 }
162
163
zm_mcs_init(zm_mcs_t * handle)164 int zm_mcs_init(zm_mcs_t *handle) {
165 void *p = new_lock();
166 *handle = (zm_mcs_t) p;
167 return 0;
168 }
169
zm_mcs_destroy(zm_mcs_t * L)170 int zm_mcs_destroy(zm_mcs_t *L) {
171 free_lock((struct zm_mcs*)(*L));
172 return 0;
173 }
174
175
176 /* Context-less API */
zm_mcs_acquire(zm_mcs_t L)177 int zm_mcs_acquire(zm_mcs_t L) {
178 /*
179 It is prohibited to convert intptr_t (=zm_mcs_t) to a non-void pointer.
180 Converting intptr_t to void*, then void* to any pointer type is permitted.
181 cf. https://stackoverflow.com/questions/34291377/converting-a-non-void-pointer-to-uintptr-t-and-vice-versa
182 */
183 return mcs_acquire((struct zm_mcs*)(void *)L) ;
184 }
185
zm_mcs_tryacq(zm_mcs_t L,int * success)186 int zm_mcs_tryacq(zm_mcs_t L, int *success) {
187 return mcs_tryacq((struct zm_mcs*)(void *)L, success) ;
188 }
189
zm_mcs_release(zm_mcs_t L)190 int zm_mcs_release(zm_mcs_t L) {
191 return mcs_release((struct zm_mcs*)(void *)L) ;
192 }
193
zm_mcs_nowaiters(zm_mcs_t L)194 int zm_mcs_nowaiters(zm_mcs_t L) {
195 return mcs_nowaiters((struct zm_mcs*)(void *)L) ;
196 }
197
198 /* Context-full API */
zm_mcs_acquire_c(zm_mcs_t L,zm_mcs_qnode_t * I)199 int zm_mcs_acquire_c(zm_mcs_t L, zm_mcs_qnode_t* I) {
200 return mcs_acquire_c((struct zm_mcs*)(void *)L, I) ;
201 }
202
zm_mcs_tryacq_c(zm_mcs_t L,zm_mcs_qnode_t * I,int * success)203 int zm_mcs_tryacq_c(zm_mcs_t L, zm_mcs_qnode_t* I, int *success) {
204 return mcs_tryacq_c((struct zm_mcs*)(void *)L, I, success) ;
205 }
206
zm_mcs_release_c(zm_mcs_t L,zm_mcs_qnode_t * I)207 int zm_mcs_release_c(zm_mcs_t L, zm_mcs_qnode_t *I) {
208 return mcs_release_c((struct zm_mcs*)(void *)L, I) ;
209 }
210
zm_mcs_nowaiters_c(zm_mcs_t L,zm_mcs_qnode_t * I)211 int zm_mcs_nowaiters_c(zm_mcs_t L, zm_mcs_qnode_t *I) {
212 return mcs_nowaiters_c((struct zm_mcs*)(void *)L, I) ;
213 }
214