1 /*
2
3 Copyright (c) 2015 Martin Sustrik
4
5 Permission is hereby granted, free of charge, to any person obtaining a copy
6 of this software and associated documentation files (the "Software"),
7 to deal in the Software without restriction, including without limitation
8 the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 and/or sell copies of the Software, and to permit persons to whom
10 the Software is furnished to do so, subject to the following conditions:
11
12 The above copyright notice and this permission notice shall be included
13 in all copies or substantial portions of the Software.
14
15 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 IN THE SOFTWARE.
22
23 */
24
25 #include <errno.h>
26 #include <stddef.h>
27 #include <stdio.h>
28
29 #if defined MILL_VALGRIND
30 #include <valgrind/valgrind.h>
31 #endif
32
33 #include "cr.h"
34 #include "debug.h"
35 #include "libmill.h"
36 #include "poller.h"
37 #include "stack.h"
38 #include "utils.h"
39
40 /* Size of the buffer for temporary storage of values received from channels.
41 It should be properly aligned and never change if there are any stacks
42 allocated at the moment. */
43 size_t mill_valbuf_size = 128;
44
45 /* Valbuf for tha main coroutine. */
46 char mill_main_valbuf[128];
47
48 volatile int mill_unoptimisable1_ = 1;
49 volatile void *mill_unoptimisable2_ = NULL;
50
51 struct mill_cr mill_main = {0};
52
53 struct mill_cr *mill_running = &mill_main;
54
55 /* Queue of coroutines scheduled for execution. */
56 struct mill_slist mill_ready = {0};
57
mill_getctx_(void)58 inline mill_ctx mill_getctx_(void) {
59 #if defined __x86_64__
60 return mill_running->ctx;
61 #else
62 return &mill_running->ctx;
63 #endif
64 }
65
mill_getvalbuf(struct mill_cr * cr,size_t size)66 static void *mill_getvalbuf(struct mill_cr *cr, size_t size) {
67 /* Small valbufs don't require dynamic allocation. Also note that main
68 coroutine doesn't have a stack allocated on the heap like other
69 coroutines, so we have to handle valbuf in a special way. */
70 if(mill_fast(cr != &mill_main)) {
71 if(mill_fast(size <= mill_valbuf_size))
72 return (void*)(((char*)cr) - mill_valbuf_size);
73 }
74 else {
75 if(mill_fast(size <= sizeof(mill_main_valbuf)))
76 return (void*)mill_main_valbuf;
77 }
78 /* Large valbufs are simply allocated on heap. */
79 if(mill_fast(cr->valbuf && cr->valbuf_sz <= size))
80 return cr->valbuf;
81 void *ptr = realloc(cr->valbuf, size);
82 if(!ptr)
83 return NULL;
84 cr->valbuf = ptr;
85 cr->valbuf_sz = size;
86 return cr->valbuf;
87 }
88
mill_goprepare_(int count,size_t stack_size,size_t val_size)89 void mill_goprepare_(int count, size_t stack_size, size_t val_size) {
90 if(mill_slow(mill_hascrs())) {errno = EAGAIN; return;}
91 /* Allocate any resources needed by the polling mechanism. */
92 mill_poller_init();
93 if(mill_slow(errno != 0)) return;
94 /* If needed, make val size slightly bigger to align properly. */
95 mill_valbuf_size = (val_size + 15) & ~((size_t)0xf);
96 /* Preallocate the valbuf for the main coroutine. */
97 if(mill_slow(!mill_getvalbuf(&mill_main, mill_valbuf_size))) {
98 errno = ENOMEM; return;}
99 /* Allocate the stacks. */
100 mill_preparestacks(count, stack_size + mill_valbuf_size +
101 sizeof(struct mill_cr));
102 }
103
mill_suspend(void)104 int mill_suspend(void) {
105 /* Even if process never gets idle, we have to process external events
106 once in a while. The external signal may very well be a deadline or
107 a user-issued command that cancels the CPU intensive operation. */
108 static int counter = 0;
109 if(counter >= 103) {
110 mill_wait(0);
111 counter = 0;
112 }
113 /* Store the context of the current coroutine, if any. */
114 if(mill_running) {
115 mill_ctx ctx = mill_getctx_();
116 if (mill_setjmp_(ctx))
117 return mill_running->result;
118 }
119 while(1) {
120 /* If there's a coroutine ready to be executed go for it. */
121 if(!mill_slist_empty(&mill_ready)) {
122 ++counter;
123 struct mill_slist_item *it = mill_slist_pop(&mill_ready);
124 mill_running = mill_cont(it, struct mill_cr, ready);
125 mill_assert(mill_running->is_ready == 1);
126 mill_running->is_ready = 0;
127 mill_longjmp_(mill_getctx_());
128 }
129 /* Otherwise, we are going to wait for sleeping coroutines
130 and for external events. */
131 mill_wait(1);
132 mill_assert(!mill_slist_empty(&mill_ready));
133 counter = 0;
134 }
135 }
136
mill_resume(struct mill_cr * cr,int result)137 inline void mill_resume(struct mill_cr *cr, int result) {
138 mill_assert(!cr->is_ready);
139 cr->result = result;
140 cr->state = MILL_READY;
141 cr->is_ready = 1;
142 mill_slist_push_back(&mill_ready, &cr->ready);
143 }
144
145 /* mill_prologue_() and mill_epilogue_() live in the same scope with
146 libdill's stack-switching black magic. As such, they are extremely
147 fragile. Therefore, the optimiser is prohibited to touch them. */
148 #if defined __clang__
149 #define dill_noopt __attribute__((optnone))
150 #elif defined __GNUC__
151 #define dill_noopt __attribute__((optimize("O0")))
152 #else
153 #error "Unsupported compiler!"
154 #endif
155
156 /* The intial part of go(). Starts the new coroutine.
157 Returns the pointer to the top of its stack. */
158 __attribute__((noinline)) dill_noopt
mill_prologue_(const char * created)159 void *mill_prologue_(const char *created) {
160 /* Ensure that debug functions are available whenever a single go()
161 statement is present in the user's code. */
162 mill_preserve_debug();
163 /* Allocate and initialise new stack. */
164 #if defined MILL_VALGRIND
165 size_t stack_size;
166 struct mill_cr *cr = ((struct mill_cr*)mill_allocstack(&stack_size));
167 int sid = VALGRIND_STACK_REGISTER(((char*)cr) - stack_size, cr);
168 --cr;
169 cr->sid = sid;
170 #else
171 struct mill_cr *cr = ((struct mill_cr*)mill_allocstack(NULL)) - 1;
172 #endif
173 mill_register_cr(&cr->debug, created);
174 cr->is_ready = 0;
175 cr->valbuf = NULL;
176 cr->valbuf_sz = 0;
177 cr->clsval = NULL;
178 cr->timer.expiry = -1;
179 cr->fd = -1;
180 cr->events = 0;
181 mill_trace(created, "{%d}=go()", (int)cr->debug.id);
182 /* Suspend the parent coroutine and make the new one running. */
183 mill_resume(mill_running, 0);
184 mill_running = cr;
185 /* Return pointer to the top of the stack. There's valbuf interposed
186 between the mill_cr structure and the stack itself. */
187 return (void*)(((char*)cr) - mill_valbuf_size);
188 }
189
190 /* The final part of go(). Cleans up after the coroutine is finished. */
191 __attribute__((noinline)) dill_noopt
mill_epilogue_(void)192 void mill_epilogue_(void) {
193 mill_trace(NULL, "go() done");
194 mill_unregister_cr(&mill_running->debug);
195 if(mill_running->valbuf)
196 free(mill_running->valbuf);
197 #if defined MILL_VALGRIND
198 VALGRIND_STACK_DEREGISTER(mill_running->sid);
199 #endif
200 mill_freestack(mill_running + 1);
201 mill_running = NULL;
202 /* Given that there's no running coroutine at this point
203 this call will never return. */
204 mill_suspend();
205 }
206
mill_yield_(const char * current)207 void mill_yield_(const char *current) {
208 mill_trace(current, "yield()");
209 mill_set_current(&mill_running->debug, current);
210 /* This looks fishy, but yes, we can resume the coroutine even before
211 suspending it. */
212 mill_resume(mill_running, 0);
213 mill_suspend();
214 }
215
mill_valbuf(struct mill_cr * cr,size_t size)216 void *mill_valbuf(struct mill_cr *cr, size_t size) {
217 void *ptr = mill_getvalbuf(cr, size);
218 if(!ptr)
219 mill_panic("not enough memory to receive from channel");
220 return ptr;
221 }
222
mill_cls_(void)223 void *mill_cls_(void) {
224 return mill_running->clsval;
225 }
226
mill_setcls_(void * val)227 void mill_setcls_(void *val) {
228 mill_running->clsval = val;
229 }
230
mill_cr_postfork(void)231 void mill_cr_postfork(void) {
232 /* Drop all coroutines in the "ready to execute" list. */
233 mill_slist_init(&mill_ready);
234 }
235
236