1 /**********************************************************************
2
3 mjit_worker.c - Worker for MRI method JIT compiler
4
5 Copyright (C) 2017 Vladimir Makarov <vmakarov@redhat.com>.
6
7 **********************************************************************/
8
9 /* NOTE: All functions in this file are executed on MJIT worker. So don't
10 call Ruby methods (C functions that may call rb_funcall) or trigger
11 GC (using ZALLOC, xmalloc, xfree, etc.) in this file. */
12
13 /* We utilize widely used C compilers (GCC and LLVM Clang) to
14 implement MJIT. We feed them a C code generated from ISEQ. The
15 industrial C compilers are slower than regular JIT engines.
16 Generated code performance of the used C compilers has a higher
17 priority over the compilation speed.
18
19 So our major goal is to minimize the ISEQ compilation time when we
20 use widely optimization level (-O2). It is achieved by
21
22 o Using a precompiled version of the header
23 o Keeping all files in `/tmp`. On modern Linux `/tmp` is a file
24 system in memory. So it is pretty fast
25 o Implementing MJIT as a multi-threaded code because we want to
26 compile ISEQs in parallel with iseq execution to speed up Ruby
27 code execution. MJIT has one thread (*worker*) to do
28 parallel compilations:
29 o It prepares a precompiled code of the minimized header.
30 It starts at the MRI execution start
31 o It generates PIC object files of ISEQs
32 o It takes one JIT unit from a priority queue unless it is empty.
33 o It translates the JIT unit ISEQ into C-code using the precompiled
34 header, calls CC and load PIC code when it is ready
35 o Currently MJIT put ISEQ in the queue when ISEQ is called
36 o MJIT can reorder ISEQs in the queue if some ISEQ has been called
37 many times and its compilation did not start yet
38 o MRI reuses the machine code if it already exists for ISEQ
39 o The machine code we generate can stop and switch to the ISEQ
40 interpretation if some condition is not satisfied as the machine
41 code can be speculative or some exception raises
42 o Speculative machine code can be canceled.
43
44 Here is a diagram showing the MJIT organization:
45
46 _______
47 |header |
48 |_______|
49 | MRI building
50 --------------|----------------------------------------
51 | MRI execution
52 |
53 _____________|_____
54 | | |
55 | ___V__ | CC ____________________
56 | | |----------->| precompiled header |
57 | | | | |____________________|
58 | | | | |
59 | | MJIT | | |
60 | | | | |
61 | | | | ____V___ CC __________
62 | |______|----------->| C code |--->| .so file |
63 | | |________| |__________|
64 | | |
65 | | |
66 | MRI machine code |<-----------------------------
67 |___________________| loading
68
69 */
70
71 #ifdef __sun
72 #define __EXTENSIONS__ 1
73 #endif
74
75 #include "vm_core.h"
76 #include "mjit.h"
77 #include "gc.h"
78 #include "ruby_assert.h"
79 #include "ruby/debug.h"
80 #include "ruby/thread.h"
81
82 #ifdef _WIN32
83 #include <winsock2.h>
84 #include <windows.h>
85 #else
86 #include <sys/wait.h>
87 #include <sys/time.h>
88 #include <dlfcn.h>
89 #endif
90 #include <errno.h>
91 #ifdef HAVE_FCNTL_H
92 #include <fcntl.h>
93 #endif
94 #ifdef HAVE_SYS_PARAM_H
95 # include <sys/param.h>
96 #endif
97 #include "dln.h"
98
99 #include "ruby/util.h"
100 #undef strdup /* ruby_strdup may trigger GC */
101
102 #ifndef MAXPATHLEN
103 # define MAXPATHLEN 1024
104 #endif
105
106 #ifdef _WIN32
107 #define dlopen(name,flag) ((void*)LoadLibrary(name))
108 #define dlerror() strerror(rb_w32_map_errno(GetLastError()))
109 #define dlsym(handle,name) ((void*)GetProcAddress((handle),(name)))
110 #define dlclose(handle) (!FreeLibrary(handle))
111 #define RTLD_NOW -1
112
113 #define waitpid(pid,stat_loc,options) (WaitForSingleObject((HANDLE)(pid), INFINITE), GetExitCodeProcess((HANDLE)(pid), (LPDWORD)(stat_loc)), CloseHandle((HANDLE)pid), (pid))
114 #define WIFEXITED(S) ((S) != STILL_ACTIVE)
115 #define WEXITSTATUS(S) (S)
116 #define WIFSIGNALED(S) (0)
117 typedef intptr_t pid_t;
118 #endif
119
120 /* Atomically set function pointer if possible. */
121 #define MJIT_ATOMIC_SET(var, val) (void)ATOMIC_PTR_EXCHANGE(var, val)
122
123 #define MJIT_TMP_PREFIX "_ruby_mjit_"
124
125 /* The unit structure that holds metadata of ISeq for MJIT. */
126 struct rb_mjit_unit {
127 /* Unique order number of unit. */
128 int id;
129 /* Dlopen handle of the loaded object file. */
130 void *handle;
131 const rb_iseq_t *iseq;
132 #ifndef _MSC_VER
133 /* This value is always set for `compact_all_jit_code`. Also used for lazy deletion. */
134 char *o_file;
135 /* TRUE if it's inherited from parent Ruby process and lazy deletion should be skipped.
136 `o_file = NULL` can't be used to skip lazy deletion because `o_file` could be used
137 by child for `compact_all_jit_code`. */
138 int o_file_inherited_p;
139 #endif
140 #if defined(_WIN32)
141 /* DLL cannot be removed while loaded on Windows. If this is set, it'll be lazily deleted. */
142 char *so_file;
143 #endif
144 /* Only used by unload_units. Flag to check this unit is currently on stack or not. */
145 char used_code_p;
146 struct list_node unode;
147 };
148
149 /* Linked list of struct rb_mjit_unit. */
150 struct rb_mjit_unit_list {
151 struct list_head head;
152 int length; /* the list length */
153 };
154
155 extern void rb_native_mutex_lock(rb_nativethread_lock_t *lock);
156 extern void rb_native_mutex_unlock(rb_nativethread_lock_t *lock);
157 extern void rb_native_mutex_initialize(rb_nativethread_lock_t *lock);
158 extern void rb_native_mutex_destroy(rb_nativethread_lock_t *lock);
159
160 extern void rb_native_cond_initialize(rb_nativethread_cond_t *cond);
161 extern void rb_native_cond_destroy(rb_nativethread_cond_t *cond);
162 extern void rb_native_cond_signal(rb_nativethread_cond_t *cond);
163 extern void rb_native_cond_broadcast(rb_nativethread_cond_t *cond);
164 extern void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex);
165
166 /* process.c */
167 extern rb_pid_t ruby_waitpid_locked(rb_vm_t *, rb_pid_t, int *status, int options, rb_nativethread_cond_t *cond);
168
169 /* A copy of MJIT portion of MRI options since MJIT initialization. We
170 need them as MJIT threads still can work when the most MRI data were
171 freed. */
172 struct mjit_options mjit_opts;
173
174 /* TRUE if MJIT is enabled. */
175 int mjit_enabled = FALSE;
176 /* TRUE if JIT-ed code should be called. When `ruby_vm_event_enabled_global_flags & ISEQ_TRACE_EVENTS`
177 and `mjit_call_p == FALSE`, any JIT-ed code execution is cancelled as soon as possible. */
178 int mjit_call_p = FALSE;
179
180 /* Priority queue of iseqs waiting for JIT compilation.
181 This variable is a pointer to head unit of the queue. */
182 static struct rb_mjit_unit_list unit_queue = { LIST_HEAD_INIT(unit_queue.head) };
183 /* List of units which are successfully compiled. */
184 static struct rb_mjit_unit_list active_units = { LIST_HEAD_INIT(active_units.head) };
185 /* List of compacted so files which will be cleaned up by `free_list()` in `mjit_finish()`. */
186 static struct rb_mjit_unit_list compact_units = { LIST_HEAD_INIT(compact_units.head) };
187 /* The number of so far processed ISEQs, used to generate unique id. */
188 static int current_unit_num;
189 /* A mutex for conitionals and critical sections. */
190 static rb_nativethread_lock_t mjit_engine_mutex;
191 /* A thread conditional to wake up `mjit_finish` at the end of PCH thread. */
192 static rb_nativethread_cond_t mjit_pch_wakeup;
193 /* A thread conditional to wake up the client if there is a change in
194 executed unit status. */
195 static rb_nativethread_cond_t mjit_client_wakeup;
196 /* A thread conditional to wake up a worker if there we have something
197 to add or we need to stop MJIT engine. */
198 static rb_nativethread_cond_t mjit_worker_wakeup;
199 /* A thread conditional to wake up workers if at the end of GC. */
200 static rb_nativethread_cond_t mjit_gc_wakeup;
201 /* True when GC is working. */
202 static int in_gc;
203 /* True when JIT is working. */
204 static int in_jit;
205 /* Set to TRUE to stop worker. */
206 static int stop_worker_p;
207 /* Set to TRUE if worker is stopped. */
208 static int worker_stopped;
209
210 /* Path of "/tmp", which can be changed to $TMP in MinGW. */
211 static char *tmp_dir;
212 /* Hash like { 1 => true, 2 => true, ... } whose keys are valid `class_serial`s.
213 This is used to invalidate obsoleted CALL_CACHE. */
214 static VALUE valid_class_serials;
215
216 /* Used C compiler path. */
217 static const char *cc_path;
218 /* Used C compiler flags. */
219 static const char **cc_common_args;
220 /* Name of the precompiled header file. */
221 static char *pch_file;
222 /* The process id which should delete the pch_file on mjit_finish. */
223 static rb_pid_t pch_owner_pid;
224 /* Status of the precompiled header creation. The status is
225 shared by the workers and the pch thread. */
226 static enum {PCH_NOT_READY, PCH_FAILED, PCH_SUCCESS} pch_status;
227
228 #ifndef _MSC_VER
229 /* Name of the header file. */
230 static char *header_file;
231 #endif
232
233 #ifdef _WIN32
234 /* Linker option to enable libruby. */
235 static char *libruby_pathflag;
236 #endif
237
238 #include "mjit_config.h"
239
240 #if defined(__GNUC__) && \
241 (!defined(__clang__) || \
242 (defined(__clang__) && (defined(__FreeBSD__) || defined(__GLIBC__))))
243 # define GCC_PIC_FLAGS "-Wfatal-errors", "-fPIC", "-shared", "-w", "-pipe",
244 # define MJIT_CFLAGS_PIPE 1
245 #else
246 # define GCC_PIC_FLAGS /* empty */
247 # define MJIT_CFLAGS_PIPE 0
248 #endif
249
250 // Use `-nodefaultlibs -nostdlib` for GCC where possible, which does not work on mingw, cygwin, AIX, and OpenBSD.
251 // This seems to improve MJIT performance on GCC.
252 #if defined __GNUC__ && !defined __clang__ && !defined(_WIN32) && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__OpenBSD__)
253 # define GCC_NOSTDLIB_FLAGS "-nodefaultlibs", "-nostdlib",
254 #else
255 # define GCC_NOSTDLIB_FLAGS /* empty */
256 #endif
257
258 static const char *const CC_COMMON_ARGS[] = {
259 MJIT_CC_COMMON MJIT_CFLAGS GCC_PIC_FLAGS
260 NULL
261 };
262
263 static const char *const CC_DEBUG_ARGS[] = {MJIT_DEBUGFLAGS NULL};
264 static const char *const CC_OPTIMIZE_ARGS[] = {MJIT_OPTFLAGS NULL};
265
266 static const char *const CC_LDSHARED_ARGS[] = {MJIT_LDSHARED GCC_PIC_FLAGS NULL};
267 static const char *const CC_DLDFLAGS_ARGS[] = {
268 MJIT_DLDFLAGS
269 #if defined __GNUC__ && !defined __clang__ && !defined(__OpenBSD__)
270 "-nostartfiles",
271 #endif
272 GCC_NOSTDLIB_FLAGS NULL
273 };
274
275 static const char *const CC_LIBS[] = {
276 #if defined(_WIN32) || defined(__CYGWIN__)
277 MJIT_LIBS // mswin, mingw, cygwin
278 #endif
279 #if defined __GNUC__ && !defined __clang__
280 # if defined(_WIN32)
281 "-lmsvcrt", // mingw
282 # endif
283 "-lgcc", // mingw, cygwin, and GCC platforms using `-nodefaultlibs -nostdlib`
284 #endif
285 NULL
286 };
287
288 #define CC_CODEFLAG_ARGS (mjit_opts.debug ? CC_DEBUG_ARGS : CC_OPTIMIZE_ARGS)
289
290 /* Print the arguments according to FORMAT to stderr only if MJIT
291 verbose option value is more or equal to LEVEL. */
292 PRINTF_ARGS(static void, 2, 3)
verbose(int level,const char * format,...)293 verbose(int level, const char *format, ...)
294 {
295 if (mjit_opts.verbose >= level) {
296 va_list args;
297 size_t len = strlen(format);
298 char *full_format = alloca(sizeof(char) * (len + 2));
299
300 /* Creating `format + '\n'` to atomically print format and '\n'. */
301 memcpy(full_format, format, len);
302 full_format[len] = '\n';
303 full_format[len+1] = '\0';
304
305 va_start(args, format);
306 vfprintf(stderr, full_format, args);
307 va_end(args);
308 }
309 }
310
311 PRINTF_ARGS(static void, 1, 2)
mjit_warning(const char * format,...)312 mjit_warning(const char *format, ...)
313 {
314 if (mjit_opts.warnings || mjit_opts.verbose) {
315 va_list args;
316
317 fprintf(stderr, "MJIT warning: ");
318 va_start(args, format);
319 vfprintf(stderr, format, args);
320 va_end(args);
321 fprintf(stderr, "\n");
322 }
323 }
324
325 /* Add unit node to the tail of doubly linked LIST. It should be not in
326 the list before. */
327 static void
add_to_list(struct rb_mjit_unit * unit,struct rb_mjit_unit_list * list)328 add_to_list(struct rb_mjit_unit *unit, struct rb_mjit_unit_list *list)
329 {
330 list_add_tail(&list->head, &unit->unode);
331 list->length++;
332 }
333
334 static void
remove_from_list(struct rb_mjit_unit * unit,struct rb_mjit_unit_list * list)335 remove_from_list(struct rb_mjit_unit *unit, struct rb_mjit_unit_list *list)
336 {
337 list_del(&unit->unode);
338 list->length--;
339 }
340
341 static void
remove_file(const char * filename)342 remove_file(const char *filename)
343 {
344 if (remove(filename)) {
345 mjit_warning("failed to remove \"%s\": %s", filename, strerror(errno));
346 }
347 }
348
349 /* Lazily delete .o and/or .so files. */
350 static void
clean_object_files(struct rb_mjit_unit * unit)351 clean_object_files(struct rb_mjit_unit *unit)
352 {
353 #ifndef _MSC_VER
354 if (unit->o_file) {
355 char *o_file = unit->o_file;
356
357 unit->o_file = NULL;
358 /* For compaction, unit->o_file is always set when compilation succeeds.
359 So save_temps needs to be checked here. */
360 if (!mjit_opts.save_temps && !unit->o_file_inherited_p)
361 remove_file(o_file);
362 free(o_file);
363 }
364 #endif
365
366 #if defined(_WIN32)
367 if (unit->so_file) {
368 char *so_file = unit->so_file;
369
370 unit->so_file = NULL;
371 /* unit->so_file is set only when mjit_opts.save_temps is FALSE. */
372 remove_file(so_file);
373 free(so_file);
374 }
375 #endif
376 }
377
378 /* This is called in the following situations:
379 1) On dequeue or `unload_units()`, associated ISeq is already GCed.
380 2) The unit is not called often and unloaded by `unload_units()`.
381 3) Freeing lists on `mjit_finish()`.
382
383 `jit_func` value does not matter for 1 and 3 since the unit won't be used anymore.
384 For the situation 2, this sets the ISeq's JIT state to NOT_COMPILED_JIT_ISEQ_FUNC
385 to prevent the situation that the same methods are continously compiled. */
386 static void
free_unit(struct rb_mjit_unit * unit)387 free_unit(struct rb_mjit_unit *unit)
388 {
389 if (unit->iseq) { /* ISeq is not GCed */
390 unit->iseq->body->jit_func = (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
391 unit->iseq->body->jit_unit = NULL;
392 }
393 if (unit->handle && dlclose(unit->handle)) { /* handle is NULL if it's in queue */
394 mjit_warning("failed to close handle for u%d: %s", unit->id, dlerror());
395 }
396 clean_object_files(unit);
397 free(unit);
398 }
399
400 /* Start a critical section. Use message MSG to print debug info at
401 LEVEL. */
402 static inline void
CRITICAL_SECTION_START(int level,const char * msg)403 CRITICAL_SECTION_START(int level, const char *msg)
404 {
405 verbose(level, "Locking %s", msg);
406 rb_native_mutex_lock(&mjit_engine_mutex);
407 verbose(level, "Locked %s", msg);
408 }
409
410 /* Finish the current critical section. Use message MSG to print
411 debug info at LEVEL. */
412 static inline void
CRITICAL_SECTION_FINISH(int level,const char * msg)413 CRITICAL_SECTION_FINISH(int level, const char *msg)
414 {
415 verbose(level, "Unlocked %s", msg);
416 rb_native_mutex_unlock(&mjit_engine_mutex);
417 }
418
419 static int
sprint_uniq_filename(char * str,size_t size,unsigned long id,const char * prefix,const char * suffix)420 sprint_uniq_filename(char *str, size_t size, unsigned long id, const char *prefix, const char *suffix)
421 {
422 return snprintf(str, size, "%s/%sp%"PRI_PIDT_PREFIX"uu%lu%s", tmp_dir, prefix, getpid(), id, suffix);
423 }
424
425 /* Return time in milliseconds as a double. */
426 #ifdef __APPLE__
427 double ruby_real_ms_time(void);
428 # define real_ms_time() ruby_real_ms_time()
429 #else
430 static double
real_ms_time(void)431 real_ms_time(void)
432 {
433 # ifdef HAVE_CLOCK_GETTIME
434 struct timespec tv;
435 # ifdef CLOCK_MONOTONIC
436 const clockid_t c = CLOCK_MONOTONIC;
437 # else
438 const clockid_t c = CLOCK_REALTIME;
439 # endif
440
441 clock_gettime(c, &tv);
442 return tv.tv_nsec / 1000000.0 + tv.tv_sec * 1000.0;
443 # else
444 struct timeval tv;
445
446 gettimeofday(&tv, NULL);
447 return tv.tv_usec / 1000.0 + tv.tv_sec * 1000.0;
448 # endif
449 }
450 #endif
451
452 /* Return TRUE if class_serial is not obsoleted. This is used by mjit_compile.c. */
453 int
mjit_valid_class_serial_p(rb_serial_t class_serial)454 mjit_valid_class_serial_p(rb_serial_t class_serial)
455 {
456 int found_p;
457
458 CRITICAL_SECTION_START(3, "in valid_class_serial_p");
459 found_p = rb_hash_stlike_lookup(valid_class_serials, LONG2FIX(class_serial), NULL);
460 CRITICAL_SECTION_FINISH(3, "in valid_class_serial_p");
461 return found_p;
462 }
463
464 /* Return the best unit from list. The best is the first
465 high priority unit or the unit whose iseq has the biggest number
466 of calls so far. */
467 static struct rb_mjit_unit *
get_from_list(struct rb_mjit_unit_list * list)468 get_from_list(struct rb_mjit_unit_list *list)
469 {
470 struct rb_mjit_unit *unit = NULL, *next, *best = NULL;
471
472 /* Find iseq with max total_calls */
473 list_for_each_safe(&list->head, unit, next, unode) {
474 if (unit->iseq == NULL) { /* ISeq is GCed. */
475 remove_from_list(unit, list);
476 free_unit(unit);
477 continue;
478 }
479
480 if (best == NULL || best->iseq->body->total_calls < unit->iseq->body->total_calls) {
481 best = unit;
482 }
483 }
484 if (best) {
485 remove_from_list(best, list);
486 }
487 return best;
488 }
489
490 /* Return length of NULL-terminated array ARGS excluding the NULL
491 marker. */
492 static size_t
args_len(char * const * args)493 args_len(char *const *args)
494 {
495 size_t i;
496
497 for (i = 0; (args[i]) != NULL;i++)
498 ;
499 return i;
500 }
501
502 /* Concatenate NUM passed NULL-terminated arrays of strings, put the
503 result (with NULL end marker) into the heap, and return the
504 result. */
505 static char **
form_args(int num,...)506 form_args(int num, ...)
507 {
508 va_list argp;
509 size_t len, n;
510 int i;
511 char **args, **res, **tmp;
512
513 va_start(argp, num);
514 res = NULL;
515 for (i = len = 0; i < num; i++) {
516 args = va_arg(argp, char **);
517 n = args_len(args);
518 if ((tmp = (char **)realloc(res, sizeof(char *) * (len + n + 1))) == NULL) {
519 free(res);
520 return NULL;
521 }
522 res = tmp;
523 MEMCPY(res + len, args, char *, n + 1);
524 len += n;
525 }
526 va_end(argp);
527 return res;
528 }
529
530 COMPILER_WARNING_PUSH
531 #ifdef __GNUC__
532 COMPILER_WARNING_IGNORED(-Wdeprecated-declarations)
533 #endif
534 /* Start an OS process of absolute executable path with arguments ARGV.
535 Return PID of the process. */
536 static pid_t
start_process(const char * abspath,char * const * argv)537 start_process(const char *abspath, char *const *argv)
538 {
539 pid_t pid;
540 /*
541 * Not calling non-async-signal-safe functions between vfork
542 * and execv for safety
543 */
544 int dev_null = rb_cloexec_open(ruby_null_device, O_WRONLY, 0);
545
546 if (mjit_opts.verbose >= 2) {
547 int i;
548 const char *arg;
549
550 fprintf(stderr, "Starting process: %s", abspath);
551 for (i = 0; (arg = argv[i]) != NULL; i++)
552 fprintf(stderr, " %s", arg);
553 fprintf(stderr, "\n");
554 }
555 #ifdef _WIN32
556 {
557 extern HANDLE rb_w32_start_process(const char *abspath, char *const *argv, int out_fd);
558 int out_fd = 0;
559 if (mjit_opts.verbose <= 1) {
560 /* Discard cl.exe's outputs like:
561 _ruby_mjit_p12u3.c
562 Creating library C:.../_ruby_mjit_p12u3.lib and object C:.../_ruby_mjit_p12u3.exp */
563 out_fd = dev_null;
564 }
565
566 pid = (pid_t)rb_w32_start_process(abspath, argv, out_fd);
567 if (pid == 0) {
568 verbose(1, "MJIT: Failed to create process: %s", dlerror());
569 return -1;
570 }
571 }
572 #else
573 if ((pid = vfork()) == 0) { /* TODO: reuse some function in process.c */
574 umask(0077);
575 if (mjit_opts.verbose == 0) {
576 /* CC can be started in a thread using a file which has been
577 already removed while MJIT is finishing. Discard the
578 messages about missing files. */
579 dup2(dev_null, STDERR_FILENO);
580 dup2(dev_null, STDOUT_FILENO);
581 }
582 (void)close(dev_null);
583 pid = execv(abspath, argv); /* Pid will be negative on an error */
584 /* Even if we successfully found CC to compile PCH we still can
585 fail with loading the CC in very rare cases for some reasons.
586 Stop the forked process in this case. */
587 verbose(1, "MJIT: Error in execv: %s", abspath);
588 _exit(1);
589 }
590 #endif
591 (void)close(dev_null);
592 return pid;
593 }
594 COMPILER_WARNING_POP
595
596 /* Execute an OS process of executable PATH with arguments ARGV.
597 Return -1 or -2 if failed to execute, otherwise exit code of the process.
598 TODO: Use a similar function in process.c */
599 static int
exec_process(const char * path,char * const argv[])600 exec_process(const char *path, char *const argv[])
601 {
602 int stat, exit_code = -2;
603 pid_t pid;
604 rb_vm_t *vm = WAITPID_USE_SIGCHLD ? GET_VM() : 0;
605 rb_nativethread_cond_t cond;
606
607 if (vm) {
608 rb_native_cond_initialize(&cond);
609 rb_native_mutex_lock(&vm->waitpid_lock);
610 }
611
612 pid = start_process(path, argv);
613 for (;pid > 0;) {
614 pid_t r = vm ? ruby_waitpid_locked(vm, pid, &stat, 0, &cond)
615 : waitpid(pid, &stat, 0);
616 if (r == -1) {
617 if (errno == EINTR) continue;
618 fprintf(stderr, "[%"PRI_PIDT_PREFIX"d] waitpid(%lu): %s (SIGCHLD=%d,%u)\n",
619 getpid(), (unsigned long)pid, strerror(errno),
620 RUBY_SIGCHLD, SIGCHLD_LOSSY);
621 break;
622 }
623 else if (r == pid) {
624 if (WIFEXITED(stat)) {
625 exit_code = WEXITSTATUS(stat);
626 break;
627 } else if (WIFSIGNALED(stat)) {
628 exit_code = -1;
629 break;
630 }
631 }
632 }
633
634 if (vm) {
635 rb_native_mutex_unlock(&vm->waitpid_lock);
636 rb_native_cond_destroy(&cond);
637 }
638 return exit_code;
639 }
640
641 static void
remove_so_file(const char * so_file,struct rb_mjit_unit * unit)642 remove_so_file(const char *so_file, struct rb_mjit_unit *unit)
643 {
644 #if defined(_WIN32)
645 /* Windows can't remove files while it's used. */
646 unit->so_file = strdup(so_file); /* lazily delete on `clean_object_files()` */
647 if (unit->so_file == NULL)
648 mjit_warning("failed to allocate memory to lazily remove '%s': %s", so_file, strerror(errno));
649 #else
650 remove_file(so_file);
651 #endif
652 }
653
654 #define append_str2(p, str, len) ((char *)memcpy((p), str, (len))+(len))
655 #define append_str(p, str) append_str2(p, str, sizeof(str)-1)
656 #define append_lit(p, str) append_str2(p, str, rb_strlen_lit(str))
657
658 #ifdef _MSC_VER
659 /* Compile C file to so. It returns 1 if it succeeds. (mswin) */
660 static int
compile_c_to_so(const char * c_file,const char * so_file)661 compile_c_to_so(const char *c_file, const char *so_file)
662 {
663 int exit_code;
664 const char *files[] = { NULL, NULL, NULL, NULL, NULL, NULL, "-link", libruby_pathflag, NULL };
665 char **args;
666 char *p, *obj_file;
667
668 /* files[0] = "-Fe*.dll" */
669 files[0] = p = alloca(sizeof(char) * (rb_strlen_lit("-Fe") + strlen(so_file) + 1));
670 p = append_lit(p, "-Fe");
671 p = append_str2(p, so_file, strlen(so_file));
672 *p = '\0';
673
674 /* files[1] = "-Fo*.obj" */
675 /* We don't need .obj file, but it's somehow created to cwd without -Fo and we want to control the output directory. */
676 files[1] = p = alloca(sizeof(char) * (rb_strlen_lit("-Fo") + strlen(so_file) - rb_strlen_lit(DLEXT) + rb_strlen_lit(".obj") + 1));
677 obj_file = p = append_lit(p, "-Fo");
678 p = append_str2(p, so_file, strlen(so_file) - rb_strlen_lit(DLEXT));
679 p = append_lit(p, ".obj");
680 *p = '\0';
681
682 /* files[2] = "-Yu*.pch" */
683 files[2] = p = alloca(sizeof(char) * (rb_strlen_lit("-Yu") + strlen(pch_file) + 1));
684 p = append_lit(p, "-Yu");
685 p = append_str2(p, pch_file, strlen(pch_file));
686 *p = '\0';
687
688 /* files[3] = "C:/.../rb_mjit_header-*.obj" */
689 files[3] = p = alloca(sizeof(char) * (strlen(pch_file) + 1));
690 p = append_str2(p, pch_file, strlen(pch_file) - strlen(".pch"));
691 p = append_lit(p, ".obj");
692 *p = '\0';
693
694 /* files[4] = "-Tc*.c" */
695 files[4] = p = alloca(sizeof(char) * (rb_strlen_lit("-Tc") + strlen(c_file) + 1));
696 p = append_lit(p, "-Tc");
697 p = append_str2(p, c_file, strlen(c_file));
698 *p = '\0';
699
700 /* files[5] = "-Fd*.pdb" */
701 files[5] = p = alloca(sizeof(char) * (rb_strlen_lit("-Fd") + strlen(pch_file) + 1));
702 p = append_lit(p, "-Fd");
703 p = append_str2(p, pch_file, strlen(pch_file) - rb_strlen_lit(".pch"));
704 p = append_lit(p, ".pdb");
705 *p = '\0';
706
707 args = form_args(5, CC_LDSHARED_ARGS, CC_CODEFLAG_ARGS,
708 files, CC_LIBS, CC_DLDFLAGS_ARGS);
709 if (args == NULL)
710 return FALSE;
711
712 exit_code = exec_process(cc_path, args);
713 free(args);
714
715 if (exit_code == 0) {
716 /* remove never-used files (.obj, .lib, .exp, .pdb). XXX: Is there any way not to generate this? */
717 if (!mjit_opts.save_temps) {
718 char *before_dot;
719 remove_file(obj_file);
720
721 before_dot = obj_file + strlen(obj_file) - rb_strlen_lit(".obj");
722 append_lit(before_dot, ".lib"); remove_file(obj_file);
723 append_lit(before_dot, ".exp"); remove_file(obj_file);
724 append_lit(before_dot, ".pdb"); remove_file(obj_file);
725 }
726 }
727 else {
728 verbose(2, "compile_c_to_so: compile error: %d", exit_code);
729 }
730 return exit_code == 0;
731 }
732 #else /* _MSC_VER */
733
734 /* The function producing the pre-compiled header. */
735 static void
make_pch(void)736 make_pch(void)
737 {
738 int exit_code;
739 const char *rest_args[] = {
740 # ifdef __clang__
741 "-emit-pch",
742 # endif
743 // -nodefaultlibs is a linker flag, but it may affect cc1 behavior on Gentoo, which should NOT be changed on pch:
744 // https://gitweb.gentoo.org/proj/gcc-patches.git/tree/7.3.0/gentoo/13_all_default-ssp-fix.patch
745 GCC_NOSTDLIB_FLAGS
746 "-o", NULL, NULL,
747 NULL,
748 };
749 char **args;
750 int len = sizeof(rest_args) / sizeof(const char *);
751
752 rest_args[len - 2] = header_file;
753 rest_args[len - 3] = pch_file;
754 verbose(2, "Creating precompiled header");
755 args = form_args(3, cc_common_args, CC_CODEFLAG_ARGS, rest_args);
756 if (args == NULL) {
757 mjit_warning("making precompiled header failed on forming args");
758 CRITICAL_SECTION_START(3, "in make_pch");
759 pch_status = PCH_FAILED;
760 CRITICAL_SECTION_FINISH(3, "in make_pch");
761 return;
762 }
763
764 exit_code = exec_process(cc_path, args);
765 free(args);
766
767 CRITICAL_SECTION_START(3, "in make_pch");
768 if (exit_code == 0) {
769 pch_status = PCH_SUCCESS;
770 }
771 else {
772 mjit_warning("Making precompiled header failed on compilation. Stopping MJIT worker...");
773 pch_status = PCH_FAILED;
774 }
775 /* wakeup `mjit_finish` */
776 rb_native_cond_broadcast(&mjit_pch_wakeup);
777 CRITICAL_SECTION_FINISH(3, "in make_pch");
778 }
779
780 /* Compile .c file to .o file. It returns 1 if it succeeds. (non-mswin) */
781 static int
compile_c_to_o(const char * c_file,const char * o_file)782 compile_c_to_o(const char *c_file, const char *o_file)
783 {
784 int exit_code;
785 const char *files[] = {
786 "-o", NULL, NULL,
787 # ifdef __clang__
788 "-include-pch", NULL,
789 # endif
790 "-c", NULL
791 };
792 char **args;
793
794 files[1] = o_file;
795 files[2] = c_file;
796 # ifdef __clang__
797 files[4] = pch_file;
798 # endif
799 args = form_args(5, cc_common_args, CC_CODEFLAG_ARGS, files, CC_LIBS, CC_DLDFLAGS_ARGS);
800 if (args == NULL)
801 return FALSE;
802
803 exit_code = exec_process(cc_path, args);
804 free(args);
805
806 if (exit_code != 0)
807 verbose(2, "compile_c_to_o: compile error: %d", exit_code);
808 return exit_code == 0;
809 }
810
811 /* Link .o files to .so file. It returns 1 if it succeeds. (non-mswin) */
812 static int
link_o_to_so(const char ** o_files,const char * so_file)813 link_o_to_so(const char **o_files, const char *so_file)
814 {
815 int exit_code;
816 const char *options[] = {
817 "-o", NULL,
818 # ifdef _WIN32
819 libruby_pathflag,
820 # endif
821 NULL
822 };
823 char **args;
824
825 options[1] = so_file;
826 args = form_args(6, CC_LDSHARED_ARGS, CC_CODEFLAG_ARGS,
827 options, o_files, CC_LIBS, CC_DLDFLAGS_ARGS);
828 if (args == NULL)
829 return FALSE;
830
831 exit_code = exec_process(cc_path, args);
832 free(args);
833
834 if (exit_code != 0)
835 verbose(2, "link_o_to_so: link error: %d", exit_code);
836 return exit_code == 0;
837 }
838
839 /* Link all cached .o files and build a .so file. Reload all JIT func from it. This
840 allows to avoid JIT code fragmentation and improve performance to call JIT-ed code. */
841 static void
compact_all_jit_code(void)842 compact_all_jit_code(void)
843 {
844 # ifndef _WIN32 /* This requires header transformation but we don't transform header on Windows for now */
845 struct rb_mjit_unit *unit, *cur = 0;
846 double start_time, end_time;
847 static const char so_ext[] = DLEXT;
848 char so_file[MAXPATHLEN];
849 const char **o_files;
850 int i = 0, success;
851
852 /* Abnormal use case of rb_mjit_unit that doesn't have ISeq */
853 unit = calloc(1, sizeof(struct rb_mjit_unit)); /* To prevent GC, don't use ZALLOC */
854 if (unit == NULL) return;
855 unit->id = current_unit_num++;
856 sprint_uniq_filename(so_file, (int)sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
857
858 /* NULL-ending for form_args */
859 o_files = alloca(sizeof(char *) * (active_units.length + 1));
860 o_files[active_units.length] = NULL;
861 CRITICAL_SECTION_START(3, "in compact_all_jit_code to keep .o files");
862 list_for_each(&active_units.head, cur, unode) {
863 o_files[i] = cur->o_file;
864 i++;
865 }
866
867 start_time = real_ms_time();
868 success = link_o_to_so(o_files, so_file);
869 end_time = real_ms_time();
870
871 /* TODO: Shrink this big critical section. For now, this is needed to prevent failure by missing .o files.
872 This assumes that o -> so link doesn't take long time because the bottleneck, which is compiler optimization,
873 is already done. But actually it takes about 500ms for 5,000 methods on my Linux machine, so it's better to
874 finish this critical section before link_o_to_so by disabling unload_units. */
875 CRITICAL_SECTION_FINISH(3, "in compact_all_jit_code to keep .o files");
876
877 if (success) {
878 void *handle = dlopen(so_file, RTLD_NOW);
879 if (handle == NULL) {
880 mjit_warning("failure in loading code from compacted '%s': %s", so_file, dlerror());
881 free(unit);
882 return;
883 }
884 unit->handle = handle;
885
886 /* lazily dlclose handle (and .so file for win32) on `mjit_finish()`. */
887 add_to_list(unit, &compact_units);
888
889 if (!mjit_opts.save_temps)
890 remove_so_file(so_file, unit);
891
892 CRITICAL_SECTION_START(3, "in compact_all_jit_code to read list");
893 list_for_each(&active_units.head, cur, unode) {
894 void *func;
895 char funcname[35]; /* TODO: reconsider `35` */
896 sprintf(funcname, "_mjit%d", cur->id);
897
898 if ((func = dlsym(handle, funcname)) == NULL) {
899 mjit_warning("skipping to reload '%s' from '%s': %s", funcname, so_file, dlerror());
900 continue;
901 }
902
903 if (cur->iseq) { /* Check whether GCed or not */
904 /* Usage of jit_code might be not in a critical section. */
905 MJIT_ATOMIC_SET(cur->iseq->body->jit_func, (mjit_func_t)func);
906 }
907 }
908 CRITICAL_SECTION_FINISH(3, "in compact_all_jit_code to read list");
909 verbose(1, "JIT compaction (%.1fms): Compacted %d methods -> %s", end_time - start_time, active_units.length, so_file);
910 }
911 else {
912 free(unit);
913 verbose(1, "JIT compaction failure (%.1fms): Failed to compact methods", end_time - start_time);
914 }
915 # endif /* _WIN32 */
916 }
917
918 #endif /* _MSC_VER */
919
920 static void *
load_func_from_so(const char * so_file,const char * funcname,struct rb_mjit_unit * unit)921 load_func_from_so(const char *so_file, const char *funcname, struct rb_mjit_unit *unit)
922 {
923 void *handle, *func;
924
925 handle = dlopen(so_file, RTLD_NOW);
926 if (handle == NULL) {
927 mjit_warning("failure in loading code from '%s': %s", so_file, dlerror());
928 return (void *)NOT_ADDED_JIT_ISEQ_FUNC;
929 }
930
931 func = dlsym(handle, funcname);
932 unit->handle = handle;
933 return func;
934 }
935
936 static void
print_jit_result(const char * result,const struct rb_mjit_unit * unit,const double duration,const char * c_file)937 print_jit_result(const char *result, const struct rb_mjit_unit *unit, const double duration, const char *c_file)
938 {
939 verbose(1, "JIT %s (%.1fms): %s@%s:%d -> %s", result,
940 duration, RSTRING_PTR(unit->iseq->body->location.label),
941 RSTRING_PTR(rb_iseq_path(unit->iseq)), FIX2INT(unit->iseq->body->location.first_lineno), c_file);
942 }
943
944 #ifndef __clang__
945 static const char *
header_name_end(const char * s)946 header_name_end(const char *s)
947 {
948 const char *e = s + strlen(s);
949 # ifdef __GNUC__ /* don't chomp .pch for mswin */
950 static const char suffix[] = ".gch";
951
952 /* chomp .gch suffix */
953 if (e > s+sizeof(suffix)-1 && strcmp(e-sizeof(suffix)+1, suffix) == 0) {
954 e -= sizeof(suffix)-1;
955 }
956 # endif
957 return e;
958 }
959 #endif
960
961 /* Print platform-specific prerequisites in generated code. */
962 static void
compile_prelude(FILE * f)963 compile_prelude(FILE *f)
964 {
965 #ifndef __clang__ /* -include-pch is used for Clang */
966 const char *s = pch_file;
967 const char *e = header_name_end(s);
968
969 fprintf(f, "#include \"");
970 /* print pch_file except .gch for gcc, but keep .pch for mswin */
971 for (; s < e; s++) {
972 switch(*s) {
973 case '\\': case '"':
974 fputc('\\', f);
975 }
976 fputc(*s, f);
977 }
978 fprintf(f, "\"\n");
979 #endif
980
981 #ifdef _WIN32
982 fprintf(f, "void _pei386_runtime_relocator(void){}\n");
983 fprintf(f, "int __stdcall DllMainCRTStartup(void* hinstDLL, unsigned int fdwReason, void* lpvReserved) { return 1; }\n");
984 #endif
985 }
986
987 /* Compile ISeq in UNIT and return function pointer of JIT-ed code.
988 It may return NOT_COMPILED_JIT_ISEQ_FUNC if something went wrong. */
989 static mjit_func_t
convert_unit_to_func(struct rb_mjit_unit * unit,struct rb_call_cache * cc_entries,union iseq_inline_storage_entry * is_entries)990 convert_unit_to_func(struct rb_mjit_unit *unit, struct rb_call_cache *cc_entries, union iseq_inline_storage_entry *is_entries)
991 {
992 char c_file_buff[MAXPATHLEN], *c_file = c_file_buff, *so_file, funcname[35]; /* TODO: reconsider `35` */
993 int success;
994 int fd;
995 FILE *f;
996 void *func;
997 double start_time, end_time;
998 int c_file_len = (int)sizeof(c_file_buff);
999 static const char c_ext[] = ".c";
1000 static const char so_ext[] = DLEXT;
1001 const int access_mode =
1002 #ifdef O_BINARY
1003 O_BINARY|
1004 #endif
1005 O_WRONLY|O_EXCL|O_CREAT;
1006 #ifndef _MSC_VER
1007 static const char o_ext[] = ".o";
1008 char *o_file;
1009 #endif
1010
1011 c_file_len = sprint_uniq_filename(c_file_buff, c_file_len, unit->id, MJIT_TMP_PREFIX, c_ext);
1012 if (c_file_len >= (int)sizeof(c_file_buff)) {
1013 ++c_file_len;
1014 c_file = alloca(c_file_len);
1015 c_file_len = sprint_uniq_filename(c_file, c_file_len, unit->id, MJIT_TMP_PREFIX, c_ext);
1016 }
1017 ++c_file_len;
1018
1019 #ifndef _MSC_VER
1020 o_file = alloca(c_file_len - sizeof(c_ext) + sizeof(o_ext));
1021 memcpy(o_file, c_file, c_file_len - sizeof(c_ext));
1022 memcpy(&o_file[c_file_len - sizeof(c_ext)], o_ext, sizeof(o_ext));
1023 #endif
1024 so_file = alloca(c_file_len - sizeof(c_ext) + sizeof(so_ext));
1025 memcpy(so_file, c_file, c_file_len - sizeof(c_ext));
1026 memcpy(&so_file[c_file_len - sizeof(c_ext)], so_ext, sizeof(so_ext));
1027
1028 sprintf(funcname, "_mjit%d", unit->id);
1029
1030 fd = rb_cloexec_open(c_file, access_mode, 0600);
1031 if (fd < 0 || (f = fdopen(fd, "w")) == NULL) {
1032 int e = errno;
1033 if (fd >= 0) (void)close(fd);
1034 verbose(1, "Failed to fopen '%s', giving up JIT for it (%s)", c_file, strerror(e));
1035 return (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
1036 }
1037
1038 /* print #include of MJIT header, etc. */
1039 compile_prelude(f);
1040
1041 /* wait until mjit_gc_finish_hook is called */
1042 CRITICAL_SECTION_START(3, "before mjit_compile to wait GC finish");
1043 while (in_gc) {
1044 verbose(3, "Waiting wakeup from GC");
1045 rb_native_cond_wait(&mjit_gc_wakeup, &mjit_engine_mutex);
1046 }
1047
1048 /* We need to check again here because we could've waited on GC above */
1049 if (unit->iseq == NULL) {
1050 fclose(f);
1051 if (!mjit_opts.save_temps)
1052 remove_file(c_file);
1053 free_unit(unit);
1054 in_jit = FALSE; /* just being explicit for return */
1055 }
1056 else {
1057 in_jit = TRUE;
1058 }
1059 CRITICAL_SECTION_FINISH(3, "before mjit_compile to wait GC finish");
1060 if (!in_jit) {
1061 return (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
1062 }
1063
1064 {
1065 VALUE s = rb_iseq_path(unit->iseq);
1066 const char *label = RSTRING_PTR(unit->iseq->body->location.label);
1067 const char *path = RSTRING_PTR(s);
1068 int lineno = FIX2INT(unit->iseq->body->location.first_lineno);
1069 verbose(2, "start compilation: %s@%s:%d -> %s", label, path, lineno, c_file);
1070 fprintf(f, "/* %s@%s:%d */\n\n", label, path, lineno);
1071 }
1072 success = mjit_compile(f, unit->iseq->body, funcname, cc_entries, is_entries);
1073
1074 /* release blocking mjit_gc_start_hook */
1075 CRITICAL_SECTION_START(3, "after mjit_compile to wakeup client for GC");
1076 in_jit = FALSE;
1077 verbose(3, "Sending wakeup signal to client in a mjit-worker for GC");
1078 rb_native_cond_signal(&mjit_client_wakeup);
1079 CRITICAL_SECTION_FINISH(3, "in worker to wakeup client for GC");
1080
1081 fclose(f);
1082 if (!success) {
1083 if (!mjit_opts.save_temps)
1084 remove_file(c_file);
1085 print_jit_result("failure", unit, 0, c_file);
1086 return (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
1087 }
1088
1089 start_time = real_ms_time();
1090 #ifdef _MSC_VER
1091 success = compile_c_to_so(c_file, so_file);
1092 #else
1093 /* splitting .c -> .o step and .o -> .so step, to cache .o files in the future */
1094 if ((success = compile_c_to_o(c_file, o_file)) != 0) {
1095 const char *o_files[2] = { NULL, NULL };
1096 o_files[0] = o_file;
1097 success = link_o_to_so(o_files, so_file);
1098
1099 /* Always set o_file for compaction. The value is also used for lazy deletion. */
1100 unit->o_file = strdup(o_file);
1101 if (unit->o_file == NULL) {
1102 mjit_warning("failed to allocate memory to remember '%s' (%s), removing it...", o_file, strerror(errno));
1103 remove_file(o_file);
1104 }
1105 }
1106 #endif
1107 end_time = real_ms_time();
1108
1109 if (!mjit_opts.save_temps)
1110 remove_file(c_file);
1111 if (!success) {
1112 verbose(2, "Failed to generate so: %s", so_file);
1113 return (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
1114 }
1115
1116 func = load_func_from_so(so_file, funcname, unit);
1117 if (!mjit_opts.save_temps)
1118 remove_so_file(so_file, unit);
1119
1120 if ((uintptr_t)func > (uintptr_t)LAST_JIT_ISEQ_FUNC) {
1121 CRITICAL_SECTION_START(3, "end of jit");
1122 add_to_list(unit, &active_units);
1123 if (unit->iseq)
1124 print_jit_result("success", unit, end_time - start_time, c_file);
1125 CRITICAL_SECTION_FINISH(3, "end of jit");
1126 }
1127 return (mjit_func_t)func;
1128 }
1129
1130 typedef struct {
1131 struct rb_mjit_unit *unit;
1132 struct rb_call_cache *cc_entries;
1133 union iseq_inline_storage_entry *is_entries;
1134 int finish_p;
1135 } mjit_copy_job_t;
1136
1137 /* Singleton MJIT copy job. This is made global since it needs to be durable even when MJIT worker thread is stopped.
1138 (ex: register job -> MJIT pause -> MJIT resume -> dispatch job. Actually this should be just cancelled by finish_p check) */
1139 static mjit_copy_job_t mjit_copy_job;
1140
1141 static void mjit_copy_job_handler(void *data);
1142
1143 /* vm_trace.c */
1144 int rb_workqueue_register(unsigned flags, rb_postponed_job_func_t , void *);
1145
1146 /* We're lazily copying cache values from main thread because these cache values
1147 could be different between ones on enqueue timing and ones on dequeue timing.
1148 Return TRUE if copy succeeds. */
1149 static int
copy_cache_from_main_thread(mjit_copy_job_t * job)1150 copy_cache_from_main_thread(mjit_copy_job_t *job)
1151 {
1152 CRITICAL_SECTION_START(3, "in copy_cache_from_main_thread");
1153 job->finish_p = FALSE; /* allow dispatching this job in mjit_copy_job_handler */
1154 CRITICAL_SECTION_FINISH(3, "in copy_cache_from_main_thread");
1155
1156 if (UNLIKELY(mjit_opts.wait)) {
1157 mjit_copy_job_handler((void *)job);
1158 return job->finish_p;
1159 }
1160
1161 if (!rb_workqueue_register(0, mjit_copy_job_handler, (void *)job))
1162 return FALSE;
1163 CRITICAL_SECTION_START(3, "in MJIT copy job wait");
1164 /* checking `stop_worker_p` too because `RUBY_VM_CHECK_INTS(ec)` may not
1165 lush mjit_copy_job_handler when EC_EXEC_TAG() is not TAG_NONE, and then
1166 `stop_worker()` could dead lock with this function. */
1167 while (!job->finish_p && !stop_worker_p) {
1168 rb_native_cond_wait(&mjit_worker_wakeup, &mjit_engine_mutex);
1169 verbose(3, "Getting wakeup from client");
1170 }
1171 CRITICAL_SECTION_FINISH(3, "in MJIT copy job wait");
1172 return job->finish_p;
1173 }
1174
1175 /* The function implementing a worker. It is executed in a separate
1176 thread by rb_thread_create_mjit_thread. It compiles precompiled header
1177 and then compiles requested ISeqs. */
1178 void
mjit_worker(void)1179 mjit_worker(void)
1180 {
1181 mjit_copy_job_t *job = &mjit_copy_job; /* just a shorthand */
1182
1183 #ifndef _MSC_VER
1184 if (pch_status == PCH_NOT_READY) {
1185 make_pch();
1186 }
1187 #endif
1188 if (pch_status == PCH_FAILED) {
1189 mjit_enabled = FALSE;
1190 CRITICAL_SECTION_START(3, "in worker to update worker_stopped");
1191 worker_stopped = TRUE;
1192 verbose(3, "Sending wakeup signal to client in a mjit-worker");
1193 rb_native_cond_signal(&mjit_client_wakeup);
1194 CRITICAL_SECTION_FINISH(3, "in worker to update worker_stopped");
1195 return; /* TODO: do the same thing in the latter half of mjit_finish */
1196 }
1197
1198 /* main worker loop */
1199 while (!stop_worker_p) {
1200 struct rb_mjit_unit *unit;
1201
1202 /* wait until unit is available */
1203 CRITICAL_SECTION_START(3, "in worker dequeue");
1204 while ((list_empty(&unit_queue.head) || active_units.length >= mjit_opts.max_cache_size) && !stop_worker_p) {
1205 rb_native_cond_wait(&mjit_worker_wakeup, &mjit_engine_mutex);
1206 verbose(3, "Getting wakeup from client");
1207 }
1208 unit = get_from_list(&unit_queue);
1209 job->finish_p = TRUE; /* disable dispatching this job in mjit_copy_job_handler while it's being modified */
1210 CRITICAL_SECTION_FINISH(3, "in worker dequeue");
1211
1212 if (unit) {
1213 mjit_func_t func;
1214 const struct rb_iseq_constant_body *body = unit->iseq->body;
1215
1216 job->unit = unit;
1217 job->cc_entries = NULL;
1218 if (body->ci_size > 0 || body->ci_kw_size > 0)
1219 job->cc_entries = alloca(sizeof(struct rb_call_cache) * (body->ci_size + body->ci_kw_size));
1220 job->is_entries = NULL;
1221 if (body->is_size > 0)
1222 job->is_entries = alloca(sizeof(union iseq_inline_storage_entry) * body->is_size);
1223
1224 /* Copy ISeq's inline caches values to avoid race condition. */
1225 if (job->cc_entries != NULL || job->is_entries != NULL) {
1226 if (copy_cache_from_main_thread(job) == FALSE) {
1227 continue; /* retry postponed_job failure, or stop worker */
1228 }
1229 }
1230
1231 /* JIT compile */
1232 func = convert_unit_to_func(unit, job->cc_entries, job->is_entries);
1233
1234 CRITICAL_SECTION_START(3, "in jit func replace");
1235 while (in_gc) { /* Make sure we're not GC-ing when touching ISeq */
1236 verbose(3, "Waiting wakeup from GC");
1237 rb_native_cond_wait(&mjit_gc_wakeup, &mjit_engine_mutex);
1238 }
1239 if (unit->iseq) { /* Check whether GCed or not */
1240 /* Usage of jit_code might be not in a critical section. */
1241 MJIT_ATOMIC_SET(unit->iseq->body->jit_func, func);
1242 }
1243 CRITICAL_SECTION_FINISH(3, "in jit func replace");
1244
1245 #ifndef _MSC_VER
1246 /* Combine .o files to one .so and reload all jit_func to improve memory locality */
1247 if ((!mjit_opts.wait && unit_queue.length == 0 && active_units.length > 1)
1248 || active_units.length == mjit_opts.max_cache_size) {
1249 compact_all_jit_code();
1250 }
1251 #endif
1252 }
1253 }
1254
1255 /* Disable dispatching this job in mjit_copy_job_handler while memory allocated by alloca
1256 could be expired after finishing this function. */
1257 job->finish_p = TRUE;
1258
1259 /* To keep mutex unlocked when it is destroyed by mjit_finish, don't wrap CRITICAL_SECTION here. */
1260 worker_stopped = TRUE;
1261 }
1262