1 /*
2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20 #include <lightning.h>
21 #include <lightning/jit_private.h>
22 #include <sys/mman.h>
23 #if defined(__sgi)
24 # include <fcntl.h>
25 #endif
26
27 #ifndef MAP_ANON
28 # define MAP_ANON MAP_ANONYMOUS
29 # ifndef MAP_ANONYMOUS
30 # define MAP_ANONYMOUS 0
31 # endif
32 #endif
33
34 #define jit_regload_reload 0 /* convert to reload */
35 #define jit_regload_delete 1 /* just remove node */
36 #define jit_regload_isdead 2 /* delete and unset live bit */
37
38 /*
39 * Prototypes
40 */
41 static jit_word_t hash_data(const void*, jit_word_t);
42
43 #define new_pool() _new_pool(_jit)
44 static void _new_pool(jit_state_t*);
45
46 #define new_node(u) _new_node(_jit, u)
47 static jit_node_t *_new_node(jit_state_t*, jit_code_t);
48
49 #define link_node(u) _link_node(_jit, u)
50 static inline jit_node_t *_link_node(jit_state_t*, jit_node_t*);
51
52 #define del_node(u, v) _del_node(_jit, u, v)
53 static inline void _del_node(jit_state_t*, jit_node_t*, jit_node_t*);
54
55 #define free_node(u) _free_node(_jit, u)
56 static inline void _free_node(jit_state_t*, jit_node_t*);
57
58 #define del_label(u, v) _del_label(_jit, u, v)
59 static void _del_label(jit_state_t*, jit_node_t*, jit_node_t*);
60
61 #define jit_dataset() _jit_dataset(_jit)
62 static void
63 _jit_dataset(jit_state_t *_jit);
64
65 #define jit_setup(block) _jit_setup(_jit, block)
66 static void
67 _jit_setup(jit_state_t *_jit, jit_block_t *block);
68
69 #define jit_follow(block, todo) _jit_follow(_jit, block, todo)
70 static void
71 _jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo);
72
73 #define jit_update(node, live, mask) _jit_update(_jit, node, live, mask)
74 static void
75 _jit_update(jit_state_t *_jit, jit_node_t *node,
76 jit_regset_t *live, jit_regset_t *mask);
77
78 #define thread_jumps() _thread_jumps(_jit)
79 static void
80 _thread_jumps(jit_state_t *_jit);
81
82 #define sequential_labels() _sequential_labels(_jit)
83 static void
84 _sequential_labels(jit_state_t *_jit);
85
86 #define shortcut_jump(prev, node) _shortcut_jump(_jit, prev, node)
87 static jit_bool_t
88 _shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
89
90 #define redundant_jump(prev, node) _redundant_jump(_jit, prev, node)
91 static jit_bool_t
92 _redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
93
94 static jit_code_t
95 reverse_jump_code(jit_code_t code);
96
97 #define reverse_jump(prev, node) _reverse_jump(_jit, prev, node)
98 static jit_bool_t
99 _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
100
101 #define redundant_store(node, jump) _redundant_store(_jit, node, jump)
102 static void
103 _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump);
104
105 #define simplify_movr(p, n, k, s) _simplify_movr(_jit, p, n, k, s)
106 static jit_bool_t
107 _simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
108 jit_int32_t kind, jit_int32_t size);
109
110 #define simplify_movi(p, n, k, s) _simplify_movi(_jit, p, n, k, s)
111 static jit_bool_t
112 _simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
113 jit_int32_t kind, jit_int32_t size);
114
115 #define simplify_ldxi(prev, node) _simplify_ldxi(_jit, prev, node)
116 static jit_bool_t
117 _simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
118
119 #define simplify_stxi(prev, node) _simplify_stxi(_jit, prev, node)
120 static jit_bool_t
121 _simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node);
122
123 #define simplify_spill(node, regno) _simplify_spill(_jit, node, regno)
124 static void
125 _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
126
127 #define simplify() _simplify(_jit)
128 static void
129 _simplify(jit_state_t *_jit);
130
131 #define jit_reg_undef -1
132 #define jit_reg_static 0
133 #define jit_reg_change 1
134 #define register_change_p(n, l, r) _register_change_p(_jit, n, l, r)
135 static jit_int32_t
136 _register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
137 jit_int32_t regno);
138
139 #define spill_reglive_p(node, regno) _spill_reglive_p(_jit, node, regno)
140 static jit_bool_t
141 _spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno);
142
143 #define patch_registers() _patch_registers(_jit)
144 static void
145 _patch_registers(jit_state_t *_jit);
146
147 #define patch_register(n,l,r,p) _patch_register(_jit,n,l,r,p)
148 static void
149 _patch_register(jit_state_t *jit, jit_node_t *node, jit_node_t *link,
150 jit_int32_t regno, jit_int32_t patch);
151
152 /*
153 * Initialization
154 */
155 #if !defined(__sgi)
156 #define mmap_fd -1
157 #endif
158
159 /*
160 * Implementation
161 */
162 void
init_jit(const char * progname)163 init_jit(const char *progname)
164 {
165 jit_get_cpu();
166 jit_init_debug(progname);
167 jit_init_size();
168 }
169
170 void
finish_jit(void)171 finish_jit(void)
172 {
173 jit_finish_debug();
174 jit_finish_size();
175 }
176
177 jit_int32_t
_jit_get_reg(jit_state_t * _jit,jit_int32_t regspec)178 _jit_get_reg(jit_state_t *_jit, jit_int32_t regspec)
179 {
180 jit_int32_t spec;
181 jit_int32_t regno;
182
183 spec = regspec & ~(jit_class_chk|jit_class_nospill);
184 if (spec & jit_class_named) {
185 regno = jit_regno(spec);
186 if (jit_regset_tstbit(&_jitc->regsav, regno))
187 /* fail if register is spilled */
188 goto fail;
189 if (jit_regset_tstbit(&_jitc->regarg, regno))
190 /* fail if register is an argument to current instruction */
191 goto fail;
192 if (jit_regset_tstbit(&_jitc->reglive, regno)) {
193 if (regspec & jit_class_nospill)
194 /* fail if register is live and should not spill/reload */
195 goto fail;
196 goto spill;
197 }
198 jit_regset_setbit(&_jitc->regarg, regno);
199 return (regno);
200 }
201 else
202 assert(jit_class(spec) != 0);
203
204 if (_jitc->emit) {
205 /* search for a free register matching spec */
206 for (regno = 0; regno < _jitc->reglen; regno++) {
207 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
208 !jit_regset_tstbit(&_jitc->regarg, regno) &&
209 !jit_regset_tstbit(&_jitc->reglive, regno))
210 goto regarg;
211 }
212
213 /* search for a register matching spec that is not an argument
214 * for the current instruction */
215 for (regno = 0; regno < _jitc->reglen; regno++) {
216 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
217 !jit_regset_tstbit(&_jitc->regsav, regno) &&
218 !jit_regset_tstbit(&_jitc->regarg, regno) &&
219 !(regspec & jit_class_nospill)) {
220 spill:
221 assert(_jitc->function != NULL);
222 if (spec & jit_class_gpr) {
223 if (!_jitc->function->regoff[regno]) {
224 _jitc->function->regoff[regno] =
225 jit_allocai(sizeof(jit_word_t));
226 _jitc->again = 1;
227 }
228 #if DEBUG
229 /* emit_stxi must not need temporary registers */
230 assert(!_jitc->getreg);
231 _jitc->getreg = 1;
232 #endif
233 emit_stxi(_jitc->function->regoff[regno], JIT_FP, regno);
234 #if DEBUG
235 _jitc->getreg = 0;
236 #endif
237 }
238 else {
239 if (!_jitc->function->regoff[regno]) {
240 _jitc->function->regoff[regno] =
241 jit_allocai(sizeof(jit_float64_t));
242 _jitc->again = 1;
243 }
244 #if DEBUG
245 /* emit_stxi must not need temporary registers */
246 assert(!_jitc->getreg);
247 _jitc->getreg = 1;
248 #endif
249 emit_stxi_d(_jitc->function->regoff[regno], JIT_FP, regno);
250 #if DEBUG
251 _jitc->getreg = 0;
252 #endif
253 }
254 jit_regset_setbit(&_jitc->regsav, regno);
255 regarg:
256 jit_regset_setbit(&_jitc->regarg, regno);
257 if (jit_class(_rvs[regno].spec) & jit_class_sav) {
258 /* if will modify callee save registers without a
259 * function prolog, better patch this assertion */
260 assert(_jitc->function != NULL);
261 if (!jit_regset_tstbit(&_jitc->function->regset, regno)) {
262 jit_regset_setbit(&_jitc->function->regset, regno);
263 _jitc->again = 1;
264 }
265 }
266 return (regno);
267 }
268 }
269 }
270 else {
271 /* nospill hint only valid during emit" */
272 assert(!(regspec & jit_class_nospill));
273 for (regno = 0; regno < _jitc->reglen; regno++) {
274 if ((jit_class(_rvs[regno].spec) & spec) == spec &&
275 !jit_regset_tstbit(&_jitc->regsav, regno) &&
276 !jit_regset_tstbit(&_jitc->regarg, regno)) {
277 jit_regset_setbit(&_jitc->regarg, regno);
278 jit_regset_setbit(&_jitc->regsav, regno);
279 jit_save(regno);
280 return (jit_regno_patch|regno);
281 }
282 }
283 }
284
285 /* Out of hardware registers */
286 fail:
287 assert(regspec & jit_class_chk);
288 return (JIT_NOREG);
289 }
290
291 void
_jit_unget_reg(jit_state_t * _jit,jit_int32_t regno)292 _jit_unget_reg(jit_state_t *_jit, jit_int32_t regno)
293 {
294 regno = jit_regno(regno);
295 if (jit_regset_tstbit(&_jitc->regsav, regno)) {
296 if (_jitc->emit) {
297 #if DEBUG
298 /* emit_ldxi must not need a temporary register */
299 assert(!_jitc->getreg);
300 _jitc->getreg = 1;
301 #endif
302 if (jit_class(_rvs[regno].spec) & jit_class_gpr)
303 emit_ldxi(regno, JIT_FP, _jitc->function->regoff[regno]);
304 else
305 emit_ldxi_d(regno, JIT_FP, _jitc->function->regoff[regno]);
306 #if DEBUG
307 /* emit_ldxi must not need a temporary register */
308 _jitc->getreg = 0;
309 #endif
310 }
311 else
312 jit_load(regno);
313 jit_regset_clrbit(&_jitc->regsav, regno);
314 }
315 #if defined(jit_carry)
316 assert((regno == jit_carry /*&& _NOREG != jit_carry*/) ||
317 jit_regset_tstbit(&_jitc->regarg, regno) != 0);
318 #else
319 assert(jit_regset_tstbit(&_jitc->regarg, regno) != 0);
320 #endif
321 jit_regset_clrbit(&_jitc->regarg, regno);
322 }
323
324 jit_bool_t
_jit_callee_save_p(jit_state_t * _jit,jit_int32_t regno)325 _jit_callee_save_p(jit_state_t *_jit, jit_int32_t regno)
326 {
327 assert(regno >= 0 && regno < JIT_NOREG);
328 return (!!(_rvs[regno].spec & jit_class_sav));
329 }
330
331 extern jit_bool_t
_jit_pointer_p(jit_state_t * _jit,jit_pointer_t address)332 _jit_pointer_p(jit_state_t *_jit, jit_pointer_t address)
333 {
334 return ((jit_uint8_t *)address >= _jit->code.ptr &&
335 (jit_word_t)address < _jit->pc.w);
336 }
337
338 #if __ia64__
339 void
jit_regset_com(jit_regset_t * u,jit_regset_t * v)340 jit_regset_com(jit_regset_t *u, jit_regset_t *v)
341 {
342 u->rl = ~v->rl; u->rh = ~v->rh;
343 u->fl = ~v->fl; u->fh = ~v->fh;
344 }
345
346 void
jit_regset_and(jit_regset_t * u,jit_regset_t * v,jit_regset_t * w)347 jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
348 {
349 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
350 u->fl = v->fl & w->fl; u->fh = v->fh & w->fh;
351 }
352
353 void
jit_regset_ior(jit_regset_t * u,jit_regset_t * v,jit_regset_t * w)354 jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
355 {
356 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
357 u->fl = v->fl | w->fl; u->fh = v->fh | w->fh;
358 }
359
360 void
jit_regset_xor(jit_regset_t * u,jit_regset_t * v,jit_regset_t * w)361 jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
362 {
363 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
364 u->fl = v->fl ^ w->fl; u->fh = v->fh ^ w->fh;
365 }
366
367 void
jit_regset_set(jit_regset_t * u,jit_regset_t * v)368 jit_regset_set(jit_regset_t *u, jit_regset_t *v)
369 {
370 u->rl = v->rl; u->rh = v->rh;
371 u->fl = v->fl; u->fh = v->fh;
372 }
373
374 void
jit_regset_set_mask(jit_regset_t * u,jit_int32_t v)375 jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
376 {
377 jit_bool_t w = !!(v & (v - 1));
378
379 assert(v >= 0 && v <= 256);
380 if (v == 0)
381 u->rl = u->rh = u->fl = u->fh = -1LL;
382 else if (v <= 64) {
383 u->rl = w ? (1LL << v) - 1 : -1LL;
384 u->rh = u->fl = u->fh = 0;
385 }
386 else if (v <= 128) {
387 u->rl = -1LL;
388 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
389 u->fl = u->fh = 0;
390 }
391 else if (v <= 192) {
392 u->rl = u->rh = -1LL;
393 u->fl = w ? (1LL << (v - 128)) - 1 : -1LL;
394 u->fh = 0;
395 }
396 else {
397 u->rl = u->rh = u->fl = -1LL;
398 u->fh = w ? (1LL << (v - 128)) - 1 : -1LL;
399 }
400 }
401
402 jit_bool_t
jit_regset_cmp_ui(jit_regset_t * u,jit_word_t v)403 jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
404 {
405 return !((u->rl == v && u->rh == 0 && u->fl == 0 && u->fh == 0));
406 }
407
408 void
jit_regset_set_ui(jit_regset_t * u,jit_word_t v)409 jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
410 {
411 u->rl = v;
412 u->rh = u->fl = u->fh = 0;
413 }
414
415 jit_bool_t
jit_regset_set_p(jit_regset_t * u)416 jit_regset_set_p(jit_regset_t *u)
417 {
418 return (u->rl || u->rh || u->fl || u->fh);
419 }
420
421 void
jit_regset_clrbit(jit_regset_t * set,jit_int32_t bit)422 jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
423 {
424 assert(bit >= 0 && bit <= 255);
425 if (bit < 64)
426 set->rl &= ~(1LL << bit);
427 else if (bit < 128)
428 set->rh &= ~(1LL << (bit - 64));
429 else if (bit < 192)
430 set->fl &= ~(1LL << (bit - 128));
431 else
432 set->fh &= ~(1LL << (bit - 192));
433 }
434
435 void
jit_regset_setbit(jit_regset_t * set,jit_int32_t bit)436 jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
437 {
438 assert(bit >= 0 && bit <= 255);
439 if (bit < 64)
440 set->rl |= 1LL << bit;
441 else if (bit < 128)
442 set->rh |= 1LL << (bit - 64);
443 else if (bit < 192)
444 set->fl |= 1LL << (bit - 128);
445 else
446 set->fh |= 1LL << (bit - 192);
447 }
448
449 jit_bool_t
jit_regset_tstbit(jit_regset_t * set,jit_int32_t bit)450 jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
451 {
452 assert(bit >= 0 && bit <= 255);
453 if (bit < 64)
454 return (!!(set->rl & (1LL << bit)));
455 else if (bit < 128)
456 return (!!(set->rh & (1LL << (bit - 64))));
457 else if (bit < 192)
458 return (!!(set->fl & (1LL << (bit - 128))));
459 return (!!(set->fh & (1LL << (bit - 192))));
460 }
461
462 unsigned long
jit_regset_scan1(jit_regset_t * set,jit_int32_t offset)463 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
464 {
465 assert(offset >= 0 && offset <= 255);
466 for (; offset < 64; offset++) {
467 if (set->rl & (1LL << offset))
468 return (offset);
469 }
470 for (; offset < 128; offset++) {
471 if (set->rh & (1LL << (offset - 64)))
472 return (offset);
473 }
474 for (; offset < 192; offset++) {
475 if (set->fl & (1LL << (offset - 128)))
476 return (offset);
477 }
478 for (; offset < 256; offset++) {
479 if (set->fh & (1LL << (offset - 192)))
480 return (offset);
481 }
482 return (ULONG_MAX);
483 }
484
485 #elif __sparc__ && __WORDSIZE == 64
486 void
jit_regset_com(jit_regset_t * u,jit_regset_t * v)487 jit_regset_com(jit_regset_t *u, jit_regset_t *v)
488 {
489 u->rl = ~v->rl; u->rh = ~v->rh;
490 }
491
492 void
jit_regset_and(jit_regset_t * u,jit_regset_t * v,jit_regset_t * w)493 jit_regset_and(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
494 {
495 u->rl = v->rl & w->rl; u->rh = v->rh & w->rh;
496 }
497
498 void
jit_regset_ior(jit_regset_t * u,jit_regset_t * v,jit_regset_t * w)499 jit_regset_ior(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
500 {
501 u->rl = v->rl | w->rl; u->rh = v->rh | w->rh;
502 }
503
504 void
jit_regset_xor(jit_regset_t * u,jit_regset_t * v,jit_regset_t * w)505 jit_regset_xor(jit_regset_t *u, jit_regset_t *v, jit_regset_t *w)
506 {
507 u->rl = v->rl ^ w->rl; u->rh = v->rh ^ w->rh;
508 }
509
510 void
jit_regset_set(jit_regset_t * u,jit_regset_t * v)511 jit_regset_set(jit_regset_t *u, jit_regset_t *v)
512 {
513 u->rl = v->rl; u->rh = v->rh;
514 }
515
516 void
jit_regset_set_mask(jit_regset_t * u,jit_int32_t v)517 jit_regset_set_mask(jit_regset_t *u, jit_int32_t v)
518 {
519 jit_bool_t w = !!(v & (v - 1));
520
521 assert(v >= 0 && v <= 128);
522 if (v == 0)
523 u->rl = u->rh = -1LL;
524 else if (v <= 64) {
525 u->rl = w ? (1LL << v) - 1 : -1LL;
526 u->rh = 0;
527 }
528 else {
529 u->rl = -1LL;
530 u->rh = w ? (1LL << (v - 64)) - 1 : -1LL;
531 }
532 }
533
534 jit_bool_t
jit_regset_cmp_ui(jit_regset_t * u,jit_word_t v)535 jit_regset_cmp_ui(jit_regset_t *u, jit_word_t v)
536 {
537 return !((u->rl == v && u->rh == 0));
538 }
539
540 void
jit_regset_set_ui(jit_regset_t * u,jit_word_t v)541 jit_regset_set_ui(jit_regset_t *u, jit_word_t v)
542 {
543 u->rl = v;
544 u->rh = 0;
545 }
546
547 jit_bool_t
jit_regset_set_p(jit_regset_t * u)548 jit_regset_set_p(jit_regset_t *u)
549 {
550 return (u->rl || u->rh);
551 }
552
553 void
jit_regset_clrbit(jit_regset_t * set,jit_int32_t bit)554 jit_regset_clrbit(jit_regset_t *set, jit_int32_t bit)
555 {
556 assert(bit >= 0 && bit <= 128);
557 if (bit < 64)
558 set->rl &= ~(1LL << bit);
559 else
560 set->rh &= ~(1LL << (bit - 64));
561 }
562
563 void
jit_regset_setbit(jit_regset_t * set,jit_int32_t bit)564 jit_regset_setbit(jit_regset_t *set, jit_int32_t bit)
565 {
566 assert(bit >= 0 && bit <= 127);
567 if (bit < 64)
568 set->rl |= 1LL << bit;
569 else
570 set->rh |= 1LL << (bit - 64);
571 }
572
573 jit_bool_t
jit_regset_tstbit(jit_regset_t * set,jit_int32_t bit)574 jit_regset_tstbit(jit_regset_t *set, jit_int32_t bit)
575 {
576 assert(bit >= 0 && bit <= 127);
577 if (bit < 64)
578 return (!!(set->rl & (1LL << bit)));
579 else
580 return (!!(set->rh & (1LL << (bit - 64))));
581 }
582
583 unsigned long
jit_regset_scan1(jit_regset_t * set,jit_int32_t offset)584 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
585 {
586 assert(offset >= 0 && offset <= 127);
587 for (; offset < 64; offset++) {
588 if (set->rl & (1LL << offset))
589 return (offset);
590 }
591 for (; offset < 128; offset++) {
592 if (set->rh & (1LL << (offset - 64)))
593 return (offset);
594 }
595 return (ULONG_MAX);
596 }
597
598 #else
599 unsigned long
jit_regset_scan1(jit_regset_t * set,jit_int32_t offset)600 jit_regset_scan1(jit_regset_t *set, jit_int32_t offset)
601 {
602 jit_regset_t mask;
603 assert(offset >= 0 && offset <= 63);
604 if ((mask = *set >> offset)) {
605 for (;;) {
606 if (mask & 1)
607 return (offset);
608 mask >>= 1;
609 ++offset;
610 }
611 }
612 return (ULONG_MAX);
613 }
614 #endif
615
616 void
_jit_save(jit_state_t * _jit,jit_int32_t reg)617 _jit_save(jit_state_t *_jit, jit_int32_t reg)
618 {
619 reg = jit_regno(reg);
620 assert(!_jitc->realize);
621 _jitc->spill[reg] = jit_new_node_w(jit_code_save, reg);
622 }
623
624 void
_jit_load(jit_state_t * _jit,jit_int32_t reg)625 _jit_load(jit_state_t *_jit, jit_int32_t reg)
626 {
627 jit_node_t *node;
628
629 reg = jit_regno(reg);
630 assert(!_jitc->realize);
631 assert(_jitc->spill[reg] != NULL);
632 node = jit_new_node_w(jit_code_load, reg);
633 /* create a path to flag the save/load is not required */
634 node->link = _jitc->spill[reg];
635 node->link->link = node;
636 _jitc->spill[reg] = NULL;
637 }
638
639 static jit_word_t
hash_data(const void * data,jit_word_t length)640 hash_data(const void *data, jit_word_t length)
641 {
642 const jit_uint8_t *ptr;
643 jit_word_t i, key;
644 for (i = key = 0, ptr = data; i < length; i++)
645 key = (key << (key & 1)) ^ ptr[i];
646 return (key);
647 }
648
649 jit_pointer_t
_jit_address(jit_state_t * _jit,jit_node_t * node)650 _jit_address(jit_state_t *_jit, jit_node_t *node)
651 {
652 assert(_jitc->done);
653 assert(node != NULL &&
654 /* If a node type that is documented to be a fixed marker */
655 (node->code == jit_code_note || node->code == jit_code_name ||
656 /* If another special fixed marker, returned by jit_indirect() */
657 (node->code == jit_code_label && (node->flag & jit_flag_use) != 0)));
658 return ((jit_pointer_t)node->u.w);
659 }
660
661 jit_node_t *
_jit_data(jit_state_t * _jit,const void * data,jit_word_t length,jit_int32_t align)662 _jit_data(jit_state_t *_jit, const void *data,
663 jit_word_t length, jit_int32_t align)
664 {
665 jit_word_t key;
666 jit_node_t *node;
667
668 assert(!_jitc->realize);
669
670 /* Ensure there is space even if asking for a duplicate */
671 if (((_jitc->data.offset + 7) & -8) + length > _jit->data.length) {
672 jit_word_t size;
673
674 size = (_jit->data.length + length + 4096) & - 4095;
675 assert(size >= _jit->data.length);
676 if (_jitc->data.ptr == NULL)
677 jit_alloc((jit_pointer_t *)&_jitc->data.ptr, size);
678 else
679 jit_realloc((jit_pointer_t *)&_jitc->data.ptr,
680 _jit->data.length, size);
681 _jit->data.length = size;
682 }
683 if (_jitc->data.table == NULL)
684 jit_alloc((jit_pointer_t *)&_jitc->data.table,
685 (_jitc->data.size = 16) * sizeof(jit_node_t*));
686
687 key = hash_data(data, length) & (_jitc->data.size - 1);
688 node = _jitc->data.table[key];
689 for (; node; node = node->next) {
690 if (node->v.w == length &&
691 memcmp(_jitc->data.ptr + node->u.w, data, length) == 0)
692 break;
693 }
694
695 if (!node) {
696 node = jit_new_node_no_link(jit_code_data);
697 if (!align)
698 align = length;
699 switch (align) {
700 case 0: case 1:
701 break;
702 case 2:
703 _jitc->data.offset = (_jitc->data.offset + 1) & -2;
704 break;
705 case 3: case 4:
706 _jitc->data.offset = (_jitc->data.offset + 3) & -4;
707 break;
708 default:
709 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
710 break;
711 }
712 node->u.w = _jitc->data.offset;
713 node->v.w = length;
714 jit_memcpy(_jitc->data.ptr + _jitc->data.offset, data, length);
715 _jitc->data.offset += length;
716
717 node->next = _jitc->data.table[key];
718 _jitc->data.table[key] = node;
719 ++_jitc->data.count;
720
721 /* Rehash if more than 75% used table */
722 if (_jitc->data.count >
723 (_jitc->data.size >> 1) + (_jitc->data.size >> 2) &&
724 (_jitc->data.size << 1) > _jitc->data.size) {
725 jit_word_t i;
726 jit_node_t **hash;
727 jit_node_t *next;
728 jit_node_t *temp;
729
730 jit_alloc((jit_pointer_t *)&hash,
731 (_jitc->data.size << 1) * sizeof(jit_node_t*));
732 for (i = 0; i < _jitc->data.size; i++) {
733 temp = _jitc->data.table[i];
734 for (; temp; temp = next) {
735 next = temp->next;
736 key = hash_data(_jitc->data.ptr + temp->u.w, temp->v.w) &
737 ((_jitc->data.size << 1) - 1);
738 temp->next = hash[key];
739 hash[key] = temp;
740 }
741 }
742 jit_free((jit_pointer_t *)&_jitc->data.table);
743 _jitc->data.table = hash;
744 _jitc->data.size <<= 1;
745 }
746 }
747
748 return (node);
749 }
750
751 static void
_new_pool(jit_state_t * _jit)752 _new_pool(jit_state_t *_jit)
753 {
754 jit_node_t *list;
755 jit_int32_t offset;
756
757 if (_jitc->pool.offset >= _jitc->pool.length) {
758 jit_int32_t length;
759
760 length = _jitc->pool.length + 16;
761 jit_realloc((jit_pointer_t *)&_jitc->pool.ptr,
762 _jitc->pool.length * sizeof(jit_node_t *),
763 length * sizeof(jit_node_t *));
764 _jitc->pool.length = length;
765 }
766 jit_alloc((jit_pointer_t *)(_jitc->pool.ptr + _jitc->pool.offset),
767 sizeof(jit_node_t) * 1024);
768 list = _jitc->pool.ptr[_jitc->pool.offset];
769 for (offset = 1; offset < 1024; offset++, list++)
770 list->next = list + 1;
771 list->next = _jitc->list;
772 _jitc->list = _jitc->pool.ptr[_jitc->pool.offset];
773 ++_jitc->pool.offset;
774 }
775
776 static jit_node_t *
_new_node(jit_state_t * _jit,jit_code_t code)777 _new_node(jit_state_t *_jit, jit_code_t code)
778 {
779 jit_node_t *node;
780
781 if (_jitc->list == NULL)
782 new_pool();
783 node = _jitc->list;
784 _jitc->list = node->next;
785 if (_jitc->synth)
786 node->flag |= jit_flag_synth;
787 node->next = NULL;
788 node->code = code;
789
790 return (node);
791 }
792
793 static inline jit_node_t *
_link_node(jit_state_t * _jit,jit_node_t * node)794 _link_node(jit_state_t *_jit, jit_node_t *node)
795 {
796 if (_jitc->tail)
797 _jitc->tail->next = node;
798 else
799 _jitc->head = node;
800 return (_jitc->tail = node);
801 }
802
803 static inline void
_del_node(jit_state_t * _jit,jit_node_t * prev,jit_node_t * node)804 _del_node(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
805 {
806 if (prev == node) {
807 assert(prev == _jitc->head);
808 _jitc->head = node->next;
809 }
810 else
811 prev->next = node->next;
812 memset(node, 0, sizeof(jit_node_t));
813 node->next = _jitc->list;
814 _jitc->list = node;
815 }
816
817 static inline void
_free_node(jit_state_t * _jit,jit_node_t * node)818 _free_node(jit_state_t *_jit, jit_node_t *node)
819 {
820 memset(node, 0, sizeof(jit_node_t));
821 node->next = _jitc->list;
822 _jitc->list = node;
823 }
824
825 static void
_del_label(jit_state_t * _jit,jit_node_t * prev,jit_node_t * node)826 _del_label(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
827 {
828 jit_block_t *block;
829
830 /* only allow call to del_label on linked labels */
831 block = _jitc->blocks.ptr + node->v.w;
832 assert(block->label == node);
833
834 /* del_label() should only be called when optimizing.
835 * This will leave an empty block index */
836 jit_regset_del(&block->reglive);
837 jit_regset_del(&block->regmask);
838 block->label = NULL;
839
840 /* redundant, should be already true */
841 assert(node->link == NULL);
842 del_node(prev, node);
843 }
844
845 jit_state_t *
jit_new_state(void)846 jit_new_state(void)
847 {
848 jit_state_t *_jit;
849
850 jit_alloc((jit_pointer_t *)&_jit, sizeof(jit_state_t));
851 jit_alloc((jit_pointer_t *)&_jitc, sizeof(jit_compiler_t));
852 jit_regset_new(&_jitc->regarg);
853 jit_regset_new(&_jitc->regsav);
854 jit_regset_new(&_jitc->reglive);
855 jit_regset_new(&_jitc->regmask);
856
857 jit_init();
858
859 jit_alloc((jit_pointer_t *)&_jitc->spill,
860 _jitc->reglen * sizeof(jit_node_t*));
861 jit_alloc((jit_pointer_t *)&_jitc->gen,
862 _jitc->reglen * sizeof(jit_int32_t));
863 jit_alloc((jit_pointer_t *)&_jitc->values,
864 _jitc->reglen * sizeof(jit_value_t));
865
866 jit_alloc((jit_pointer_t *)&_jitc->patches.ptr,
867 (_jitc->patches.length = 1024) * sizeof(jit_patch_t));
868 jit_alloc((jit_pointer_t *)&_jitc->functions.ptr,
869 (_jitc->functions.length = 16) * sizeof(jit_function_t));
870 jit_alloc((jit_pointer_t *)&_jitc->pool.ptr,
871 (_jitc->pool.length = 16) * sizeof(jit_node_t*));
872 jit_alloc((jit_pointer_t *)&_jitc->blocks.ptr,
873 (_jitc->blocks.length = 16) * sizeof(jit_block_t));
874 #if __arm__ && DISASSEMBLER
875 jit_alloc((jit_pointer_t *)&_jitc->data_info.ptr,
876 (_jitc->data_info.length = 1024) * sizeof(jit_data_info_t));
877 #endif
878
879 /* allocate at most one extra note in case jit_name() is
880 * never called, or called after adding at least one note */
881 _jit->note.length = 1;
882 _jitc->note.size = sizeof(jit_note_t);
883
884 return (_jit);
885 }
886
887 void
_jit_clear_state(jit_state_t * _jit)888 _jit_clear_state(jit_state_t *_jit)
889 {
890 #if DEVEL_DISASSEMBLER
891 # define jit_really_clear_state() _jit_really_clear_state(_jit)
892 }
893
_jit_really_clear_state(jit_state_t * _jit)894 void _jit_really_clear_state(jit_state_t *_jit)
895 {
896 #endif
897 jit_word_t offset;
898 jit_function_t *function;
899
900 /* release memory not required at jit execution time and set
901 * pointers to NULL to explicitly know they are released */
902 _jitc->head = _jitc->tail = NULL;
903
904 jit_free((jit_pointer_t *)&_jitc->data.table);
905 _jitc->data.size = _jitc->data.count = 0;
906
907 jit_free((jit_pointer_t *)&_jitc->spill);
908 jit_free((jit_pointer_t *)&_jitc->gen);
909 jit_free((jit_pointer_t *)&_jitc->values);
910
911 jit_free((jit_pointer_t *)&_jitc->blocks.ptr);
912
913 jit_free((jit_pointer_t *)&_jitc->patches.ptr);
914 _jitc->patches.offset = _jitc->patches.length = 0;
915
916 for (offset = 0; offset < _jitc->functions.offset; offset++) {
917 function = _jitc->functions.ptr + offset;
918 jit_free((jit_pointer_t *)&function->regoff);
919 }
920 jit_free((jit_pointer_t *)&_jitc->functions.ptr);
921 _jitc->functions.offset = _jitc->functions.length = 0;
922 _jitc->function = NULL;
923
924 for (offset = 0; offset < _jitc->pool.offset; offset++)
925 jit_free((jit_pointer_t *)(_jitc->pool.ptr + offset));
926 jit_free((jit_pointer_t *)&_jitc->pool.ptr);
927 _jitc->pool.offset = _jitc->pool.length = 0;
928 _jitc->list = NULL;
929
930 _jitc->note.head = _jitc->note.tail =
931 _jitc->note.name = _jitc->note.note = NULL;
932 _jitc->note.base = NULL;
933
934 #if __arm__ && DISASSEMBLER
935 jit_free((jit_pointer_t *)&_jitc->data_info.ptr);
936 #endif
937
938 #if (__powerpc__ && _CALL_AIXDESC) || __ia64__
939 jit_free((jit_pointer_t *)&_jitc->prolog.ptr);
940 #endif
941
942 #if __ia64__
943 jit_regset_del(&_jitc->regs);
944 #endif
945
946 jit_free((jit_pointer_t *)&_jitc);
947 }
948
949 void
_jit_destroy_state(jit_state_t * _jit)950 _jit_destroy_state(jit_state_t *_jit)
951 {
952 #if DEVEL_DISASSEMBLER
953 jit_really_clear_state();
954 #endif
955 if (!_jit->user_code)
956 munmap(_jit->code.ptr, _jit->code.length);
957 if (!_jit->user_data)
958 munmap(_jit->data.ptr, _jit->data.length);
959 jit_free((jit_pointer_t *)&_jit);
960 }
961
962 void
_jit_synth_inc(jit_state_t * _jit)963 _jit_synth_inc(jit_state_t *_jit)
964 {
965 assert(_jitc->synth < 8);
966 ++_jitc->synth;
967 }
968
969 jit_node_t *
_jit_new_node(jit_state_t * _jit,jit_code_t code)970 _jit_new_node(jit_state_t *_jit, jit_code_t code)
971 {
972 assert(!_jitc->realize);
973 return (link_node(new_node(code)));
974 }
975
976 jit_node_t *
_jit_new_node_no_link(jit_state_t * _jit,jit_code_t code)977 _jit_new_node_no_link(jit_state_t *_jit, jit_code_t code)
978 {
979 assert(!_jitc->realize);
980 return (new_node(code));
981 }
982
983 void
_jit_link_node(jit_state_t * _jit,jit_node_t * node)984 _jit_link_node(jit_state_t *_jit, jit_node_t *node)
985 {
986 assert(!_jitc->realize);
987 link_node(node);
988 }
989
990 void
_jit_synth_dec(jit_state_t * _jit)991 _jit_synth_dec(jit_state_t *_jit)
992 {
993 assert(_jitc->synth > 0);
994 --_jitc->synth;
995 }
996
997 jit_node_t *
_jit_new_node_w(jit_state_t * _jit,jit_code_t code,jit_word_t u)998 _jit_new_node_w(jit_state_t *_jit, jit_code_t code,
999 jit_word_t u)
1000 {
1001 jit_node_t *node = new_node(code);
1002 assert(!_jitc->realize);
1003 node->u.w = u;
1004 return (link_node(node));
1005 }
1006
1007 jit_node_t *
_jit_new_node_f(jit_state_t * _jit,jit_code_t code,jit_float32_t u)1008 _jit_new_node_f(jit_state_t *_jit, jit_code_t code,
1009 jit_float32_t u)
1010 {
1011 jit_node_t *node = new_node(code);
1012 assert(!_jitc->realize);
1013 node->u.f = u;
1014 return (link_node(node));
1015 }
1016
1017 jit_node_t *
_jit_new_node_d(jit_state_t * _jit,jit_code_t code,jit_float64_t u)1018 _jit_new_node_d(jit_state_t *_jit, jit_code_t code,
1019 jit_float64_t u)
1020 {
1021 jit_node_t *node = new_node(code);
1022 assert(!_jitc->realize);
1023 node->u.d = u;
1024 return (link_node(node));
1025 }
1026
1027 jit_node_t *
_jit_new_node_p(jit_state_t * _jit,jit_code_t code,jit_pointer_t u)1028 _jit_new_node_p(jit_state_t *_jit, jit_code_t code,
1029 jit_pointer_t u)
1030 {
1031 jit_node_t *node = new_node(code);
1032 assert(!_jitc->realize);
1033 node->u.p = u;
1034 return (link_node(node));
1035 }
1036
1037 jit_node_t *
_jit_new_node_ww(jit_state_t * _jit,jit_code_t code,jit_word_t u,jit_word_t v)1038 _jit_new_node_ww(jit_state_t *_jit, jit_code_t code,
1039 jit_word_t u, jit_word_t v)
1040 {
1041 jit_node_t *node = new_node(code);
1042 assert(!_jitc->realize);
1043 node->u.w = u;
1044 node->v.w = v;
1045 return (link_node(node));
1046 }
1047
1048 jit_node_t *
_jit_new_node_wp(jit_state_t * _jit,jit_code_t code,jit_word_t u,jit_pointer_t v)1049 _jit_new_node_wp(jit_state_t *_jit, jit_code_t code,
1050 jit_word_t u, jit_pointer_t v)
1051 {
1052 return (jit_new_node_ww(code, u, (jit_word_t)v));
1053 }
1054
1055 jit_node_t *
_jit_new_node_fp(jit_state_t * _jit,jit_code_t code,jit_float32_t u,jit_pointer_t v)1056 _jit_new_node_fp(jit_state_t *_jit, jit_code_t code,
1057 jit_float32_t u, jit_pointer_t v)
1058 {
1059 jit_node_t *node = new_node(code);
1060 assert(!_jitc->realize);
1061 node->u.f = u;
1062 node->v.w = (jit_word_t)v;
1063 return (link_node(node));
1064 }
1065
1066 jit_node_t *
_jit_new_node_dp(jit_state_t * _jit,jit_code_t code,jit_float64_t u,jit_pointer_t v)1067 _jit_new_node_dp(jit_state_t *_jit, jit_code_t code,
1068 jit_float64_t u, jit_pointer_t v)
1069 {
1070 jit_node_t *node = new_node(code);
1071 assert(!_jitc->realize);
1072 node->u.d = u;
1073 node->v.w = (jit_word_t)v;
1074 return (link_node(node));
1075 }
1076
1077 jit_node_t *
_jit_new_node_pw(jit_state_t * _jit,jit_code_t code,jit_pointer_t u,jit_word_t v)1078 _jit_new_node_pw(jit_state_t *_jit, jit_code_t code,
1079 jit_pointer_t u, jit_word_t v)
1080 {
1081 return (jit_new_node_ww(code, (jit_word_t)u, v));
1082 }
1083
1084 jit_node_t *
_jit_new_node_wf(jit_state_t * _jit,jit_code_t code,jit_word_t u,jit_float32_t v)1085 _jit_new_node_wf(jit_state_t *_jit, jit_code_t code,
1086 jit_word_t u, jit_float32_t v)
1087 {
1088 jit_node_t *node = new_node(code);
1089 assert(!_jitc->realize);
1090 node->u.w = u;
1091 node->v.f = v;
1092 return (link_node(node));
1093 }
1094
1095 jit_node_t *
_jit_new_node_wd(jit_state_t * _jit,jit_code_t code,jit_word_t u,jit_float64_t v)1096 _jit_new_node_wd(jit_state_t *_jit, jit_code_t code,
1097 jit_word_t u, jit_float64_t v)
1098 {
1099 jit_node_t *node = new_node(code);
1100 assert(!_jitc->realize);
1101 node->u.w = u;
1102 node->v.d = v;
1103 return (link_node(node));
1104 }
1105
1106 jit_node_t *
_jit_new_node_www(jit_state_t * _jit,jit_code_t code,jit_word_t u,jit_word_t v,jit_word_t w)1107 _jit_new_node_www(jit_state_t *_jit, jit_code_t code,
1108 jit_word_t u, jit_word_t v, jit_word_t w)
1109 {
1110 jit_node_t *node = new_node(code);
1111 assert(!_jitc->realize);
1112 node->u.w = u;
1113 node->v.w = v;
1114 node->w.w = w;
1115 return (link_node(node));
1116 }
1117
1118 jit_node_t *
_jit_new_node_qww(jit_state_t * _jit,jit_code_t code,jit_int32_t l,jit_int32_t h,jit_word_t v,jit_word_t w)1119 _jit_new_node_qww(jit_state_t *_jit, jit_code_t code,
1120 jit_int32_t l, jit_int32_t h,
1121 jit_word_t v, jit_word_t w)
1122 {
1123 jit_node_t *node = new_node(code);
1124 assert(!_jitc->realize);
1125 assert(l != h);
1126 node->u.q.l = l;
1127 node->u.q.h = h;
1128 node->v.w = v;
1129 node->w.w = w;
1130 return (link_node(node));
1131 }
1132
1133 jit_node_t *
_jit_new_node_wwf(jit_state_t * _jit,jit_code_t code,jit_word_t u,jit_word_t v,jit_float32_t w)1134 _jit_new_node_wwf(jit_state_t *_jit, jit_code_t code,
1135 jit_word_t u, jit_word_t v, jit_float32_t w)
1136 {
1137 jit_node_t *node = new_node(code);
1138 assert(!_jitc->realize);
1139 node->u.w = u;
1140 node->v.w = v;
1141 node->w.f = w;
1142 return (link_node(node));
1143 }
1144
1145 jit_node_t *
_jit_new_node_wwd(jit_state_t * _jit,jit_code_t code,jit_word_t u,jit_word_t v,jit_float64_t w)1146 _jit_new_node_wwd(jit_state_t *_jit, jit_code_t code,
1147 jit_word_t u, jit_word_t v, jit_float64_t w)
1148 {
1149 jit_node_t *node = new_node(code);
1150 assert(!_jitc->realize);
1151 node->u.w = u;
1152 node->v.w = v;
1153 node->w.d = w;
1154 return (link_node(node));
1155 }
1156
1157 jit_node_t *
_jit_new_node_pww(jit_state_t * _jit,jit_code_t code,jit_pointer_t u,jit_word_t v,jit_word_t w)1158 _jit_new_node_pww(jit_state_t *_jit, jit_code_t code,
1159 jit_pointer_t u, jit_word_t v, jit_word_t w)
1160 {
1161 jit_node_t *node = new_node(code);
1162 assert(!_jitc->realize);
1163 node->u.p = u;
1164 node->v.w = v;
1165 node->w.w = w;
1166 return (link_node(node));
1167 }
1168
1169 jit_node_t *
_jit_new_node_pwf(jit_state_t * _jit,jit_code_t code,jit_pointer_t u,jit_word_t v,jit_float32_t w)1170 _jit_new_node_pwf(jit_state_t *_jit, jit_code_t code,
1171 jit_pointer_t u, jit_word_t v, jit_float32_t w)
1172 {
1173 jit_node_t *node = new_node(code);
1174 assert(!_jitc->realize);
1175 node->u.p = u;
1176 node->v.w = v;
1177 node->w.f = w;
1178 return (link_node(node));
1179 }
1180
1181 jit_node_t *
_jit_new_node_pwd(jit_state_t * _jit,jit_code_t code,jit_pointer_t u,jit_word_t v,jit_float64_t w)1182 _jit_new_node_pwd(jit_state_t *_jit, jit_code_t code,
1183 jit_pointer_t u, jit_word_t v, jit_float64_t w)
1184 {
1185 jit_node_t *node = new_node(code);
1186 assert(!_jitc->realize);
1187 node->u.p = u;
1188 node->v.w = v;
1189 node->w.d = w;
1190 return (link_node(node));
1191 }
1192
1193 jit_node_t *
_jit_label(jit_state_t * _jit)1194 _jit_label(jit_state_t *_jit)
1195 {
1196 jit_node_t *node;
1197
1198 if (!(node = _jitc->tail) || node->code != jit_code_label) {
1199 node = jit_forward();
1200 jit_link(node);
1201 }
1202
1203 return (node);
1204 }
1205
1206 jit_node_t *
_jit_forward(jit_state_t * _jit)1207 _jit_forward(jit_state_t *_jit)
1208 {
1209 return (jit_new_node_no_link(jit_code_label));
1210 }
1211
1212 jit_node_t *
_jit_indirect(jit_state_t * _jit)1213 _jit_indirect(jit_state_t *_jit)
1214 {
1215 jit_node_t *node;
1216
1217 node = jit_label();
1218 node->flag |= jit_flag_use;
1219
1220 return (node);
1221 }
1222
1223 void
_jit_link(jit_state_t * _jit,jit_node_t * node)1224 _jit_link(jit_state_t *_jit, jit_node_t *node)
1225 {
1226 jit_block_t *block;
1227
1228 assert((node->code == jit_code_label ||
1229 node->code == jit_code_prolog ||
1230 node->code == jit_code_epilog) && !node->next);
1231 jit_link_node(node);
1232 if (_jitc->blocks.offset >= _jitc->blocks.length) {
1233 jit_word_t length;
1234
1235 length = _jitc->blocks.length + 16;
1236 jit_realloc((jit_pointer_t *)&_jitc->blocks.ptr,
1237 _jitc->blocks.length * sizeof(jit_block_t),
1238 length * sizeof(jit_block_t));
1239 _jitc->blocks.length = length;
1240 }
1241 block = _jitc->blocks.ptr + _jitc->blocks.offset;
1242 block->label = node;
1243 node->v.w = _jitc->blocks.offset;
1244 jit_regset_new(&block->reglive);
1245 jit_regset_new(&block->regmask);
1246 ++_jitc->blocks.offset;
1247 }
1248
1249 jit_bool_t
_jit_forward_p(jit_state_t * _jit,jit_node_t * node)1250 _jit_forward_p(jit_state_t *_jit, jit_node_t *node)
1251 {
1252 return (node->code == jit_code_label && !node->next && node != _jitc->tail);
1253 }
1254
1255 jit_bool_t
_jit_indirect_p(jit_state_t * _jit,jit_node_t * node)1256 _jit_indirect_p(jit_state_t *_jit, jit_node_t *node)
1257 {
1258 return (node->code == jit_code_label && !!(node->flag & jit_flag_use));
1259 }
1260
1261 jit_bool_t
_jit_target_p(jit_state_t * _jit,jit_node_t * node)1262 _jit_target_p(jit_state_t *_jit, jit_node_t *node)
1263 {
1264 return (node->code == jit_code_label && !!node->link);
1265 }
1266
1267 void
_jit_prepare(jit_state_t * _jit)1268 _jit_prepare(jit_state_t *_jit)
1269 {
1270 assert(_jitc->function != NULL);
1271 _jitc->function->call.call = jit_call_default;
1272 _jitc->function->call.argi =
1273 _jitc->function->call.argf =
1274 _jitc->function->call.size = 0;
1275 _jitc->prepare = jit_new_node(jit_code_prepare);
1276 }
1277
1278 void
_jit_patch(jit_state_t * _jit,jit_node_t * instr)1279 _jit_patch(jit_state_t* _jit, jit_node_t *instr)
1280 {
1281 jit_node_t *label;
1282
1283 if (!(label = _jitc->tail) || label->code != jit_code_label)
1284 label = jit_label();
1285 jit_patch_at(instr, label);
1286 }
1287
1288 jit_int32_t
_jit_classify(jit_state_t * _jit,jit_code_t code)1289 _jit_classify(jit_state_t *_jit, jit_code_t code)
1290 {
1291 jit_int32_t mask;
1292
1293 switch (code) {
1294 case jit_code_data: case jit_code_save: case jit_code_load:
1295 case jit_code_name: case jit_code_label: case jit_code_note:
1296 case jit_code_prolog: case jit_code_ellipsis: case jit_code_va_push:
1297 case jit_code_epilog: case jit_code_ret: case jit_code_prepare:
1298 mask = 0;
1299 break;
1300 case jit_code_live: case jit_code_va_end:
1301 case jit_code_retr: case jit_code_retr_f: case jit_code_retr_d:
1302 case jit_code_pushargr: case jit_code_pushargr_f:
1303 case jit_code_pushargr_d:
1304 case jit_code_finishr: /* synthesized will set jit_cc_a0_jmp */
1305 mask = jit_cc_a0_reg;
1306 break;
1307 case jit_code_align: case jit_code_reti: case jit_code_pushargi:
1308 case jit_code_finishi: /* synthesized will set jit_cc_a0_jmp */
1309 mask = jit_cc_a0_int;
1310 break;
1311 case jit_code_reti_f: case jit_code_pushargi_f:
1312 mask = jit_cc_a0_flt;
1313 break;
1314 case jit_code_reti_d: case jit_code_pushargi_d:
1315 mask = jit_cc_a0_dbl;
1316 break;
1317 case jit_code_allocai:
1318 mask = jit_cc_a0_int|jit_cc_a1_int;
1319 break;
1320 case jit_code_arg: case jit_code_arg_f: case jit_code_arg_d:
1321 mask = jit_cc_a0_int|jit_cc_a0_arg;
1322 break;
1323 case jit_code_calli: case jit_code_jmpi:
1324 mask = jit_cc_a0_jmp;
1325 break;
1326 case jit_code_callr: case jit_code_jmpr:
1327 mask = jit_cc_a0_reg|jit_cc_a0_jmp;
1328 break;
1329 case jit_code_retval_c: case jit_code_retval_uc:
1330 case jit_code_retval_s: case jit_code_retval_us:
1331 case jit_code_retval_i: case jit_code_retval_ui:
1332 case jit_code_retval_l:
1333 case jit_code_retval_f: case jit_code_retval_d:
1334 case jit_code_va_start:
1335 mask = jit_cc_a0_reg|jit_cc_a0_chg;
1336 break;
1337 case jit_code_getarg_c: case jit_code_getarg_uc:
1338 case jit_code_getarg_s: case jit_code_getarg_us:
1339 case jit_code_getarg_i: case jit_code_getarg_ui:
1340 case jit_code_getarg_l:
1341 case jit_code_getarg_f: case jit_code_getarg_d:
1342 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_arg;
1343 break;
1344 case jit_code_putargr: case jit_code_putargr_f:
1345 case jit_code_putargr_d:
1346 mask = jit_cc_a0_reg|jit_cc_a1_arg;
1347 break;
1348 case jit_code_putargi:
1349 mask = jit_cc_a0_int|jit_cc_a1_arg;
1350 break;
1351 case jit_code_putargi_f:
1352 mask = jit_cc_a0_flt|jit_cc_a1_arg;
1353 break;
1354 case jit_code_putargi_d:
1355 mask = jit_cc_a0_dbl|jit_cc_a1_arg;
1356 break;
1357 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1358 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1359 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1360 case jit_code_ldi_d:
1361 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_int;
1362 break;
1363 case jit_code_movi_f: case jit_code_movi_f_w:
1364 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_flt;
1365 break;
1366 case jit_code_movi_d: case jit_code_movi_d_w:
1367 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_dbl;
1368 break;
1369 case jit_code_movi_d_ww:
1370 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1371 jit_cc_a2_dbl;
1372 break;
1373 case jit_code_negr: case jit_code_comr: case jit_code_movr:
1374 case jit_code_extr_c: case jit_code_extr_uc: case jit_code_extr_s:
1375 case jit_code_extr_us: case jit_code_extr_i: case jit_code_extr_ui:
1376 case jit_code_truncr_f_i: case jit_code_truncr_f_l:
1377 case jit_code_truncr_d_i: case jit_code_truncr_d_l:
1378 case jit_code_htonr_us: case jit_code_htonr_ui: case jit_code_htonr_ul:
1379 case jit_code_ldr_c: case jit_code_ldr_uc:
1380 case jit_code_ldr_s: case jit_code_ldr_us: case jit_code_ldr_i:
1381 case jit_code_ldr_ui: case jit_code_ldr_l: case jit_code_negr_f:
1382 case jit_code_absr_f: case jit_code_sqrtr_f: case jit_code_movr_f:
1383 case jit_code_extr_f: case jit_code_extr_d_f: case jit_code_ldr_f:
1384 case jit_code_negr_d: case jit_code_absr_d: case jit_code_sqrtr_d:
1385 case jit_code_movr_d: case jit_code_extr_d: case jit_code_extr_f_d:
1386 case jit_code_ldr_d:
1387 case jit_code_movr_w_f: case jit_code_movr_f_w:
1388 case jit_code_movr_w_d: case jit_code_movr_d_w:
1389 case jit_code_va_arg: case jit_code_va_arg_d:
1390 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg;
1391 break;
1392 case jit_code_movr_d_ww:
1393 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a1_chg|
1394 jit_cc_a2_reg;
1395 break;
1396 case jit_code_addi: case jit_code_addxi: case jit_code_addci:
1397 case jit_code_subi: case jit_code_subxi: case jit_code_subci:
1398 case jit_code_rsbi:
1399 case jit_code_muli: case jit_code_divi: case jit_code_divi_u:
1400 case jit_code_remi: case jit_code_remi_u: case jit_code_andi:
1401 case jit_code_ori: case jit_code_xori: case jit_code_lshi:
1402 case jit_code_rshi: case jit_code_rshi_u: case jit_code_lti:
1403 case jit_code_lti_u: case jit_code_lei: case jit_code_lei_u:
1404 case jit_code_eqi: case jit_code_gei: case jit_code_gei_u:
1405 case jit_code_gti: case jit_code_gti_u: case jit_code_nei:
1406 case jit_code_ldxi_c: case jit_code_ldxi_uc: case jit_code_ldxi_s:
1407 case jit_code_ldxi_us: case jit_code_ldxi_i: case jit_code_ldxi_ui:
1408 case jit_code_ldxi_l: case jit_code_ldxi_f: case jit_code_ldxi_d:
1409 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_int;
1410 break;
1411 case jit_code_qmuli: case jit_code_qmuli_u:
1412 case jit_code_qdivi: case jit_code_qdivi_u:
1413 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1414 jit_cc_a1_reg|jit_cc_a2_int;
1415 break;
1416 case jit_code_addi_f: case jit_code_subi_f: case jit_code_rsbi_f:
1417 case jit_code_muli_f: case jit_code_divi_f: case jit_code_lti_f:
1418 case jit_code_lei_f: case jit_code_eqi_f: case jit_code_gei_f:
1419 case jit_code_gti_f: case jit_code_nei_f: case jit_code_unlti_f:
1420 case jit_code_unlei_f: case jit_code_uneqi_f: case jit_code_ungei_f:
1421 case jit_code_ungti_f: case jit_code_ltgti_f: case jit_code_ordi_f:
1422 case jit_code_unordi_f:
1423 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_flt;
1424 break;
1425 case jit_code_addi_d: case jit_code_subi_d: case jit_code_rsbi_d:
1426 case jit_code_muli_d: case jit_code_divi_d: case jit_code_lti_d:
1427 case jit_code_lei_d: case jit_code_eqi_d: case jit_code_gei_d:
1428 case jit_code_gti_d: case jit_code_nei_d: case jit_code_unlti_d:
1429 case jit_code_unlei_d: case jit_code_uneqi_d: case jit_code_ungei_d:
1430 case jit_code_ungti_d: case jit_code_ltgti_d: case jit_code_ordi_d:
1431 case jit_code_unordi_d:
1432 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_dbl;
1433 break;
1434 case jit_code_addr: case jit_code_addxr: case jit_code_addcr:
1435 case jit_code_subr: case jit_code_subxr: case jit_code_subcr:
1436 case jit_code_mulr: case jit_code_divr: case jit_code_divr_u:
1437 case jit_code_remr: case jit_code_remr_u: case jit_code_andr:
1438 case jit_code_orr: case jit_code_xorr: case jit_code_lshr:
1439 case jit_code_rshr: case jit_code_rshr_u: case jit_code_ltr:
1440 case jit_code_ltr_u: case jit_code_ler: case jit_code_ler_u:
1441 case jit_code_eqr: case jit_code_ger: case jit_code_ger_u:
1442 case jit_code_gtr: case jit_code_gtr_u: case jit_code_ner:
1443 case jit_code_ldxr_c: case jit_code_ldxr_uc: case jit_code_ldxr_s:
1444 case jit_code_ldxr_us: case jit_code_ldxr_i: case jit_code_ldxr_ui:
1445 case jit_code_ldxr_l: case jit_code_addr_f: case jit_code_subr_f:
1446 case jit_code_mulr_f: case jit_code_divr_f: case jit_code_ltr_f:
1447 case jit_code_ler_f: case jit_code_eqr_f: case jit_code_ger_f:
1448 case jit_code_gtr_f: case jit_code_ner_f: case jit_code_unltr_f:
1449 case jit_code_unler_f: case jit_code_uneqr_f: case jit_code_unger_f:
1450 case jit_code_ungtr_f: case jit_code_ltgtr_f: case jit_code_ordr_f:
1451 case jit_code_unordr_f: case jit_code_ldxr_f: case jit_code_addr_d:
1452 case jit_code_subr_d: case jit_code_mulr_d: case jit_code_divr_d:
1453 case jit_code_ltr_d: case jit_code_ler_d: case jit_code_eqr_d:
1454 case jit_code_ger_d: case jit_code_gtr_d: case jit_code_ner_d:
1455 case jit_code_unltr_d: case jit_code_unler_d: case jit_code_uneqr_d:
1456 case jit_code_unger_d: case jit_code_ungtr_d: case jit_code_ltgtr_d:
1457 case jit_code_ordr_d: case jit_code_unordr_d: case jit_code_ldxr_d:
1458 case jit_code_movr_ww_d:
1459 mask = jit_cc_a0_reg|jit_cc_a0_chg|jit_cc_a1_reg|jit_cc_a2_reg;
1460 break;
1461 case jit_code_qmulr: case jit_code_qmulr_u:
1462 case jit_code_qdivr: case jit_code_qdivr_u:
1463 mask = jit_cc_a0_reg|jit_cc_a0_rlh|jit_cc_a0_chg|
1464 jit_cc_a1_reg|jit_cc_a2_reg;
1465 break;
1466 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1467 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1468 mask = jit_cc_a0_int|jit_cc_a1_reg;
1469 break;
1470 case jit_code_blti: case jit_code_blti_u: case jit_code_blei:
1471 case jit_code_blei_u: case jit_code_beqi: case jit_code_bgei:
1472 case jit_code_bgei_u: case jit_code_bgti: case jit_code_bgti_u:
1473 case jit_code_bnei: case jit_code_bmsi: case jit_code_bmci:
1474 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_int;
1475 break;
1476 case jit_code_blti_f: case jit_code_blei_f: case jit_code_beqi_f:
1477 case jit_code_bgei_f: case jit_code_bgti_f: case jit_code_bnei_f:
1478 case jit_code_bunlti_f: case jit_code_bunlei_f: case jit_code_buneqi_f:
1479 case jit_code_bungei_f: case jit_code_bungti_f: case jit_code_bltgti_f:
1480 case jit_code_bordi_f: case jit_code_bunordi_f:
1481 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_flt;
1482 break;
1483 case jit_code_blti_d: case jit_code_blei_d: case jit_code_beqi_d:
1484 case jit_code_bgei_d: case jit_code_bgti_d: case jit_code_bnei_d:
1485 case jit_code_bunlti_d: case jit_code_bunlei_d: case jit_code_buneqi_d:
1486 case jit_code_bungei_d: case jit_code_bungti_d: case jit_code_bltgti_d:
1487 case jit_code_bordi_d: case jit_code_bunordi_d:
1488 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_dbl;
1489 break;
1490 case jit_code_allocar: /* synthesized instructions make it
1491 * equivalent to jit_cc_a0_chg */
1492 case jit_code_str_c: case jit_code_str_s: case jit_code_str_i:
1493 case jit_code_str_l: case jit_code_str_f: case jit_code_str_d:
1494 mask = jit_cc_a0_reg|jit_cc_a1_reg;
1495 break;
1496 case jit_code_stxi_c: case jit_code_stxi_s: case jit_code_stxi_i:
1497 case jit_code_stxi_l: case jit_code_stxi_f: case jit_code_stxi_d:
1498 mask = jit_cc_a0_int|jit_cc_a1_reg|jit_cc_a2_reg;
1499 break;
1500 case jit_code_bltr: case jit_code_bltr_u: case jit_code_bler:
1501 case jit_code_bler_u: case jit_code_beqr: case jit_code_bger:
1502 case jit_code_bger_u: case jit_code_bgtr: case jit_code_bgtr_u:
1503 case jit_code_bner: case jit_code_bmsr: case jit_code_bmcr:
1504 case jit_code_bltr_f: case jit_code_bler_f: case jit_code_beqr_f:
1505 case jit_code_bger_f: case jit_code_bgtr_f: case jit_code_bner_f:
1506 case jit_code_bunltr_f: case jit_code_bunler_f: case jit_code_buneqr_f:
1507 case jit_code_bunger_f: case jit_code_bungtr_f: case jit_code_bltgtr_f:
1508 case jit_code_bordr_f: case jit_code_bunordr_f:case jit_code_bltr_d:
1509 case jit_code_bler_d: case jit_code_beqr_d: case jit_code_bger_d:
1510 case jit_code_bgtr_d: case jit_code_bner_d: case jit_code_bunltr_d:
1511 case jit_code_bunler_d: case jit_code_buneqr_d: case jit_code_bunger_d:
1512 case jit_code_bungtr_d: case jit_code_bltgtr_d: case jit_code_bordr_d:
1513 case jit_code_bunordr_d:
1514 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a2_reg;
1515 break;
1516 case jit_code_boaddi: case jit_code_boaddi_u: case jit_code_bxaddi:
1517 case jit_code_bxaddi_u: case jit_code_bosubi: case jit_code_bosubi_u:
1518 case jit_code_bxsubi: case jit_code_bxsubi_u:
1519 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_int;
1520 break;
1521 case jit_code_stxr_c: case jit_code_stxr_s: case jit_code_stxr_i:
1522 case jit_code_stxr_l: case jit_code_stxr_f: case jit_code_stxr_d:
1523 mask = jit_cc_a0_reg|jit_cc_a1_reg|jit_cc_a2_reg;
1524 break;
1525 case jit_code_boaddr: case jit_code_boaddr_u: case jit_code_bxaddr:
1526 case jit_code_bxaddr_u: case jit_code_bosubr: case jit_code_bosubr_u:
1527 case jit_code_bxsubr: case jit_code_bxsubr_u:
1528 mask = jit_cc_a0_jmp|jit_cc_a1_reg|jit_cc_a1_chg|jit_cc_a2_reg;
1529 break;
1530 default:
1531 abort();
1532 }
1533
1534 return (mask);
1535 }
1536
1537 void
_jit_patch_abs(jit_state_t * _jit,jit_node_t * instr,jit_pointer_t address)1538 _jit_patch_abs(jit_state_t *_jit, jit_node_t *instr, jit_pointer_t address)
1539 {
1540 jit_int32_t mask;
1541
1542 switch (instr->code) {
1543 case jit_code_movi: case jit_code_ldi_c: case jit_code_ldi_uc:
1544 case jit_code_ldi_s: case jit_code_ldi_us: case jit_code_ldi_i:
1545 case jit_code_ldi_ui: case jit_code_ldi_l: case jit_code_ldi_f:
1546 case jit_code_ldi_d:
1547 instr->v.p = address;
1548 break;
1549 case jit_code_sti_c: case jit_code_sti_s: case jit_code_sti_i:
1550 case jit_code_sti_l: case jit_code_sti_f: case jit_code_sti_d:
1551 instr->u.p = address;
1552 break;
1553 default:
1554 mask = jit_classify(instr->code);
1555 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1556 instr->u.p = address;
1557 }
1558 }
1559
1560 void
_jit_patch_at(jit_state_t * _jit,jit_node_t * instr,jit_node_t * label)1561 _jit_patch_at(jit_state_t *_jit, jit_node_t *instr, jit_node_t *label)
1562 {
1563 jit_int32_t mask;
1564
1565 assert(!(instr->flag & jit_flag_node));
1566 instr->flag |= jit_flag_node;
1567 switch (instr->code) {
1568 case jit_code_movi:
1569 assert(label->code == jit_code_label ||
1570 label->code == jit_code_data);
1571 instr->v.n = label;
1572 if (label->code == jit_code_data)
1573 instr->flag |= jit_flag_data;
1574 break;
1575 case jit_code_jmpi:
1576 assert(label->code == jit_code_label ||
1577 label->code == jit_code_epilog);
1578 instr->u.n = label;
1579 break;
1580 default:
1581 mask = jit_classify(instr->code);
1582 assert((mask & (jit_cc_a0_reg|jit_cc_a0_jmp)) == jit_cc_a0_jmp);
1583 assert(label->code == jit_code_label);
1584 instr->u.n = label;
1585 break;
1586 }
1587 /* link field is used as list of nodes associated with a given label */
1588 instr->link = label->link;
1589 label->link = instr;
1590 }
1591
1592 void
_jit_optimize(jit_state_t * _jit)1593 _jit_optimize(jit_state_t *_jit)
1594 {
1595 jit_bool_t jump;
1596 jit_bool_t todo;
1597 jit_int32_t mask;
1598 jit_node_t *node;
1599 jit_block_t *block;
1600 jit_word_t offset;
1601
1602 _jitc->function = NULL;
1603
1604 thread_jumps();
1605 sequential_labels();
1606
1607 /* create initial mapping of live register values
1608 * at the start of a basic block */
1609 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1610 block = _jitc->blocks.ptr + offset;
1611 if (!block->label)
1612 continue;
1613 if (block->label->code != jit_code_epilog)
1614 jit_setup(block);
1615 }
1616
1617 /* set live state of registers not referenced in a block, but
1618 * referenced in a jump target or normal flow */
1619 do {
1620 todo = 0;
1621 for (offset = 0; offset < _jitc->blocks.offset; offset++) {
1622 block = _jitc->blocks.ptr + offset;
1623 if (!block->label)
1624 continue;
1625 if (block->label->code != jit_code_epilog)
1626 jit_follow(block, &todo);
1627 }
1628 } while (todo);
1629
1630 patch_registers();
1631 simplify();
1632
1633 /* figure out labels that are only reached with a jump
1634 * and is required to do a simple redundant_store removal
1635 * on jit_beqi below */
1636 jump = 1;
1637 for (node = _jitc->head; node; node = node->next) {
1638 switch (node->code) {
1639 case jit_code_label:
1640 if (!jump)
1641 node->flag |= jit_flag_head;
1642 break;
1643 case jit_code_jmpi: case jit_code_jmpr:
1644 case jit_code_epilog:
1645 jump = 1;
1646 break;
1647 case jit_code_data: case jit_code_note:
1648 break;
1649 default:
1650 jump = 0;
1651 break;
1652 }
1653 }
1654
1655 for (node = _jitc->head; node; node = node->next) {
1656 mask = jit_classify(node->code);
1657 if (mask & jit_cc_a0_reg)
1658 node->u.w &= ~jit_regno_patch;
1659 if (mask & jit_cc_a1_reg)
1660 node->v.w &= ~jit_regno_patch;
1661 if (mask & jit_cc_a2_reg)
1662 node->w.w &= ~jit_regno_patch;
1663 switch (node->code) {
1664 case jit_code_prolog:
1665 _jitc->function = _jitc->functions.ptr + node->w.w;
1666 break;
1667 case jit_code_epilog:
1668 _jitc->function = NULL;
1669 break;
1670 case jit_code_beqi:
1671 redundant_store(node, 1);
1672 break;
1673 case jit_code_bnei:
1674 redundant_store(node, 0);
1675 break;
1676 default:
1677 #if JIT_HASH_CONSTS
1678 if (mask & jit_cc_a0_flt) {
1679 node->u.p = jit_data(&node->u.f, sizeof(jit_float32_t), 4);
1680 node->flag |= jit_flag_node | jit_flag_data;
1681 }
1682 else if (mask & jit_cc_a0_dbl) {
1683 node->u.p = jit_data(&node->u.d, sizeof(jit_float64_t), 8);
1684 node->flag |= jit_flag_node | jit_flag_data;
1685 }
1686 else if (mask & jit_cc_a1_flt) {
1687 node->v.p = jit_data(&node->v.f, sizeof(jit_float32_t), 4);
1688 node->flag |= jit_flag_node | jit_flag_data;
1689 }
1690 else if (mask & jit_cc_a1_dbl) {
1691 node->v.p = jit_data(&node->v.d, sizeof(jit_float64_t), 8);
1692 node->flag |= jit_flag_node | jit_flag_data;
1693 }
1694 else if (mask & jit_cc_a2_flt) {
1695 node->w.p = jit_data(&node->w.f, sizeof(jit_float32_t), 4);
1696 node->flag |= jit_flag_node | jit_flag_data;
1697 }
1698 else if (mask & jit_cc_a2_dbl) {
1699 node->w.p = jit_data(&node->w.d, sizeof(jit_float64_t), 8);
1700 node->flag |= jit_flag_node | jit_flag_data;
1701 }
1702 #endif
1703 if (_jitc->function) {
1704 if ((mask & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
1705 (jit_cc_a0_reg|jit_cc_a0_chg)) {
1706 if (mask & jit_cc_a0_rlh) {
1707 jit_regset_setbit(&_jitc->function->regset,
1708 jit_regno(node->u.q.l));
1709 jit_regset_setbit(&_jitc->function->regset,
1710 jit_regno(node->u.q.h));
1711 }
1712 else
1713 jit_regset_setbit(&_jitc->function->regset,
1714 jit_regno(node->u.w));
1715 }
1716 if ((mask & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
1717 (jit_cc_a1_reg|jit_cc_a1_chg))
1718 jit_regset_setbit(&_jitc->function->regset,
1719 jit_regno(node->v.w));
1720 if ((mask & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
1721 (jit_cc_a2_reg|jit_cc_a2_chg))
1722 jit_regset_setbit(&_jitc->function->regset,
1723 jit_regno(node->w.w));
1724 }
1725 break;
1726 }
1727 }
1728 }
1729
1730 void
_jit_reglive(jit_state_t * _jit,jit_node_t * node)1731 _jit_reglive(jit_state_t *_jit, jit_node_t *node)
1732 {
1733 jit_int32_t spec;
1734 jit_int32_t value;
1735 jit_block_t *block;
1736
1737 switch (node->code) {
1738 case jit_code_label: case jit_code_prolog: case jit_code_epilog:
1739 block = _jitc->blocks.ptr + node->v.w;
1740 jit_regset_set(&_jitc->reglive, &block->reglive);
1741 break;
1742 case jit_code_callr:
1743 value = jit_regno(node->u.w);
1744 if (!(node->u.w & jit_regno_patch)) {
1745 jit_regset_setbit(&_jitc->reglive, value);
1746 }
1747 case jit_code_calli:
1748 for (value = 0; value < _jitc->reglen; value++) {
1749 spec = jit_class(_rvs[value].spec);
1750 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
1751 jit_regset_setbit(&_jitc->reglive, value);
1752 else if (!(spec & jit_class_sav))
1753 jit_regset_clrbit(&_jitc->reglive, value);
1754 }
1755 break;
1756 default:
1757 value = jit_classify(node->code);
1758 if (value & jit_cc_a0_reg) {
1759 if (value & jit_cc_a0_rlh) {
1760 if (!(node->u.q.l & jit_regno_patch)) {
1761 if (value & jit_cc_a0_chg) {
1762 jit_regset_clrbit(&_jitc->reglive, node->u.q.l);
1763 jit_regset_setbit(&_jitc->regmask, node->u.q.l);
1764 }
1765 else
1766 jit_regset_setbit(&_jitc->reglive, node->u.q.l);
1767 }
1768 if (!(node->u.q.h & jit_regno_patch)) {
1769 if (value & jit_cc_a0_chg) {
1770 jit_regset_clrbit(&_jitc->reglive, node->u.q.h);
1771 jit_regset_setbit(&_jitc->regmask, node->u.q.h);
1772 }
1773 else
1774 jit_regset_setbit(&_jitc->reglive, node->u.q.h);
1775 }
1776 }
1777 else {
1778 if (!(node->u.w & jit_regno_patch)) {
1779 if (value & jit_cc_a0_chg) {
1780 jit_regset_clrbit(&_jitc->reglive, node->u.w);
1781 jit_regset_setbit(&_jitc->regmask, node->u.w);
1782 }
1783 else
1784 jit_regset_setbit(&_jitc->reglive, node->u.w);
1785 }
1786 }
1787 }
1788 if ((value & jit_cc_a1_reg) && !(node->v.w & jit_regno_patch)) {
1789 if (value & jit_cc_a1_chg) {
1790 jit_regset_clrbit(&_jitc->reglive, node->v.w);
1791 jit_regset_setbit(&_jitc->regmask, node->v.w);
1792 }
1793 else
1794 jit_regset_setbit(&_jitc->reglive, node->v.w);
1795 }
1796 if ((value & jit_cc_a2_reg) && !(node->w.w & jit_regno_patch)) {
1797 if (value & jit_cc_a2_chg) {
1798 jit_regset_clrbit(&_jitc->reglive, node->w.w);
1799 jit_regset_setbit(&_jitc->regmask, node->w.w);
1800 }
1801 else
1802 jit_regset_setbit(&_jitc->reglive, node->w.w);
1803 }
1804 if (jit_regset_set_p(&_jitc->regmask)) {
1805 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
1806 if (jit_regset_set_p(&_jitc->regmask)) {
1807 /* any unresolved live state is considered as live */
1808 jit_regset_ior(&_jitc->reglive,
1809 &_jitc->reglive, &_jitc->regmask);
1810 jit_regset_set_ui(&_jitc->regmask, 0);
1811 }
1812 }
1813 break;
1814 }
1815 }
1816
1817 void
_jit_regarg_set(jit_state_t * _jit,jit_node_t * node,jit_int32_t value)1818 _jit_regarg_set(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
1819 {
1820 #if GET_JIT_SIZE
1821 jit_size_prepare();
1822 #endif
1823 if (value & jit_cc_a0_reg) {
1824 if (value & jit_cc_a0_rlh) {
1825 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.l));
1826 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.q.h));
1827 }
1828 else
1829 jit_regset_setbit(&_jitc->regarg, jit_regno(node->u.w));
1830 }
1831 if (value & jit_cc_a1_reg)
1832 jit_regset_setbit(&_jitc->regarg, jit_regno(node->v.w));
1833 if (value & jit_cc_a2_reg)
1834 jit_regset_setbit(&_jitc->regarg, jit_regno(node->w.w));
1835 }
1836
1837 void
_jit_regarg_clr(jit_state_t * _jit,jit_node_t * node,jit_int32_t value)1838 _jit_regarg_clr(jit_state_t *_jit, jit_node_t *node, jit_int32_t value)
1839 {
1840 #if GET_JIT_SIZE
1841 jit_size_collect(node);
1842 #endif
1843 if (value & jit_cc_a0_reg) {
1844 if (value & jit_cc_a0_rlh) {
1845 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.l));
1846 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.q.h));
1847 }
1848 else
1849 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->u.w));
1850 }
1851 if (value & jit_cc_a1_reg)
1852 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->v.w));
1853 if (value & jit_cc_a2_reg)
1854 jit_regset_clrbit(&_jitc->regarg, jit_regno(node->w.w));
1855 }
1856
1857 void
_jit_realize(jit_state_t * _jit)1858 _jit_realize(jit_state_t *_jit)
1859 {
1860 assert(!_jitc->realize);
1861 if (_jitc->function)
1862 jit_epilog();
1863 jit_optimize();
1864 _jitc->realize = 1;
1865
1866 /* ensure it is aligned */
1867 _jitc->data.offset = (_jitc->data.offset + 7) & -8;
1868
1869 #if GET_JIT_SIZE
1870 /* Heuristic to guess code buffer size */
1871 _jitc->mult = 4;
1872 _jit->code.length = _jitc->pool.length * 1024 * _jitc->mult;
1873 #else
1874 _jit->code.length = jit_get_size();
1875 #endif
1876 }
1877
1878 void
_jit_dataset(jit_state_t * _jit)1879 _jit_dataset(jit_state_t *_jit)
1880 {
1881 jit_uint8_t *ptr;
1882 jit_node_t *node;
1883 jit_word_t offset;
1884 #if defined(__sgi)
1885 int mmap_fd;
1886 #endif
1887
1888 assert(!_jitc->dataset);
1889 if (!_jit->user_data) {
1890
1891 /* create read only data buffer */
1892 _jit->data.length = (_jitc->data.offset +
1893 /* reserve space for annotations */
1894 _jitc->note.size + 4095) & -4096;
1895 #if defined(__sgi)
1896 mmap_fd = open("/dev/zero", O_RDWR);
1897 #endif
1898 _jit->data.ptr = mmap(NULL, _jit->data.length,
1899 PROT_READ | PROT_WRITE,
1900 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
1901 assert(_jit->data.ptr != MAP_FAILED);
1902 #if defined(__sgi)
1903 close(mmap_fd);
1904 #endif
1905 }
1906
1907 if (!_jitc->no_data)
1908 jit_memcpy(_jit->data.ptr, _jitc->data.ptr, _jitc->data.offset);
1909
1910 if (_jitc->no_note) {
1911 /* Space for one note is always allocated, so revert it here
1912 * if after jit_new_state was called, it is also requested to
1913 * not generate annotation information */
1914 _jit->note.length = 0;
1915 _jitc->note.size = 0;
1916 }
1917 else {
1918 _jitc->note.base = _jit->data.ptr;
1919 if (!_jitc->no_data)
1920 _jitc->note.base += _jitc->data.offset;
1921 memset(_jitc->note.base, 0, _jitc->note.size);
1922 }
1923
1924 if (_jit->user_data)
1925 /* Need the temporary hashed data until jit_emit is finished */
1926 ptr = _jitc->no_data ? _jitc->data.ptr : _jit->data.ptr;
1927 else {
1928 ptr = _jit->data.ptr;
1929 /* Temporary hashed data no longer required */
1930 jit_free((jit_pointer_t *)&_jitc->data.ptr);
1931 }
1932
1933 for (offset = 0; offset < _jitc->data.size; offset++) {
1934 for (node = _jitc->data.table[offset]; node; node = node->next) {
1935 node->flag |= jit_flag_patch;
1936 node->u.w = (jit_word_t)(ptr + node->u.w);
1937 }
1938 }
1939
1940 _jitc->dataset = 1;
1941 }
1942
1943 jit_pointer_t
_jit_get_code(jit_state_t * _jit,jit_word_t * length)1944 _jit_get_code(jit_state_t *_jit, jit_word_t *length)
1945 {
1946 assert(_jitc->realize);
1947 if (length) {
1948 if (_jitc->done)
1949 /* If code already generated, return exact size of code */
1950 *length = _jit->pc.uc - _jit->code.ptr;
1951 else
1952 /* Else return current size of the code buffer */
1953 *length = _jit->code.length;
1954 }
1955
1956 return (_jit->code.ptr);
1957 }
1958
1959 void
_jit_set_code(jit_state_t * _jit,jit_pointer_t ptr,jit_word_t length)1960 _jit_set_code(jit_state_t *_jit, jit_pointer_t ptr, jit_word_t length)
1961 {
1962 assert(_jitc->realize);
1963 _jit->code.ptr = ptr;
1964 _jit->code.length = length;
1965 _jit->user_code = 1;
1966 }
1967
1968 jit_pointer_t
_jit_get_data(jit_state_t * _jit,jit_word_t * data_size,jit_word_t * note_size)1969 _jit_get_data(jit_state_t *_jit, jit_word_t *data_size, jit_word_t *note_size)
1970 {
1971 assert(_jitc->realize);
1972 if (data_size)
1973 *data_size = _jitc->data.offset;
1974 if (note_size)
1975 *note_size = _jitc->note.size;
1976 return (_jit->data.ptr);
1977 }
1978
1979 void
_jit_set_data(jit_state_t * _jit,jit_pointer_t ptr,jit_word_t length,jit_word_t flags)1980 _jit_set_data(jit_state_t *_jit, jit_pointer_t ptr,
1981 jit_word_t length, jit_word_t flags)
1982 {
1983 assert(_jitc->realize);
1984 if (flags & JIT_DISABLE_DATA)
1985 _jitc->no_data = 1;
1986 else
1987 assert(length >= _jitc->data.offset);
1988 if (flags & JIT_DISABLE_NOTE)
1989 _jitc->no_note = 1;
1990 else {
1991 if (flags & JIT_DISABLE_DATA)
1992 assert(length >= _jitc->note.size);
1993 else
1994 assert(length >= _jitc->data.offset + _jitc->note.size);
1995 }
1996 _jit->data.ptr = ptr;
1997 _jit->data.length = length;
1998 _jit->user_data = 1;
1999 }
2000
2001 jit_pointer_t
_jit_emit(jit_state_t * _jit)2002 _jit_emit(jit_state_t *_jit)
2003 {
2004 jit_pointer_t code;
2005 jit_node_t *node;
2006 size_t length;
2007 int result;
2008 #if defined(__sgi)
2009 int mmap_fd;
2010 #endif
2011
2012 if (!_jitc->realize)
2013 jit_realize();
2014
2015 if (!_jitc->dataset)
2016 jit_dataset();
2017
2018 _jitc->emit = 1;
2019
2020 if (!_jit->user_code) {
2021 #if defined(__sgi)
2022 mmap_fd = open("/dev/zero", O_RDWR);
2023 #endif
2024 _jit->code.ptr = mmap(NULL, _jit->code.length,
2025 PROT_EXEC | PROT_READ | PROT_WRITE,
2026 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2027 assert(_jit->code.ptr != MAP_FAILED);
2028 }
2029 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2030 jit_get_max_instr();
2031 _jit->pc.uc = _jit->code.ptr;
2032
2033 for (;;) {
2034 if ((code = emit_code()) == NULL) {
2035 _jitc->patches.offset = 0;
2036 for (node = _jitc->head; node; node = node->next) {
2037 if (node->link &&
2038 (node->code == jit_code_label ||
2039 node->code == jit_code_epilog))
2040 node->flag &= ~jit_flag_patch;
2041 }
2042 if (_jit->user_code)
2043 goto fail;
2044 #if GET_JIT_SIZE
2045 ++_jitc->mult;
2046 length = _jitc->pool.length * 1024 * _jitc->mult;
2047 #else
2048 /* Should only happen on very special cases */
2049 length = _jit->code.length + 4096;
2050 #endif
2051
2052 #if !HAVE_MREMAP
2053 munmap(_jit->code.ptr, _jit->code.length);
2054 #endif
2055
2056 #if HAVE_MREMAP
2057 # if __NetBSD__
2058 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2059 _jit->code.ptr, length, 0);
2060 # else
2061 _jit->code.ptr = mremap(_jit->code.ptr, _jit->code.length,
2062 length, MREMAP_MAYMOVE, NULL);
2063 # endif
2064 #else
2065 _jit->code.ptr = mmap(NULL, length,
2066 PROT_EXEC | PROT_READ | PROT_WRITE,
2067 MAP_PRIVATE | MAP_ANON, mmap_fd, 0);
2068 #endif
2069
2070 assert(_jit->code.ptr != MAP_FAILED);
2071 _jit->code.length = length;
2072 _jitc->code.end = _jit->code.ptr + _jit->code.length -
2073 jit_get_max_instr();
2074 _jit->pc.uc = _jit->code.ptr;
2075 }
2076 else
2077 break;
2078 }
2079
2080 #if defined(__sgi)
2081 if (!_jit->user_code)
2082 close(mmap_fd);
2083 #endif
2084
2085 _jitc->done = 1;
2086 if (!_jitc->no_note)
2087 jit_annotate();
2088
2089 if (_jit->user_data)
2090 jit_free((jit_pointer_t *)&_jitc->data.ptr);
2091 else {
2092 result = mprotect(_jit->data.ptr, _jit->data.length, PROT_READ);
2093 assert(result == 0);
2094 }
2095 if (!_jit->user_code) {
2096 result = mprotect(_jit->code.ptr, _jit->code.length,
2097 PROT_READ | PROT_EXEC);
2098 assert(result == 0);
2099 }
2100
2101 return (_jit->code.ptr);
2102 fail:
2103 return (NULL);
2104 }
2105
2106 void
_jit_frame(jit_state_t * _jit,jit_int32_t frame)2107 _jit_frame(jit_state_t *_jit, jit_int32_t frame)
2108 {
2109 jit_trampoline(frame, 1);
2110 }
2111
2112 void
_jit_tramp(jit_state_t * _jit,jit_int32_t frame)2113 _jit_tramp(jit_state_t *_jit, jit_int32_t frame)
2114 {
2115 jit_trampoline(frame, 0);
2116 }
2117
2118 void
_jit_trampoline(jit_state_t * _jit,jit_int32_t frame,jit_bool_t prolog)2119 _jit_trampoline(jit_state_t *_jit, jit_int32_t frame, jit_bool_t prolog)
2120 {
2121 jit_int32_t regno;
2122
2123 /* Must be called after prolog, actually, just to simplify
2124 * tests and know there is a current function and that
2125 * _jitc->function->self.aoff is at the before any alloca value */
2126 assert(_jitc->tail && _jitc->tail->code == jit_code_prolog);
2127
2128 /* + 24 for 3 possible spilled temporaries (that could be a double) */
2129 frame += 24;
2130 #if defined(__hppa__)
2131 frame += _jitc->function->self.aoff;
2132 #else
2133 frame -= _jitc->function->self.aoff;
2134 #endif
2135 _jitc->function->frame = frame;
2136 if (prolog)
2137 _jitc->function->define_frame = 1;
2138 else
2139 _jitc->function->assume_frame = 1;
2140 for (regno = 0; regno < _jitc->reglen; regno++)
2141 if (jit_class(_rvs[regno].spec) & jit_class_sav)
2142 jit_regset_setbit(&_jitc->function->regset, regno);
2143 }
2144
2145 /* Compute initial reglive and regmask set values of a basic block.
2146 * reglive is the set of known live registers
2147 * regmask is the set of registers not referenced in the block
2148 * Registers in regmask might be live.
2149 */
2150 static void
_jit_setup(jit_state_t * _jit,jit_block_t * block)2151 _jit_setup(jit_state_t *_jit, jit_block_t *block)
2152 {
2153 #define reglive block->reglive
2154 #define regmask block->regmask
2155 jit_node_t *node;
2156 jit_bool_t live;
2157 unsigned long value;
2158
2159 jit_regset_set_mask(®mask, _jitc->reglen);
2160 for (value = 0; value < _jitc->reglen; ++value)
2161 if (!(jit_class(_rvs[value].spec) & (jit_class_gpr|jit_class_fpr)))
2162 jit_regset_clrbit(®mask, value);
2163
2164 for (node = block->label->next; node; node = node->next) {
2165 switch (node->code) {
2166 case jit_code_label: case jit_code_prolog:
2167 case jit_code_epilog:
2168 return;
2169 default:
2170 /* Check argument registers in reverse order to properly
2171 * handle registers that are both, argument and result */
2172 value = jit_classify(node->code);
2173 if ((value & jit_cc_a2_reg) &&
2174 !(node->w.w & jit_regno_patch) &&
2175 jit_regset_tstbit(®mask, node->w.w)) {
2176 live = !(value & jit_cc_a2_chg);
2177 jit_regset_clrbit(®mask, node->w.w);
2178 if (live)
2179 jit_regset_setbit(®live, node->w.w);
2180 }
2181 if ((value & jit_cc_a1_reg) &&
2182 !(node->v.w & jit_regno_patch) &&
2183 jit_regset_tstbit(®mask, node->v.w)) {
2184 live = !(value & jit_cc_a1_chg);
2185 jit_regset_clrbit(®mask, node->v.w);
2186 if (live)
2187 jit_regset_setbit(®live, node->v.w);
2188 }
2189 if (value & jit_cc_a0_reg) {
2190 live = !(value & jit_cc_a0_chg);
2191 if (value & jit_cc_a0_rlh) {
2192 if (!(node->u.q.l & jit_regno_patch) &&
2193 jit_regset_tstbit(®mask, node->u.q.l)) {
2194 jit_regset_clrbit(®mask, node->u.q.l);
2195 if (live)
2196 jit_regset_setbit(®live, node->u.q.l);
2197 }
2198 if (!(node->u.q.h & jit_regno_patch) &&
2199 jit_regset_tstbit(®mask, node->u.q.h)) {
2200 jit_regset_clrbit(®mask, node->u.q.h);
2201 if (live)
2202 jit_regset_setbit(®live, node->u.q.h);
2203 }
2204 }
2205 else {
2206 if (!(node->u.w & jit_regno_patch) &&
2207 jit_regset_tstbit(®mask, node->u.w)) {
2208 jit_regset_clrbit(®mask, node->u.w);
2209 if (live)
2210 jit_regset_setbit(®live, node->u.w);
2211 }
2212 }
2213 }
2214 break;
2215 }
2216 }
2217 #undef regmask
2218 #undef reglive
2219 }
2220
2221 /* Update regmask and reglive of blocks at entry point of branch targets
2222 * or normal flow that have a live register not used in this block.
2223 */
2224 static void
_jit_follow(jit_state_t * _jit,jit_block_t * block,jit_bool_t * todo)2225 _jit_follow(jit_state_t *_jit, jit_block_t *block, jit_bool_t *todo)
2226 {
2227 jit_node_t *node;
2228 jit_block_t *next;
2229 jit_int32_t spec;
2230 jit_int32_t regno;
2231 unsigned long value;
2232 jit_node_t *label;
2233 jit_regset_t reglive;
2234 jit_regset_t regmask;
2235 jit_regset_t regtemp;
2236
2237 jit_regset_set(®live, &block->reglive);
2238 jit_regset_set(®mask, &block->regmask);
2239 for (node = block->label->next; node; node = node->next) {
2240 switch (node->code) {
2241 case jit_code_label:
2242 /* Do not consider jmpi and jmpr cannot jump to the
2243 * next instruction. */
2244 next = _jitc->blocks.ptr + node->v.w;
2245 /* Set of live registers in next block that are at unknown
2246 * state in this block. */
2247 jit_regset_and(®temp, ®mask, &next->reglive);
2248 if (jit_regset_set_p(®temp)) {
2249 /* Add live state of next block to current block. */
2250 jit_regset_ior(&block->reglive, &block->reglive, ®temp);
2251 /* Remove from unknown state bitmask. */
2252 jit_regset_com(®temp, ®temp);
2253 jit_regset_and(&block->regmask, &block->regmask, ®temp);
2254 *todo = 1;
2255 }
2256 case jit_code_prolog:
2257 case jit_code_epilog:
2258 return;
2259 case jit_code_callr:
2260 value = jit_regno(node->u.w);
2261 if (!(node->u.w & jit_regno_patch)) {
2262 if (jit_regset_tstbit(®mask, value)) {
2263 jit_regset_clrbit(®mask, value);
2264 jit_regset_setbit(®live, value);
2265 }
2266 }
2267 case jit_code_calli:
2268 for (value = 0; value < _jitc->reglen; ++value) {
2269 value = jit_regset_scan1(®mask, value);
2270 if (value >= _jitc->reglen)
2271 break;
2272 spec = jit_class(_rvs[value].spec);
2273 if (!(spec & jit_class_sav))
2274 jit_regset_clrbit(®mask, value);
2275 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2276 jit_regset_setbit(®live, value);
2277 }
2278 break;
2279 default:
2280 value = jit_classify(node->code);
2281 if (value & jit_cc_a2_reg) {
2282 if (!(node->w.w & jit_regno_patch)) {
2283 if (jit_regset_tstbit(®mask, node->w.w)) {
2284 jit_regset_clrbit(®mask, node->w.w);
2285 if (!(value & jit_cc_a2_chg))
2286 jit_regset_setbit(®live, node->w.w);
2287 }
2288 }
2289 }
2290 if (value & jit_cc_a1_reg) {
2291 if (!(node->v.w & jit_regno_patch)) {
2292 if (jit_regset_tstbit(®mask, node->v.w)) {
2293 jit_regset_clrbit(®mask, node->v.w);
2294 if (!(value & jit_cc_a1_chg))
2295 jit_regset_setbit(®live, node->v.w);
2296 }
2297 }
2298 }
2299 if (value & jit_cc_a0_reg) {
2300 if (value & jit_cc_a0_rlh) {
2301 if (!(node->u.q.l & jit_regno_patch)) {
2302 if (jit_regset_tstbit(®mask, node->u.q.l)) {
2303 jit_regset_clrbit(®mask, node->u.q.l);
2304 if (!(value & jit_cc_a0_chg))
2305 jit_regset_setbit(®live, node->u.q.l);
2306 }
2307 }
2308 if (!(node->u.q.h & jit_regno_patch)) {
2309 if (jit_regset_tstbit(®mask, node->u.q.h)) {
2310 jit_regset_clrbit(®mask, node->u.q.h);
2311 if (!(value & jit_cc_a0_chg))
2312 jit_regset_setbit(®live, node->u.q.h);
2313 }
2314 }
2315 }
2316 else {
2317 if (!(node->u.w & jit_regno_patch)) {
2318 if (jit_regset_tstbit(®mask, node->u.w)) {
2319 jit_regset_clrbit(®mask, node->u.w);
2320 if (!(value & jit_cc_a0_chg))
2321 jit_regset_setbit(®live, node->u.w);
2322 }
2323 }
2324 }
2325 }
2326 if (value & jit_cc_a0_jmp) {
2327 if (node->flag & jit_flag_node) {
2328 label = node->u.n;
2329 /* Do not consider jmpi and jmpr cannot jump to the
2330 * next instruction. */
2331 next = _jitc->blocks.ptr + label->v.w;
2332 jit_regset_and(®temp, ®mask, &next->reglive);
2333 if (jit_regset_set_p(®temp)) {
2334 /* Add live state. */
2335 jit_regset_ior(&block->reglive,
2336 &block->reglive, ®temp);
2337 /* Remove from unknown state bitmask. */
2338 jit_regset_com(®temp, ®temp);
2339 jit_regset_and(&block->regmask,
2340 &block->regmask, ®temp);
2341 *todo = 1;
2342 }
2343 }
2344 else {
2345 /* Jump to unknown location.
2346 * This is a pitfall of the implementation.
2347 * Only jmpi to not a jit code should reach here,
2348 * or a jmpr of a computed address.
2349 * Because the implementation needs jit_class_nospill
2350 * registers, must treat jmpr as a function call. This
2351 * means that only JIT_Vn registers can be trusted on
2352 * arrival of jmpr.
2353 */
2354 for (regno = 0; regno < _jitc->reglen; regno++) {
2355 spec = jit_class(_rvs[regno].spec);
2356 if (jit_regset_tstbit(®mask, regno) &&
2357 (spec & (jit_class_gpr|jit_class_fpr)) &&
2358 !(spec & jit_class_sav))
2359 jit_regset_clrbit(®mask, regno);
2360 }
2361 /* Assume non callee save registers are live due
2362 * to jump to unknown location. */
2363 /* Treat all callee save as live. */
2364 jit_regset_ior(®live, ®live, ®mask);
2365 /* Treat anything else as dead. */
2366 jit_regset_set_ui(®mask, 0);
2367 }
2368 }
2369 break;
2370 }
2371 }
2372 }
2373
2374 /* Follow code generation up to finding a label or end of code.
2375 * When finding a label, update the set of live registers.
2376 * On branches, update based on taken branch or normal flow.
2377 */
2378 static void
_jit_update(jit_state_t * _jit,jit_node_t * node,jit_regset_t * live,jit_regset_t * mask)2379 _jit_update(jit_state_t *_jit, jit_node_t *node,
2380 jit_regset_t *live, jit_regset_t *mask)
2381 {
2382 jit_int32_t spec;
2383 jit_int32_t regno;
2384 unsigned long value;
2385 jit_block_t *block;
2386 jit_node_t *label;
2387 jit_regset_t regtemp;
2388
2389 for (; node; node = node->next) {
2390 if (jit_regset_set_p(mask) == 0)
2391 break;
2392 switch (node->code) {
2393 case jit_code_label:
2394 block = _jitc->blocks.ptr + node->v.w;
2395 jit_regset_and(®temp, mask, &block->reglive);
2396 if (jit_regset_set_p(®temp)) {
2397 /* Add live state. */
2398 jit_regset_ior(live, live, ®temp);
2399 /* Remove from unknown state bitmask. */
2400 jit_regset_com(®temp, ®temp);
2401 jit_regset_and(mask, mask, ®temp);
2402 }
2403 return;
2404 case jit_code_prolog:
2405 jit_regset_set_ui(mask, 0);
2406 return;
2407 case jit_code_epilog:
2408 jit_regset_set_ui(mask, 0);
2409 return;
2410 case jit_code_callr:
2411 value = jit_regno(node->u.w);
2412 if (!(node->u.w & jit_regno_patch)) {
2413 if (jit_regset_tstbit(mask, value)) {
2414 jit_regset_clrbit(mask, value);
2415 jit_regset_setbit(live, value);
2416 }
2417 }
2418 case jit_code_calli:
2419 for (value = 0; value < _jitc->reglen; ++value) {
2420 value = jit_regset_scan1(mask, value);
2421 if (value >= _jitc->reglen)
2422 break;
2423 spec = jit_class(_rvs[value].spec);
2424 if (!(spec & jit_class_sav))
2425 jit_regset_clrbit(mask, value);
2426 if ((spec & jit_class_arg) && jit_regarg_p(node, value))
2427 jit_regset_setbit(live, value);
2428 }
2429 break;
2430 default:
2431 value = jit_classify(node->code);
2432 if (value & jit_cc_a2_reg) {
2433 if (!(node->w.w & jit_regno_patch)) {
2434 if (jit_regset_tstbit(mask, node->w.w)) {
2435 jit_regset_clrbit(mask, node->w.w);
2436 if (!(value & jit_cc_a2_chg))
2437 jit_regset_setbit(live, node->w.w);
2438 }
2439 }
2440 }
2441 if (value & jit_cc_a1_reg) {
2442 if (!(node->v.w & jit_regno_patch)) {
2443 if (jit_regset_tstbit(mask, node->v.w)) {
2444 jit_regset_clrbit(mask, node->v.w);
2445 if (!(value & jit_cc_a1_chg))
2446 jit_regset_setbit(live, node->v.w);
2447 }
2448 }
2449 }
2450 if (value & jit_cc_a0_reg) {
2451 if (value & jit_cc_a0_rlh) {
2452 if (!(node->u.q.l & jit_regno_patch)) {
2453 if (jit_regset_tstbit(mask, node->u.q.l)) {
2454 jit_regset_clrbit(mask, node->u.q.l);
2455 if (!(value & jit_cc_a0_chg))
2456 jit_regset_setbit(live, node->u.q.l);
2457 }
2458 }
2459 if (!(node->u.q.h & jit_regno_patch)) {
2460 if (jit_regset_tstbit(mask, node->u.q.h)) {
2461 jit_regset_clrbit(mask, node->u.q.h);
2462 if (!(value & jit_cc_a0_chg))
2463 jit_regset_setbit(live, node->u.q.h);
2464 }
2465 }
2466 }
2467 else {
2468 if (!(node->u.w & jit_regno_patch)) {
2469 if (jit_regset_tstbit(mask, node->u.w)) {
2470 jit_regset_clrbit(mask, node->u.w);
2471 if (!(value & jit_cc_a0_chg))
2472 jit_regset_setbit(live, node->u.w);
2473 }
2474 }
2475 }
2476 }
2477 if (value & jit_cc_a0_jmp) {
2478 if (node->flag & jit_flag_node) {
2479 label = node->u.n;
2480 /* Do not consider jmpi and jmpr cannot jump to the
2481 * next instruction. */
2482 block = _jitc->blocks.ptr + label->v.w;
2483 jit_regset_and(®temp, mask, &block->reglive);
2484 if (jit_regset_set_p(®temp)) {
2485 /* Add live state. */
2486 jit_regset_ior(live, live, ®temp);
2487 /* Remove from unknown state bitmask. */
2488 jit_regset_com(®temp, ®temp);
2489 jit_regset_and(mask, mask, ®temp);
2490 }
2491 }
2492 else {
2493 /* Jump to unknown location.
2494 * This is a pitfall of the implementation.
2495 * Only jmpi to not a jit code should reach here,
2496 * or a jmpr of a computed address.
2497 * Because the implementation needs jit_class_nospill
2498 * registers, must treat jmpr as a function call. This
2499 * means that only JIT_Vn registers can be trusted on
2500 * arrival of jmpr.
2501 */
2502 for (regno = 0; regno < _jitc->reglen; regno++) {
2503 spec = jit_class(_rvs[regno].spec);
2504 if (jit_regset_tstbit(mask, regno) &&
2505 (spec & (jit_class_gpr|jit_class_fpr)) &&
2506 !(spec & jit_class_sav))
2507 jit_regset_clrbit(mask, regno);
2508 }
2509 /* Assume non callee save registers are live due
2510 * to jump to unknown location. */
2511 /* Treat all callee save as live. */
2512 jit_regset_ior(live, live, mask);
2513 /* Treat anything else as dead. */
2514 jit_regset_set_ui(mask, 0);
2515 }
2516 }
2517 break;
2518 }
2519 }
2520 }
2521
2522 static void
_thread_jumps(jit_state_t * _jit)2523 _thread_jumps(jit_state_t *_jit)
2524 {
2525 jit_node_t *prev;
2526 jit_node_t *node;
2527 jit_node_t *next;
2528 jit_int32_t mask;
2529
2530 for (prev = node = _jitc->head; node;) {
2531 next = node->next;
2532 switch (node->code) {
2533 case jit_code_jmpi:
2534 if (redundant_jump(prev, node)) {
2535 node = prev;
2536 continue;
2537 }
2538 if (shortcut_jump(prev, node))
2539 continue;
2540 break;
2541 case jit_code_jmpr:
2542 case jit_code_callr: case jit_code_calli:
2543 /* non optimizable jump like code */
2544 break;
2545 default:
2546 mask = jit_classify(node->code);
2547 if (mask & jit_cc_a0_jmp) {
2548 if (reverse_jump(prev, node) ||
2549 shortcut_jump(prev, node))
2550 continue;
2551 }
2552 break;
2553 }
2554 prev = node;
2555 node = next;
2556 }
2557 }
2558
2559 static void
_sequential_labels(jit_state_t * _jit)2560 _sequential_labels(jit_state_t *_jit)
2561 {
2562 jit_node_t *jump;
2563 jit_node_t *link;
2564 jit_node_t *prev;
2565 jit_node_t *next;
2566 jit_node_t *node;
2567
2568 for (prev = node = _jitc->head; node; node = next) {
2569 next = node->next;
2570 if (node->code == jit_code_label) {
2571 if (!node->flag) {
2572 if (!node->link) {
2573 del_label(prev, node);
2574 continue;
2575 }
2576 if (prev != node && prev->code == jit_code_label) {
2577 if ((jump = node->link)) {
2578 for (; jump; jump = link) {
2579 link = jump->link;
2580 jump->u.n = prev;
2581 jump->link = prev->link;
2582 prev->link = jump;
2583 }
2584 node->link = NULL;
2585 }
2586 del_label(prev, node);
2587 continue;
2588 }
2589 }
2590 if (next && next->code == jit_code_label && !next->flag) {
2591 if ((jump = next->link)) {
2592 for (; jump; jump = link) {
2593 link = jump->link;
2594 jump->u.n = node;
2595 jump->link = node->link;
2596 node->link = jump;
2597 }
2598 next->link = NULL;
2599 }
2600 del_label(node, next);
2601 next = node->next;
2602 continue;
2603 }
2604 }
2605 prev = node;
2606 }
2607 }
2608
2609 static jit_bool_t
_shortcut_jump(jit_state_t * _jit,jit_node_t * prev,jit_node_t * node)2610 _shortcut_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2611 {
2612 jit_bool_t cond;
2613 jit_node_t *jump;
2614 jit_node_t *next;
2615 jit_node_t *temp;
2616
2617 if (!(node->flag & jit_flag_node))
2618 return (0);
2619 assert(node->code != jit_code_jmpr);
2620 cond = node->code != jit_code_jmpi;
2621 jump = node->u.n;
2622 for (next = jump->next; next; next = next->next) {
2623 switch (next->code) {
2624 case jit_code_jmpi:
2625 if (!(next->flag & jit_flag_node))
2626 return (0);
2627 if (jump->link == node)
2628 jump->link = node->link;
2629 else {
2630 for (temp = jump->link;
2631 temp->link != node;
2632 temp = temp->link)
2633 assert(temp != NULL);
2634 temp->link = node->link;
2635 }
2636 jump = next->u.n;
2637 node->u.n = jump;
2638 node->link = jump->link;
2639 jump->link = node;
2640 return (1);
2641 case jit_code_jmpr:
2642 if (cond)
2643 return (0);
2644 node->code = jit_code_jmpr;
2645 node->u.w = next->u.w;
2646 node->link = NULL;
2647 node->flag &= ~jit_flag_node;
2648 return (1);
2649 case jit_code_note: case jit_code_label:
2650 break;
2651 default:
2652 return (0);
2653 }
2654 }
2655 return (0);
2656 }
2657
2658 static jit_bool_t
_redundant_jump(jit_state_t * _jit,jit_node_t * prev,jit_node_t * node)2659 _redundant_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2660 {
2661 jit_node_t *local_prev;
2662 jit_node_t *local_next;
2663
2664 if (!(node->flag & jit_flag_node))
2665 return (0);
2666 for (local_prev = node, local_next = node->next;
2667 local_next;
2668 local_prev = local_next, local_next = local_next->next) {
2669
2670 switch (local_next->code) {
2671 case jit_code_label: case jit_code_epilog:
2672 if (node->u.n == local_next) {
2673 if (local_next->link == node)
2674 local_next->link = node->link;
2675 else {
2676 for (local_prev = local_next->link;
2677 local_prev->link != node;
2678 local_prev = local_prev->link)
2679 assert(local_prev != NULL);
2680 local_prev->link = node->link;
2681 }
2682 del_node(prev, node);
2683 return (1);
2684 }
2685 break;
2686 case jit_code_name: case jit_code_note:
2687 case jit_code_align:
2688 break;
2689 default:
2690 return (0);
2691 }
2692 }
2693 return (0);
2694 }
2695
2696 static jit_code_t
reverse_jump_code(jit_code_t code)2697 reverse_jump_code(jit_code_t code)
2698 {
2699 switch (code) {
2700 case jit_code_bltr: return (jit_code_bger);
2701 case jit_code_blti: return (jit_code_bgei);
2702 case jit_code_bltr_u: return (jit_code_bger_u);
2703 case jit_code_blti_u: return (jit_code_bgei_u);
2704 case jit_code_bler: return (jit_code_bgtr);
2705 case jit_code_blei: return (jit_code_bgti);
2706 case jit_code_bler_u: return (jit_code_bgtr_u);
2707 case jit_code_blei_u: return (jit_code_bgti_u);
2708 case jit_code_beqr: return (jit_code_bner);
2709 case jit_code_beqi: return (jit_code_bnei);
2710 case jit_code_bger: return (jit_code_bltr);
2711 case jit_code_bgei: return (jit_code_blti);
2712 case jit_code_bger_u: return (jit_code_bltr_u);
2713 case jit_code_bgei_u: return (jit_code_blti_u);
2714 case jit_code_bgtr: return (jit_code_bler);
2715 case jit_code_bgti: return (jit_code_blei);
2716 case jit_code_bgtr_u: return (jit_code_bler_u);
2717 case jit_code_bgti_u: return (jit_code_blei_u);
2718 case jit_code_bner: return (jit_code_beqr);
2719 case jit_code_bnei: return (jit_code_beqi);
2720 case jit_code_bmsr: return (jit_code_bmcr);
2721 case jit_code_bmsi: return (jit_code_bmci);
2722 case jit_code_bmcr: return (jit_code_bmsr);
2723 case jit_code_bmci: return (jit_code_bmsi);
2724 case jit_code_bltr_f: return (jit_code_bunger_f);
2725 case jit_code_blti_f: return (jit_code_bungei_f);
2726 case jit_code_bler_f: return (jit_code_bungtr_f);
2727 case jit_code_blei_f: return (jit_code_bungti_f);
2728
2729 case jit_code_beqr_f: return (jit_code_bner_f);
2730 case jit_code_beqi_f: return (jit_code_bnei_f);
2731
2732 case jit_code_bger_f: return (jit_code_bunltr_f);
2733 case jit_code_bgei_f: return (jit_code_bunlti_f);
2734 case jit_code_bgtr_f: return (jit_code_bunler_f);
2735 case jit_code_bgti_f: return (jit_code_bunlei_f);
2736
2737 case jit_code_bner_f: return (jit_code_beqr_f);
2738 case jit_code_bnei_f: return (jit_code_beqr_f);
2739
2740 case jit_code_bunltr_f: return (jit_code_bger_f);
2741 case jit_code_bunlti_f: return (jit_code_bgei_f);
2742 case jit_code_bunler_f: return (jit_code_bgtr_f);
2743 case jit_code_bunlei_f: return (jit_code_bgti_f);
2744
2745 case jit_code_buneqr_f: return (jit_code_bltgtr_f);
2746 case jit_code_buneqi_f: return (jit_code_bltgti_f);
2747
2748 case jit_code_bunger_f: return (jit_code_bltr_f);
2749 case jit_code_bungei_f: return (jit_code_blti_f);
2750 case jit_code_bungtr_f: return (jit_code_bler_f);
2751 case jit_code_bungti_f: return (jit_code_blei_f);
2752
2753 case jit_code_bltgtr_f: return (jit_code_buneqr_f);
2754 case jit_code_bltgti_f: return (jit_code_buneqi_f);
2755
2756 case jit_code_bordr_f: return (jit_code_bunordr_f);
2757 case jit_code_bordi_f: return (jit_code_bunordi_f);
2758 case jit_code_bunordr_f:return (jit_code_bordr_f);
2759 case jit_code_bunordi_f:return (jit_code_bordi_f);
2760 case jit_code_bltr_d: return (jit_code_bunger_d);
2761 case jit_code_blti_d: return (jit_code_bungei_d);
2762 case jit_code_bler_d: return (jit_code_bungtr_d);
2763 case jit_code_blei_d: return (jit_code_bungti_d);
2764
2765 case jit_code_beqr_d: return (jit_code_bner_d);
2766 case jit_code_beqi_d: return (jit_code_bnei_d);
2767
2768 case jit_code_bger_d: return (jit_code_bunltr_d);
2769 case jit_code_bgei_d: return (jit_code_bunlti_d);
2770 case jit_code_bgtr_d: return (jit_code_bunler_d);
2771 case jit_code_bgti_d: return (jit_code_bunlei_d);
2772
2773 case jit_code_bner_d: return (jit_code_beqr_d);
2774 case jit_code_bnei_d: return (jit_code_beqi_d);
2775
2776 case jit_code_bunltr_d: return (jit_code_bger_d);
2777 case jit_code_bunlti_d: return (jit_code_bgei_d);
2778 case jit_code_bunler_d: return (jit_code_bgtr_d);
2779 case jit_code_bunlei_d: return (jit_code_bgti_d);
2780
2781 case jit_code_buneqr_d: return (jit_code_bltgtr_d);
2782 case jit_code_buneqi_d: return (jit_code_bltgti_d);
2783
2784 case jit_code_bunger_d: return (jit_code_bltr_d);
2785 case jit_code_bungei_d: return (jit_code_blti_d);
2786 case jit_code_bungtr_d: return (jit_code_bler_d);
2787 case jit_code_bungti_d: return (jit_code_blei_d);
2788
2789 case jit_code_bltgtr_d: return (jit_code_buneqr_d);
2790 case jit_code_bltgti_d: return (jit_code_buneqi_d);
2791
2792 case jit_code_bordr_d: return (jit_code_bunordr_d);
2793 case jit_code_bordi_d: return (jit_code_bunordi_d);
2794 case jit_code_bunordr_d:return (jit_code_bordr_d);
2795 case jit_code_bunordi_d:return (jit_code_bordi_d);
2796 case jit_code_boaddr: return (jit_code_bxaddr);
2797 case jit_code_boaddi: return (jit_code_bxaddi);
2798 case jit_code_boaddr_u: return (jit_code_bxaddr_u);
2799 case jit_code_boaddi_u: return (jit_code_bxaddi_u);
2800 case jit_code_bxaddr: return (jit_code_boaddr);
2801 case jit_code_bxaddi: return (jit_code_boaddi);
2802 case jit_code_bxaddr_u: return (jit_code_boaddr_u);
2803 case jit_code_bxaddi_u: return (jit_code_boaddi_u);
2804 case jit_code_bosubr: return (jit_code_bxsubr);
2805 case jit_code_bosubi: return (jit_code_bxsubi);
2806 case jit_code_bosubr_u: return (jit_code_bxsubr_u);
2807 case jit_code_bosubi_u: return (jit_code_bxsubi_u);
2808 case jit_code_bxsubr: return (jit_code_bosubr);
2809 case jit_code_bxsubi: return (jit_code_bosubi);
2810 case jit_code_bxsubr_u: return (jit_code_bosubr_u);
2811 case jit_code_bxsubi_u: return (jit_code_bosubi_u);
2812 default: abort(); /* invalid jump code */
2813 }
2814 }
2815
2816 /*
2817 * change common pattern:
2818 * <cond_jump L0> <jump L1> <label L0>
2819 * into
2820 * <reverse_cond_jump L1>
2821 */
2822 static jit_bool_t
_reverse_jump(jit_state_t * _jit,jit_node_t * prev,jit_node_t * node)2823 _reverse_jump(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
2824 {
2825 jit_node_t *local_prev;
2826 jit_node_t *local_next;
2827 jit_node_t *local_jump;
2828
2829 if (!(node->flag & jit_flag_node))
2830 return (0);
2831 /* =><cond_jump L0> <jump L1> <label L0> */
2832 local_next = node->next;
2833 if (local_next->code != jit_code_jmpi ||
2834 !(local_next->flag & jit_flag_node))
2835 return (0);
2836 /* <cond_jump L0> =><jump L1> <label L0> */
2837
2838 local_jump = local_next->u.n;
2839 for (local_prev = local_next, local_next = local_next->next;
2840 local_next;
2841 local_prev = local_next, local_next = local_next->next) {
2842 switch (local_next->code) {
2843 case jit_code_label: case jit_code_epilog:
2844 if (node->u.n == local_next) {
2845 if (local_next->link == node)
2846 local_next->link = node->link;
2847 else {
2848 for (local_prev = local_next->link;
2849 local_prev->link != node;
2850 local_prev = local_prev->link)
2851 assert(local_prev != NULL);
2852 local_prev->link = node->link;
2853 }
2854 del_node(node, node->next);
2855 node->code = reverse_jump_code(node->code);
2856 node->u.n = local_jump;
2857 node->link = local_jump->link;
2858 local_jump->link = node;
2859 return (1);
2860 }
2861 break;
2862 case jit_code_note:
2863 break;
2864 default:
2865 return (0);
2866 }
2867 }
2868 return (0);
2869 }
2870
2871 static void
_redundant_store(jit_state_t * _jit,jit_node_t * node,jit_bool_t jump)2872 _redundant_store(jit_state_t *_jit, jit_node_t *node, jit_bool_t jump)
2873 {
2874 jit_node_t *iter;
2875 jit_node_t *prev;
2876 jit_word_t word;
2877 jit_int32_t spec;
2878 jit_int32_t regno;
2879
2880 if (jump) {
2881 prev = node->u.n;
2882 if (prev->code == jit_code_epilog)
2883 return;
2884 assert(prev->code == jit_code_label);
2885 if ((prev->flag & jit_flag_head) || node->link || prev->link != node)
2886 /* multiple sources */
2887 return;
2888 /* if there are sequential labels it will return below */
2889 }
2890 else
2891 prev = node;
2892 word = node->w.w;
2893 regno = jit_regno(node->v.w);
2894 for (iter = prev->next; iter; prev = iter, iter = iter->next) {
2895 switch (iter->code) {
2896 case jit_code_label: case jit_code_prolog:
2897 case jit_code_epilog:
2898 return;
2899 case jit_code_movi:
2900 if (regno == jit_regno(iter->u.w)) {
2901 if (iter->flag || iter->v.w != word)
2902 return;
2903 del_node(prev, iter);
2904 iter = prev;
2905 }
2906 break;
2907 default:
2908 spec = jit_classify(iter->code);
2909 if (spec & jit_cc_a0_jmp)
2910 return;
2911 if ((spec & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
2912 (jit_cc_a0_reg|jit_cc_a0_chg)) {
2913 if (spec & jit_cc_a0_rlh) {
2914 if (regno == jit_regno(iter->u.q.l) ||
2915 regno == jit_regno(iter->u.q.h))
2916 return;
2917 }
2918 else {
2919 if (regno == jit_regno(iter->u.w))
2920 return;
2921 }
2922 }
2923 if ((spec & (jit_cc_a1_reg|jit_cc_a1_chg)) ==
2924 (jit_cc_a1_reg|jit_cc_a1_chg)) {
2925 if (regno == jit_regno(iter->v.w))
2926 return;
2927 }
2928 if ((spec & (jit_cc_a2_reg|jit_cc_a2_chg)) ==
2929 (jit_cc_a2_reg|jit_cc_a2_chg)) {
2930 if (regno == jit_regno(iter->w.w))
2931 return;
2932 }
2933 break;
2934 }
2935 }
2936 }
2937
2938 static jit_bool_t
_simplify_movr(jit_state_t * _jit,jit_node_t * prev,jit_node_t * node,jit_int32_t kind,jit_int32_t size)2939 _simplify_movr(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
2940 jit_int32_t kind, jit_int32_t size)
2941 {
2942 jit_int32_t regno;
2943 jit_int32_t right;
2944 jit_value_t *value;
2945
2946 regno = jit_regno(node->u.w);
2947 right = jit_regno(node->v.w);
2948 value = _jitc->values + regno;
2949 if ((value->kind == jit_kind_register &&
2950 jit_regno(value->base.q.l) == right &&
2951 value->base.q.h == _jitc->gen[right]) ||
2952 (value->kind == kind && _jitc->values[right].kind == kind &&
2953 memcmp(&value->base.w, &_jitc->values[right].base.w, size) == 0)) {
2954 del_node(prev, node);
2955 return (1);
2956 }
2957 if (_jitc->values[right].kind == jit_kind_word)
2958 jit_memcpy(value, _jitc->values + right, sizeof(jit_value_t));
2959 else {
2960 value->kind = jit_kind_register;
2961 value->base.q.l = right;
2962 value->base.q.h = _jitc->gen[right];
2963 }
2964 ++_jitc->gen[regno];
2965
2966 return (0);
2967 }
2968
2969 static jit_bool_t
_simplify_movi(jit_state_t * _jit,jit_node_t * prev,jit_node_t * node,jit_int32_t kind,jit_int32_t size)2970 _simplify_movi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node,
2971 jit_int32_t kind, jit_int32_t size)
2972 {
2973 jit_value_t *value;
2974 jit_int32_t spec;
2975 jit_int32_t regno;
2976 jit_int32_t offset;
2977
2978 regno = jit_regno(node->u.w);
2979 value = _jitc->values + regno;
2980 if (node->flag & jit_flag_node) {
2981 /* set to undefined if value will be patched */
2982 value->kind = 0;
2983 ++_jitc->gen[regno];
2984 return (0);
2985 }
2986 if (value->kind == kind) {
2987 if (memcmp(&node->v.w, &value->base.w, size) == 0) {
2988 del_node(prev, node);
2989 return (1);
2990 }
2991 spec = jit_class(_rvs[regno].spec);
2992 if (kind == jit_kind_word)
2993 spec &= jit_class_gpr;
2994 else
2995 spec &= (jit_class_xpr | jit_class_fpr);
2996 for (offset = 0; offset < _jitc->reglen; offset++) {
2997 if (_jitc->values[offset].kind == kind &&
2998 memcmp(&node->v.w, &_jitc->values[offset].base.w, size) == 0 &&
2999 (jit_class(_rvs[offset].spec) & spec) == spec) {
3000 if (kind == jit_kind_word)
3001 node->code = jit_code_movr;
3002 else if (kind == jit_kind_float32)
3003 node->code = jit_code_movr_f;
3004 else
3005 node->code = jit_code_movr_d;
3006 node->v.w = offset;
3007 jit_memcpy(value, _jitc->values + offset, sizeof(jit_value_t));
3008 ++_jitc->gen[regno];
3009 return (0);
3010 }
3011 }
3012 }
3013 value->kind = kind;
3014 jit_memcpy(&value->base.w, &node->v.w, size);
3015 ++_jitc->gen[regno];
3016
3017 return (0);
3018 }
3019
3020 /* simple/safe redundandy test not checking if another register
3021 * holds the same value
3022 */
3023 static jit_bool_t
_simplify_ldxi(jit_state_t * _jit,jit_node_t * prev,jit_node_t * node)3024 _simplify_ldxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3025 {
3026 jit_value_t *value;
3027 jit_int32_t regno;
3028 jit_int32_t right;
3029
3030 regno = jit_regno(node->u.w);
3031 right = jit_regno(node->v.w);
3032 value = _jitc->values + regno;
3033 if (regno != right &&
3034 value->kind == jit_kind_code && value->code == node->code &&
3035 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3036 node->w.w == value->disp.w) {
3037 del_node(prev, node);
3038 return (1);
3039 }
3040 value->kind = jit_kind_code;
3041 value->code = node->code;
3042 value->base.q.l = right;
3043 value->base.q.h = _jitc->gen[right];
3044 value->disp.w = node->w.w;
3045 ++_jitc->gen[regno];
3046
3047 return (0);
3048 }
3049
3050 static jit_bool_t
_simplify_stxi(jit_state_t * _jit,jit_node_t * prev,jit_node_t * node)3051 _simplify_stxi(jit_state_t *_jit, jit_node_t *prev, jit_node_t *node)
3052 {
3053 jit_value_t *value;
3054 jit_int32_t regno;
3055 jit_int32_t right;
3056 jit_int32_t offset;
3057
3058 regno = jit_regno(node->w.w);
3059 right = jit_regno(node->v.w);
3060 value = _jitc->values + regno;
3061
3062 /* check for redundant store after load */
3063 if (regno != right &&
3064 value->kind == jit_kind_code && value->code == node->code &&
3065 value->base.q.l == right && value->base.q.h == _jitc->gen[right] &&
3066 node->u.w == value->disp.w) {
3067 del_node(prev, node);
3068 return (1);
3069 }
3070
3071 /* assume anything can alias, and invalidate tracked values */
3072 for (offset = 0; offset < _jitc->reglen; offset++) {
3073 if (_jitc->values[offset].kind == jit_kind_code) {
3074 _jitc->values[offset].kind = 0;
3075 ++_jitc->gen[offset];
3076 }
3077 }
3078
3079 /* no multiple information, so, if set to a constant,
3080 * prefer to keep that information */
3081 if (value->kind == 0) {
3082 value->kind = jit_kind_code;
3083 switch (node->code) {
3084 /* no information about signed/unsigned either */
3085 case jit_code_stxi_c: value->code = jit_code_ldxi_c; break;
3086 case jit_code_stxi_s: value->code = jit_code_ldxi_s; break;
3087 case jit_code_stxi_i: value->code = jit_code_ldxi_i; break;
3088 case jit_code_stxi_l: value->code = jit_code_ldxi_l; break;
3089 case jit_code_stxi_f: value->code = jit_code_ldxi_f; break;
3090 case jit_code_stxi_d: value->code = jit_code_ldxi_d; break;
3091 default: abort();
3092 }
3093 value->kind = jit_kind_code;
3094 value->base.q.l = right;
3095 value->base.q.h = _jitc->gen[right];
3096 value->disp.w = node->u.w;
3097 }
3098
3099 return (0);
3100 }
3101
3102 /* usually there should be only one store in the
3103 * jit_get_reg/jit_unget_reg, but properly handle
3104 * multiple ones by moving the save node */
3105 static void
_simplify_spill(jit_state_t * _jit,jit_node_t * node,jit_int32_t regno)3106 _simplify_spill(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3107 {
3108 jit_node_t *save;
3109 jit_node_t *temp;
3110
3111 if ((temp = _jitc->spill[regno]) && (save = temp->next) != node) {
3112 temp->next = save->next;
3113 save->next = node->next;
3114 node->next = save;
3115 _jitc->spill[regno] = node;
3116 }
3117 }
3118
3119 /* checks for simple cases where a register is set more than
3120 * once to the same value, and is a common pattern of calls
3121 * to jit_pushargi and jit_pushargr
3122 */
3123 static void
_simplify(jit_state_t * _jit)3124 _simplify(jit_state_t *_jit)
3125 {
3126 jit_node_t *prev;
3127 jit_node_t *node;
3128 jit_node_t *next;
3129 jit_int32_t info;
3130 jit_int32_t regno;
3131
3132 for (prev = NULL, node = _jitc->head; node; prev = node, node = next) {
3133 next = node->next;
3134 switch (node->code) {
3135 case jit_code_label: case jit_code_prolog:
3136 case jit_code_callr: case jit_code_calli:
3137 reset:
3138 memset(_jitc->gen, 0, sizeof(jit_int32_t) * _jitc->reglen);
3139 memset(_jitc->values, 0, sizeof(jit_value_t) * _jitc->reglen);
3140 break;
3141 case jit_code_save:
3142 _jitc->spill[jit_regno(node->u.w)] = prev;
3143 break;
3144 case jit_code_load:
3145 regno = jit_regno(node->u.w);
3146 if (register_change_p(node->link->next, node, regno) !=
3147 jit_reg_change) {
3148 /* spill not required due to optimizing common
3149 * redundancy case of calling jit_get_reg/jit_unget_reg
3150 * and then setting the register to the value it is
3151 * already holding */
3152 patch_register(node->link->next, node,
3153 jit_regno_patch|regno, regno);
3154 del_node(_jitc->spill[regno], node->link);
3155 del_node(prev, node);
3156 node = prev;
3157 }
3158 _jitc->spill[regno] = NULL;
3159 break;
3160 case jit_code_movr:
3161 regno = jit_regno(node->u.w);
3162 if (simplify_movr(prev, node,
3163 jit_kind_word, sizeof(jit_word_t)))
3164 simplify_spill(node = prev, regno);
3165 break;
3166 case jit_code_movi:
3167 regno = jit_regno(node->u.w);
3168 if (simplify_movi(prev, node,
3169 jit_kind_word, sizeof(jit_word_t)))
3170 simplify_spill(node = prev, regno);
3171 break;
3172 case jit_code_movr_f:
3173 regno = jit_regno(node->u.w);
3174 if (simplify_movr(prev, node,
3175 jit_kind_float32, sizeof(jit_float32_t)))
3176 simplify_spill(node = prev, regno);
3177 break;
3178 case jit_code_movi_f:
3179 regno = jit_regno(node->u.w);
3180 if (simplify_movi(prev, node,
3181 jit_kind_float32, sizeof(jit_float32_t)))
3182 simplify_spill(node = prev, regno);
3183 break;
3184 case jit_code_movr_d:
3185 regno = jit_regno(node->u.w);
3186 if (simplify_movr(prev, node,
3187 jit_kind_float64, sizeof(jit_float64_t)))
3188 simplify_spill(node = prev, regno);
3189 break;
3190 case jit_code_movi_d:
3191 regno = jit_regno(node->u.w);
3192 if (simplify_movi(prev, node,
3193 jit_kind_float64, sizeof(jit_float64_t)))
3194 simplify_spill(node = prev, regno);
3195 break;
3196 case jit_code_ldxi_c: case jit_code_ldxi_uc:
3197 case jit_code_ldxi_s: case jit_code_ldxi_us:
3198 case jit_code_ldxi_i: case jit_code_ldxi_ui:
3199 case jit_code_ldxi_l:
3200 case jit_code_ldxi_f: case jit_code_ldxi_d:
3201 regno = jit_regno(node->u.w);
3202 if (simplify_ldxi(prev, node))
3203 simplify_spill(node = prev, regno);
3204 break;
3205 case jit_code_stxi_c: case jit_code_stxi_s:
3206 case jit_code_stxi_i: case jit_code_stxi_l:
3207 case jit_code_stxi_f: case jit_code_stxi_d:
3208 regno = jit_regno(node->u.w);
3209 if (simplify_stxi(prev, node))
3210 simplify_spill(node = prev, regno);
3211 break;
3212 default:
3213 info = jit_classify(node->code);
3214 if (info & jit_cc_a0_jmp)
3215 /* labels are not implicitly added when not taking
3216 * a conditional branch */
3217 goto reset;
3218 if (info & jit_cc_a0_chg) {
3219 if (info & jit_cc_a0_rlh) {
3220 regno = jit_regno(node->u.q.l);
3221 _jitc->values[regno].kind = 0;
3222 ++_jitc->gen[regno];
3223 regno = jit_regno(node->u.q.h);
3224 _jitc->values[regno].kind = 0;
3225 ++_jitc->gen[regno];
3226 }
3227 else {
3228 regno = jit_regno(node->u.w);
3229 _jitc->values[regno].kind = 0;
3230 ++_jitc->gen[regno];
3231 }
3232 }
3233 if (info & jit_cc_a1_chg) {
3234 regno = jit_regno(node->v.w);
3235 _jitc->values[regno].kind = 0;
3236 ++_jitc->gen[regno];
3237 }
3238 if (info & jit_cc_a2_chg) {
3239 regno = jit_regno(node->w.w);
3240 _jitc->values[regno].kind = 0;
3241 ++_jitc->gen[regno];
3242 }
3243 break;
3244 }
3245 }
3246 }
3247
3248 static jit_int32_t
_register_change_p(jit_state_t * _jit,jit_node_t * node,jit_node_t * link,jit_int32_t regno)3249 _register_change_p(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3250 jit_int32_t regno)
3251 {
3252 jit_int32_t value;
3253
3254 for (; node != link; node = node->next) {
3255 switch (node->code) {
3256 case jit_code_label: case jit_code_prolog:
3257 /* lack of extra information so cannot say it is undefined */
3258 return (jit_reg_change);
3259 case jit_code_callr: case jit_code_calli:
3260 if (!(jit_class(_rvs[regno].spec) & jit_class_sav))
3261 return (jit_reg_undef);
3262 break;
3263 default:
3264 value = jit_classify(node->code);
3265 /* lack of extra information */
3266 if (value & jit_cc_a0_jmp)
3267 return (jit_reg_change);
3268 else if ((value & (jit_cc_a0_reg|jit_cc_a0_chg)) ==
3269 (jit_cc_a0_reg|jit_cc_a0_chg) &&
3270 (((value & jit_cc_a0_rlh) &&
3271 (node->u.q.l == regno || node->u.q.h == regno)) ||
3272 (!(value & jit_cc_a0_rlh) &&
3273 node->u.w == regno)))
3274 return (jit_reg_change);
3275 else if ((value & jit_cc_a1_reg) && node->v.w == regno &&
3276 (value & jit_cc_a1_chg))
3277 return (jit_reg_change);
3278 else if ((value & jit_cc_a2_reg) && node->w.w == regno &&
3279 (value & jit_cc_a2_chg))
3280 return (jit_reg_change);
3281 }
3282 }
3283
3284 return (jit_reg_static);
3285 }
3286
3287 /* most of this could be done at the same time as generating jit, but
3288 * avoid complications on different cpu backends and patch spill/loads
3289 * here, by simulating jit generation */
3290 static jit_bool_t
_spill_reglive_p(jit_state_t * _jit,jit_node_t * node,jit_int32_t regno)3291 _spill_reglive_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
3292 {
3293 if (!jit_regset_tstbit(&_jitc->reglive, regno)) {
3294 jit_regset_setbit(&_jitc->regmask, regno);
3295 jit_update(node->next, &_jitc->reglive, &_jitc->regmask);
3296 if (!jit_regset_tstbit(&_jitc->reglive, regno) &&
3297 register_change_p(node->next, node->link, regno) != jit_reg_change)
3298 return (0);
3299 }
3300
3301 return (1);
3302 }
3303
3304 static void
_patch_registers(jit_state_t * _jit)3305 _patch_registers(jit_state_t *_jit)
3306 {
3307 jit_node_t *prev;
3308 jit_node_t *node;
3309 jit_node_t *next;
3310 jit_int32_t info;
3311 jit_int32_t spec;
3312 jit_int32_t regno;
3313 jit_int32_t value;
3314
3315 _jitc->function = NULL;
3316
3317 jit_reglive_setup();
3318 for (prev = NULL, node = _jitc->head; node; node = next) {
3319 next = node->next;
3320
3321 info = jit_classify(node->code);
3322 jit_regarg_set(node, info);
3323
3324 switch (node->code) {
3325 case jit_code_save:
3326 regno = jit_regno(node->u.w);
3327 if (!spill_reglive_p(node, regno)) {
3328 /* register is not live, just remove spill/reload */
3329 jit_regarg_clr(node, info);
3330 node->link->v.w = jit_regload_delete;
3331 del_node(prev, node);
3332 continue;
3333 }
3334 else {
3335 /* try to find a free register of the same class */
3336 spec = jit_class(_rvs[regno].spec) & ~jit_class_arg;
3337 for (value = 0; value < _jitc->reglen; value++) {
3338 if (value != regno &&
3339 ((jit_class(_rvs[value].spec) & spec) &
3340 ~jit_class_arg) == spec &&
3341 !jit_regset_tstbit(&_jitc->regarg, value) &&
3342 !spill_reglive_p(node, value))
3343 break;
3344 }
3345 if (value < _jitc->reglen) {
3346 jit_regarg_clr(node, info);
3347 patch_register(node->next, node->link,
3348 jit_regno_patch|node->u.w,
3349 jit_regno_patch|value);
3350 /* mark as live just in case there are nested
3351 * register patches, so that next patch will
3352 * not want to use the same register */
3353 jit_regset_setbit(&_jitc->reglive, value);
3354 /* register is not live, just remove spill/reload */
3355 node->link->v.w = jit_regload_isdead;
3356 del_node(prev, node);
3357 continue;
3358 }
3359 else {
3360 /* failed to find a free register */
3361 if (spec & jit_class_gpr) {
3362 if (!_jitc->function->regoff[regno])
3363 _jitc->function->regoff[regno] =
3364 jit_allocai(sizeof(jit_word_t));
3365 #if __WORDSIZE == 32
3366 node->code = jit_code_stxi_i;
3367 #else
3368 node->code = jit_code_stxi_l;
3369 #endif
3370 }
3371 else {
3372 node->code = jit_code_stxi_d;
3373 if (!_jitc->function->regoff[regno])
3374 _jitc->function->regoff[regno] =
3375 jit_allocai(sizeof(jit_float64_t));
3376 }
3377 node->u.w = _jitc->function->regoff[regno];
3378 node->v.w = JIT_FP;
3379 node->w.w = regno;
3380 node->link = NULL;
3381 }
3382 }
3383 break;
3384 case jit_code_load:
3385 regno = jit_regno(node->u.w);
3386 if (node->v.w) {
3387 if (node->v.w == jit_regload_isdead)
3388 jit_regset_clrbit(&_jitc->reglive, regno);
3389 del_node(prev, node);
3390 continue;
3391 }
3392 spec = jit_class(_rvs[regno].spec);
3393 if (spec & jit_class_gpr) {
3394 #if __WORDSIZE == 32
3395 node->code = jit_code_ldxi_i;
3396 #else
3397 node->code = jit_code_ldxi_l;
3398 #endif
3399 }
3400 else
3401 node->code = jit_code_ldxi_d;
3402 node->v.w = regno;
3403 node->v.w = JIT_FP;
3404 node->w.w = _jitc->function->regoff[regno];
3405 node->link = NULL;
3406 break;
3407 case jit_code_prolog:
3408 _jitc->function = _jitc->functions.ptr + node->w.w;
3409 break;
3410 case jit_code_epilog:
3411 _jitc->function = NULL;
3412 break;
3413 default:
3414 break;
3415 }
3416
3417 jit_regarg_clr(node, info);
3418 /* update register live state */
3419 jit_reglive(node);
3420 prev = node;
3421 }
3422 }
3423
3424 static void
_patch_register(jit_state_t * _jit,jit_node_t * node,jit_node_t * link,jit_int32_t regno,jit_int32_t patch)3425 _patch_register(jit_state_t *_jit, jit_node_t *node, jit_node_t *link,
3426 jit_int32_t regno, jit_int32_t patch)
3427 {
3428 jit_int32_t value;
3429
3430 for (; node != link; node = node->next) {
3431 value = jit_classify(node->code);
3432 if (value & jit_cc_a0_reg) {
3433 if (value & jit_cc_a0_rlh) {
3434 if (node->u.q.l == regno)
3435 node->u.q.l = patch;
3436 if (node->u.q.h == regno)
3437 node->u.q.h = patch;
3438 }
3439 else {
3440 if (node->u.w == regno)
3441 node->u.w = patch;
3442 }
3443 }
3444 if ((value & jit_cc_a1_reg) && node->v.w == regno)
3445 node->v.w = patch;
3446 if ((value & jit_cc_a2_reg) && node->w.w == regno)
3447 node->w.w = patch;
3448 }
3449 }
3450
3451 #if defined(__i386__) || defined(__x86_64__)
3452 # include "jit_x86.c"
3453 #elif defined(__mips__)
3454 # include "jit_mips.c"
3455 #elif defined(__arm__)
3456 # include "jit_arm.c"
3457 #elif defined(__powerpc__)
3458 # include "jit_ppc.c"
3459 #elif defined(__sparc__)
3460 # include "jit_sparc.c"
3461 #elif defined(__ia64__)
3462 # include "jit_ia64.c"
3463 #elif defined(__hppa__)
3464 # include "jit_hppa.c"
3465 #elif defined(__aarch64__)
3466 # include "jit_aarch64.c"
3467 #elif defined(__s390__) || defined(__s390x__)
3468 # include "jit_s390.c"
3469 #elif defined(__alpha__)
3470 # include "jit_alpha.c"
3471 #elif defined(__riscv)
3472 # include "jit_riscv.c"
3473 #endif
3474