1 /*
2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20 #if defined(__linux__)
21 # include <stdio.h>
22 #endif
23
24 #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 4)
25 #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < 16)
26 #define jit_arg_d_reg_p(i) ((i) >= 0 && (i) < 15)
27
28 #define arm_patch_node 0x80000000
29 #define arm_patch_word 0x40000000
30 #define arm_patch_jump 0x20000000
31 #define arm_patch_load 0x00000000
32
33 #define jit_fpr_p(rn) ((rn) > 15)
34
35 #define arg_base() \
36 (stack_framesize - 16 + (jit_cpu.abi ? 64 : 0))
37 #define arg_offset(n) \
38 ((n) < 4 ? arg_base() + ((n) << 2) : (n))
39
40 /* Assume functions called never match jit instruction set, that is
41 * libc, gmp, mpfr, etc functions are in thumb mode and jit is in
42 * arm mode, what may cause a crash upon return of that function
43 * if generating jit for a relative jump.
44 */
45 #define jit_exchange_p() 1
46
47 /* FIXME is it really required to not touch _R10? */
48
49 /*
50 * Types
51 */
52 typedef union _jit_thumb_t {
53 jit_int32_t i;
54 jit_int16_t s[2];
55 } jit_thumb_t;
56
57 typedef jit_pointer_t jit_va_list;
58
59 /*
60 * Prototypes
61 */
62 #define jit_make_arg(node) _jit_make_arg(_jit,node)
63 static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*);
64 #define jit_make_arg_f(node) _jit_make_arg_f(_jit,node)
65 static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*);
66 #define jit_make_arg_d(node) _jit_make_arg_d(_jit,node)
67 static jit_node_t *_jit_make_arg_d(jit_state_t*,jit_node_t*);
68 #define jit_get_reg_pair() _jit_get_reg_pair(_jit)
69 static jit_int32_t _jit_get_reg_pair(jit_state_t*);
70 #define jit_unget_reg_pair(rn) _jit_unget_reg_pair(_jit,rn)
71 static void _jit_unget_reg_pair(jit_state_t*,jit_int32_t);
72 # define must_align_p(node) _must_align_p(_jit, node)
73 static jit_bool_t _must_align_p(jit_state_t*,jit_node_t*);
74 #define load_const(uniq,r0,i0) _load_const(_jit,uniq,r0,i0)
75 static void _load_const(jit_state_t*,jit_bool_t,jit_int32_t,jit_word_t);
76 #define flush_consts() _flush_consts(_jit)
77 static void _flush_consts(jit_state_t*);
78 #define invalidate_consts() _invalidate_consts(_jit)
79 static void _invalidate_consts(jit_state_t*);
80 #define patch(instr, node) _patch(_jit, instr, node)
81 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
82
83 #if defined(__GNUC__)
84 /* libgcc */
85 extern void __clear_cache(void *, void *);
86 #endif
87
88 #define PROTO 1
89 # include "jit_rewind.c"
90 # include "jit_arm-cpu.c"
91 # include "jit_arm-swf.c"
92 # include "jit_arm-vfp.c"
93 #undef PROTO
94
95 /*
96 * Initialization
97 */
98 jit_cpu_t jit_cpu;
99 jit_register_t _rvs[] = {
100 { rc(gpr) | 0x0c, "ip" },
101 { rc(sav) | rc(gpr) | 0x04, "r4" },
102 { rc(sav) | rc(gpr) | 0x05, "r5" },
103 { rc(sav) | rc(gpr) | 0x06, "r6" },
104 { rc(sav) | rc(gpr) | 0x07, "r7" },
105 { rc(sav) | rc(gpr) | 0x08, "r8" },
106 { rc(sav) | rc(gpr) | 0x09, "r9" },
107 { rc(sav) | 0x0a, "sl" },
108 { rc(sav) | 0x0b, "fp" },
109 { rc(sav) | 0x0d, "sp" },
110 { rc(sav) | 0x0e, "lr" },
111 { 0x0f, "pc" },
112 { rc(arg) | rc(gpr) | 0x03, "r3" },
113 { rc(arg) | rc(gpr) | 0x02, "r2" },
114 { rc(arg) | rc(gpr) | 0x01, "r1" },
115 { rc(arg) | rc(gpr) | 0x00, "r0" },
116 { rc(fpr) | 0x20, "d8" },
117 { 0x21, "s17" },
118 { rc(fpr) | 0x22, "d9" },
119 { 0x23, "s19" },
120 { rc(fpr) | 0x24, "d10" },
121 { 0x25, "s21" },
122 { rc(fpr) | 0x26, "d11" },
123 { 0x27, "s23" },
124 { rc(fpr) | 0x28, "d12" },
125 { 0x29, "s25" },
126 { rc(fpr) | 0x2a, "d13" },
127 { 0x2b, "s27" },
128 { rc(fpr) | 0x2c, "d14" },
129 { 0x2d, "s29" },
130 { rc(fpr) | 0x2e, "d15" },
131 { 0x2f, "s31" },
132 { rc(arg) | 0x1f, "s15" },
133 { rc(arg)|rc(sft)|rc(fpr)|0x1e, "d7" },
134 { rc(arg) | 0x1d, "s13" },
135 { rc(arg)|rc(sft)|rc(fpr)|0x1c, "d6" },
136 { rc(arg) | 0x1b, "s11" },
137 { rc(arg)|rc(sft)|rc(fpr)|0x1a, "d5" },
138 { rc(arg) | 0x19, "s9" },
139 { rc(arg)|rc(sft)|rc(fpr)|0x18, "d4" },
140 { rc(arg) | 0x17, "s7" },
141 { rc(arg)|rc(sft)|rc(fpr)|0x16, "d3" },
142 { rc(arg) | 0x15, "s5" },
143 { rc(arg)|rc(sft)|rc(fpr)|0x14, "d2" },
144 { rc(arg) | 0x13, "s3" },
145 { rc(arg)|rc(sft)|rc(fpr)|0x12, "d1" },
146 { rc(arg) | 0x11, "s1" },
147 { rc(arg)|rc(sft)|rc(fpr)|0x10, "d0" },
148 { _NOREG, "<none>" },
149 };
150
151 /*
152 * Implementation
153 */
154 void
jit_get_cpu(void)155 jit_get_cpu(void)
156 {
157 #if defined(__linux__)
158 FILE *fp;
159 char *ptr;
160 char buf[128];
161
162 if ((fp = fopen("/proc/cpuinfo", "r")) != NULL) {
163 while (fgets(buf, sizeof(buf), fp)) {
164 if (strncmp(buf, "CPU architecture:", 17) == 0) {
165 jit_cpu.version = strtol(buf + 17, &ptr, 10);
166 while (*ptr) {
167 if (*ptr == 'T' || *ptr == 't') {
168 ++ptr;
169 jit_cpu.thumb = 1;
170 }
171 else if (*ptr == 'E' || *ptr == 'e') {
172 jit_cpu.extend = 1;
173 ++ptr;
174 }
175 else
176 ++ptr;
177 }
178 }
179 else if (strncmp(buf, "Features\t:", 10) == 0) {
180 if ((ptr = strstr(buf + 10, "vfpv")))
181 jit_cpu.vfp = strtol(ptr + 4, NULL, 0);
182 if ((ptr = strstr(buf + 10, "neon")))
183 jit_cpu.neon = 1;
184 if ((ptr = strstr(buf + 10, "thumb")))
185 jit_cpu.thumb = 1;
186 }
187 }
188 fclose(fp);
189 }
190 #endif
191 #if defined(__ARM_PCS_VFP)
192 if (!jit_cpu.vfp)
193 jit_cpu.vfp = 3;
194 if (!jit_cpu.version)
195 jit_cpu.version = 7;
196 jit_cpu.abi = 1;
197 #endif
198 #if defined(__thumb2__)
199 jit_cpu.thumb = 1;
200 #endif
201 /* armv6t2 todo (software float and thumb2) */
202 if (!jit_cpu.vfp && jit_cpu.thumb)
203 jit_cpu.thumb = 0;
204 }
205
206 void
_jit_init(jit_state_t * _jit)207 _jit_init(jit_state_t *_jit)
208 {
209 jit_int32_t regno;
210 static jit_bool_t first = 1;
211
212 _jitc->reglen = jit_size(_rvs) - 1;
213 if (first) {
214 /* jit_get_cpu() should have been already called, and only once */
215 if (!jit_cpu.vfp) {
216 /* cause register to never be allocated, because simple
217 * software float only allocates stack space for 8 slots */
218 for (regno = _D8; regno < _D7; regno++)
219 _rvs[regno].spec = 0;
220 }
221 if (!jit_cpu.abi) {
222 for (regno = _S15; regno <= _D0; regno++)
223 _rvs[regno].spec &= ~rc(arg);
224 }
225 first = 0;
226 }
227 }
228
229 void
_jit_prolog(jit_state_t * _jit)230 _jit_prolog(jit_state_t *_jit)
231 {
232 jit_int32_t offset;
233
234 if (_jitc->function)
235 jit_epilog();
236 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
237 jit_regset_set_ui(&_jitc->regsav, 0);
238 offset = _jitc->functions.offset;
239 if (offset >= _jitc->functions.length) {
240 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
241 _jitc->functions.length * sizeof(jit_function_t),
242 (_jitc->functions.length + 16) * sizeof(jit_function_t));
243 _jitc->functions.length += 16;
244 }
245 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
246 _jitc->function->self.size = stack_framesize;
247 if (jit_cpu.abi)
248 _jitc->function->self.size += 64;
249 _jitc->function->self.argi = _jitc->function->self.argf =
250 _jitc->function->self.alen = 0;
251 if (jit_swf_p())
252 /* 8 soft float registers */
253 _jitc->function->self.aoff = -64;
254 else
255 _jitc->function->self.aoff = 0;
256 _jitc->function->self.call = jit_call_default;
257 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
258 _jitc->reglen * sizeof(jit_int32_t));
259
260 /* _no_link here does not mean the jit_link() call can be removed
261 * by rewriting as:
262 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
263 */
264 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
265 jit_link(_jitc->function->prolog);
266 _jitc->function->prolog->w.w = offset;
267 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
268 /* u: label value
269 * v: offset in blocks vector
270 * w: offset in functions vector
271 */
272 _jitc->function->epilog->w.w = offset;
273
274 jit_regset_new(&_jitc->function->regset);
275 }
276
277 jit_int32_t
_jit_allocai(jit_state_t * _jit,jit_int32_t length)278 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
279 {
280 assert(_jitc->function);
281 switch (length) {
282 case 0: case 1: break;
283 case 2: _jitc->function->self.aoff &= -2; break;
284 case 3: case 4: _jitc->function->self.aoff &= -4; break;
285 default: _jitc->function->self.aoff &= -8; break;
286 }
287 _jitc->function->self.aoff -= length;
288 if (!_jitc->realize) {
289 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
290 jit_dec_synth();
291 }
292 return (_jitc->function->self.aoff);
293 }
294
295 void
_jit_allocar(jit_state_t * _jit,jit_int32_t u,jit_int32_t v)296 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
297 {
298 jit_int32_t reg;
299 assert(_jitc->function);
300 jit_inc_synth_ww(allocar, u, v);
301 if (!_jitc->function->allocar) {
302 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
303 _jitc->function->allocar = 1;
304 }
305 reg = jit_get_reg(jit_class_gpr);
306 jit_negr(reg, v);
307 jit_andi(reg, reg, -8);
308 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
309 jit_addr(u, u, reg);
310 jit_addr(JIT_SP, JIT_SP, reg);
311 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
312 jit_unget_reg(reg);
313 jit_dec_synth();
314 }
315
316 void
_jit_ret(jit_state_t * _jit)317 _jit_ret(jit_state_t *_jit)
318 {
319 jit_node_t *instr;
320 assert(_jitc->function);
321 jit_inc_synth(ret);
322 /* jump to epilog */
323 instr = jit_jmpi();
324 jit_patch_at(instr, _jitc->function->epilog);
325 jit_dec_synth();
326 }
327
328 void
_jit_retr(jit_state_t * _jit,jit_int32_t u)329 _jit_retr(jit_state_t *_jit, jit_int32_t u)
330 {
331 jit_inc_synth_w(retr, u);
332 if (JIT_RET != u)
333 jit_movr(JIT_RET, u);
334 jit_live(JIT_RET);
335 jit_ret();
336 jit_dec_synth();
337 }
338
339 void
_jit_reti(jit_state_t * _jit,jit_word_t u)340 _jit_reti(jit_state_t *_jit, jit_word_t u)
341 {
342 jit_inc_synth_w(reti, u);
343 jit_movi(JIT_RET, u);
344 jit_ret();
345 jit_dec_synth();
346 }
347
348 void
_jit_retr_f(jit_state_t * _jit,jit_int32_t u)349 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
350 {
351 jit_inc_synth_w(retr_f, u);
352 if (jit_cpu.abi) {
353 if (u != JIT_FRET)
354 jit_movr_f(JIT_FRET, u);
355 else
356 jit_live(JIT_FRET);
357 }
358 else {
359 if (u != JIT_RET)
360 jit_movr_f_w(JIT_RET, u);
361 else
362 jit_live(JIT_RET);
363 }
364 jit_ret();
365 jit_dec_synth();
366 }
367
368 void
_jit_reti_f(jit_state_t * _jit,jit_float32_t u)369 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
370 {
371 jit_inc_synth_f(reti_f, u);
372 if (jit_cpu.abi)
373 jit_movi_f(JIT_FRET, u);
374 else
375 jit_movi_f_w(JIT_RET, u);
376 jit_ret();
377 jit_dec_synth();
378 }
379
380 void
_jit_retr_d(jit_state_t * _jit,jit_int32_t u)381 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
382 {
383 jit_inc_synth_w(retr_d, u);
384 if (jit_cpu.abi) {
385 if (u != JIT_FRET)
386 jit_movr_d(JIT_FRET, u);
387 else
388 jit_live(JIT_FRET);
389 }
390 else {
391 if (u != JIT_RET)
392 jit_movr_d_ww(JIT_RET, _R1, u);
393 else
394 jit_live(JIT_RET);
395 }
396 jit_ret();
397 jit_dec_synth();
398 }
399
400 void
_jit_reti_d(jit_state_t * _jit,jit_float64_t u)401 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
402 {
403 jit_inc_synth_d(reti_d, u);
404 if (jit_cpu.abi)
405 jit_movi_d(JIT_FRET, u);
406 else
407 jit_movi_d_ww(JIT_RET, _R1, u);
408 jit_ret();
409 jit_dec_synth();
410 }
411
412 void
_jit_epilog(jit_state_t * _jit)413 _jit_epilog(jit_state_t *_jit)
414 {
415 assert(_jitc->function);
416 assert(_jitc->function->epilog->next == NULL);
417 jit_link(_jitc->function->epilog);
418 _jitc->function = NULL;
419 }
420
421 jit_bool_t
_jit_arg_register_p(jit_state_t * _jit,jit_node_t * u)422 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
423 {
424 if (u->code != jit_code_arg) {
425 if (u->code == jit_code_arg_f) {
426 if (jit_cpu.abi)
427 return (jit_arg_f_reg_p(u->u.w));
428 }
429 else {
430 assert(u->code == jit_code_arg_d);
431 if (jit_cpu.abi)
432 return (jit_arg_d_reg_p(u->u.w));
433 }
434 }
435 return (jit_arg_reg_p(u->u.w));
436 }
437
438 static jit_node_t *
_jit_make_arg(jit_state_t * _jit,jit_node_t * node)439 _jit_make_arg(jit_state_t *_jit, jit_node_t *node)
440 {
441 jit_int32_t offset;
442 if (jit_arg_reg_p(_jitc->function->self.argi))
443 offset = _jitc->function->self.argi++;
444 else {
445 offset = _jitc->function->self.size;
446 _jitc->function->self.size += sizeof(jit_word_t);
447 }
448 if (node == (jit_node_t *)0)
449 node = jit_new_node(jit_code_arg);
450 else
451 link_node(node);
452 node->u.w = offset;
453 node->v.w = ++_jitc->function->self.argn;
454 jit_link_prolog();
455 return (node);
456 }
457
458 jit_node_t *
_jit_make_arg_f(jit_state_t * _jit,jit_node_t * node)459 _jit_make_arg_f(jit_state_t *_jit, jit_node_t *node)
460 {
461 jit_int32_t offset;
462 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
463 if (jit_arg_f_reg_p(_jitc->function->self.argf)) {
464 offset = _jitc->function->self.argf++;
465 goto done;
466 }
467 }
468 else {
469 if (jit_arg_reg_p(_jitc->function->self.argi)) {
470 offset = _jitc->function->self.argi++;
471 goto done;
472 }
473 }
474 offset = _jitc->function->self.size;
475 _jitc->function->self.size += sizeof(jit_float32_t);
476 done:
477 if (node == (jit_node_t *)0)
478 node = jit_new_node(jit_code_arg_f);
479 else
480 link_node(node);
481 node->u.w = offset;
482 node->v.w = ++_jitc->function->self.argn;
483 jit_link_prolog();
484 return (node);
485 }
486
487 jit_node_t *
_jit_make_arg_d(jit_state_t * _jit,jit_node_t * node)488 _jit_make_arg_d(jit_state_t *_jit, jit_node_t *node)
489 {
490 jit_int32_t offset;
491 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
492 if (jit_arg_d_reg_p(_jitc->function->self.argf)) {
493 if (_jitc->function->self.argf & 1)
494 ++_jitc->function->self.argf;
495 offset = _jitc->function->self.argf;
496 _jitc->function->self.argf += 2;
497 goto done;
498 }
499 }
500 else {
501 if (_jitc->function->self.argi & 1)
502 ++_jitc->function->self.argi;
503 if (jit_arg_reg_p(_jitc->function->self.argi)) {
504 offset = _jitc->function->self.argi;
505 _jitc->function->self.argi += 2;
506 goto done;
507 }
508 }
509 if (_jitc->function->self.size & 7)
510 _jitc->function->self.size += 4;
511 offset = _jitc->function->self.size;
512 _jitc->function->self.size += sizeof(jit_float64_t);
513 done:
514 if (node == (jit_node_t *)0)
515 node = jit_new_node(jit_code_arg_d);
516 else
517 link_node(node);
518 node->u.w = offset;
519 node->v.w = ++_jitc->function->self.argn;
520 jit_link_prolog();
521 return (node);
522 }
523
524 void
_jit_ellipsis(jit_state_t * _jit)525 _jit_ellipsis(jit_state_t *_jit)
526 {
527 if (_jitc->prepare) {
528 assert(!(_jitc->function->call.call & jit_call_varargs));
529 _jitc->function->call.call |= jit_call_varargs;
530 if (jit_cpu.abi && _jitc->function->call.argf)
531 rewind_prepare();
532 }
533 else {
534 assert(!(_jitc->function->self.call & jit_call_varargs));
535 _jitc->function->self.call |= jit_call_varargs;
536 if (jit_cpu.abi && _jitc->function->self.argf)
537 rewind_prolog();
538 /* First 4 stack addresses are always spilled r0-r3 */
539 if (jit_arg_reg_p(_jitc->function->self.argi))
540 _jitc->function->vagp = _jitc->function->self.argi * 4;
541 else
542 _jitc->function->vagp = 16;
543 }
544 jit_inc_synth(ellipsis);
545 if (_jitc->prepare)
546 jit_link_prepare();
547 else
548 jit_link_prolog();
549 jit_dec_synth();
550 }
551
552 void
_jit_va_push(jit_state_t * _jit,jit_int32_t u)553 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
554 {
555 jit_inc_synth_w(va_push, u);
556 jit_pushargr(u);
557 jit_dec_synth();
558 }
559
560 jit_node_t *
_jit_arg(jit_state_t * _jit)561 _jit_arg(jit_state_t *_jit)
562 {
563 assert(_jitc->function);
564 return (jit_make_arg((jit_node_t*)0));
565 }
566
567 jit_node_t *
_jit_arg_f(jit_state_t * _jit)568 _jit_arg_f(jit_state_t *_jit)
569 {
570 assert(_jitc->function);
571 return (jit_make_arg_f((jit_node_t*)0));
572 }
573
574 jit_node_t *
_jit_arg_d(jit_state_t * _jit)575 _jit_arg_d(jit_state_t *_jit)
576 {
577 assert(_jitc->function);
578 return (jit_make_arg_d((jit_node_t*)0));
579 }
580
581 void
_jit_getarg_c(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)582 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
583 {
584 assert(v->code == jit_code_arg);
585 jit_inc_synth_wp(getarg_c, u, v);
586 if (jit_swf_p())
587 jit_ldxi_c(u, JIT_FP, arg_offset(v->u.w));
588 else if (jit_arg_reg_p(v->u.w))
589 jit_extr_c(u, JIT_RA0 - v->u.w);
590 else
591 jit_ldxi_c(u, JIT_FP, v->u.w);
592 jit_dec_synth();
593 }
594
595 void
_jit_getarg_uc(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)596 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
597 {
598 assert(v->code == jit_code_arg);
599 jit_inc_synth_wp(getarg_uc, u, v);
600 if (jit_swf_p())
601 jit_ldxi_uc(u, JIT_FP, arg_offset(v->u.w));
602 else if (jit_arg_reg_p(v->u.w))
603 jit_extr_uc(u, JIT_RA0 - v->u.w);
604 else
605 jit_ldxi_uc(u, JIT_FP, v->u.w);
606 jit_dec_synth();
607 }
608
609 void
_jit_getarg_s(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)610 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
611 {
612 assert(v->code == jit_code_arg);
613 jit_inc_synth_wp(getarg_s, u, v);
614 if (jit_swf_p())
615 jit_ldxi_s(u, JIT_FP, arg_offset(v->u.w));
616 else if (jit_arg_reg_p(v->u.w))
617 jit_extr_s(u, JIT_RA0 - v->u.w);
618 else
619 jit_ldxi_s(u, JIT_FP, v->u.w);
620 jit_dec_synth();
621 }
622
623 void
_jit_getarg_us(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)624 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
625 {
626 assert(v->code == jit_code_arg);
627 jit_inc_synth_wp(getarg_us, u, v);
628 if (jit_swf_p())
629 jit_ldxi_us(u, JIT_FP, arg_offset(v->u.w));
630 else if (jit_arg_reg_p(v->u.w))
631 jit_extr_us(u, JIT_RA0 - v->u.w);
632 else
633 jit_ldxi_us(u, JIT_FP, v->u.w);
634 jit_dec_synth();
635 }
636
637 void
_jit_getarg_i(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)638 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
639 {
640 assert(v->code == jit_code_arg);
641 jit_inc_synth_wp(getarg_i, u, v);
642 if (jit_swf_p())
643 jit_ldxi_i(u, JIT_FP, arg_offset(v->u.w));
644 else if (jit_arg_reg_p(v->u.w))
645 jit_movr(u, JIT_RA0 - v->u.w);
646 else
647 jit_ldxi_i(u, JIT_FP, v->u.w);
648 jit_dec_synth();
649 }
650
651 void
_jit_putargr(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)652 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
653 {
654 assert(v->code == jit_code_arg);
655 jit_inc_synth_wp(putargr, u, v);
656 if (jit_swf_p())
657 jit_stxi(arg_offset(v->u.w), JIT_FP, u);
658 else if (jit_arg_reg_p(v->u.w))
659 jit_movr(JIT_RA0 - v->u.w, u);
660 else
661 jit_stxi(v->u.w, JIT_FP, u);
662 jit_dec_synth();
663 }
664
665 void
_jit_putargi(jit_state_t * _jit,jit_word_t u,jit_node_t * v)666 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
667 {
668 jit_int32_t regno;
669 assert(v->code == jit_code_arg);
670 jit_inc_synth_wp(putargi, u, v);
671 if (jit_swf_p()) {
672 regno = jit_get_reg(jit_class_gpr);
673 jit_movi(regno, u);
674 jit_stxi(arg_offset(v->u.w), JIT_FP, regno);
675 jit_unget_reg(regno);
676 }
677 else if (jit_arg_reg_p(v->u.w))
678 jit_movi(JIT_RA0 - v->u.w, u);
679 else {
680 regno = jit_get_reg(jit_class_gpr);
681 jit_movi(regno, u);
682 jit_stxi(v->u.w, JIT_FP, regno);
683 jit_unget_reg(regno);
684 }
685 jit_dec_synth();
686 }
687
688 void
_jit_getarg_f(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)689 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
690 {
691 assert(v->code == jit_code_arg_f);
692 jit_inc_synth_wp(getarg_f, u, v);
693 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
694 if (jit_arg_f_reg_p(v->u.w))
695 jit_movr_f(u, JIT_FA0 - v->u.w);
696 else
697 jit_ldxi_f(u, JIT_FP, v->u.w);
698 }
699 else if (jit_swf_p())
700 jit_ldxi_f(u, JIT_FP, arg_offset(v->u.w));
701 else {
702 if (jit_arg_reg_p(v->u.w))
703 jit_movr_w_f(u, JIT_RA0 - v->u.w);
704 else
705 jit_ldxi_f(u, JIT_FP, v->u.w);
706 }
707 jit_dec_synth();
708 }
709
710 void
_jit_putargr_f(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)711 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
712 {
713 assert(v->code == jit_code_arg_f);
714 jit_inc_synth_wp(putargr_f, u, v);
715 if (jit_cpu.abi) {
716 if (jit_arg_f_reg_p(v->u.w))
717 jit_movr_f(JIT_FA0 - v->u.w, u);
718 else
719 jit_stxi_f(v->u.w, JIT_FP, u);
720 }
721 else if (jit_swf_p())
722 jit_stxi_f(arg_offset(v->u.w), JIT_FP, u);
723 else {
724 if (jit_arg_reg_p(v->u.w))
725 jit_movr_f_w(JIT_RA0 - v->u.w, u);
726 else
727 jit_stxi_f(v->u.w, JIT_FP, u);
728 }
729 jit_dec_synth();
730 }
731
732 void
_jit_putargi_f(jit_state_t * _jit,jit_float32_t u,jit_node_t * v)733 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
734 {
735 jit_int32_t regno;
736 assert(v->code == jit_code_arg_f);
737 jit_inc_synth_fp(putargi_f, u, v);
738 if (jit_cpu.abi) {
739 if (jit_arg_f_reg_p(v->u.w))
740 jit_movi_f(JIT_FA0 - v->u.w, u);
741 else {
742 regno = jit_get_reg(jit_class_fpr);
743 jit_movi_f(regno, u);
744 jit_stxi_f(v->u.w, JIT_FP, regno);
745 jit_unget_reg(regno);
746 }
747 }
748 else if (jit_swf_p()) {
749 regno = jit_get_reg(jit_class_fpr);
750 jit_movi_f(regno, u);
751 jit_stxi_f(arg_offset(v->u.w), JIT_FP, regno);
752 jit_unget_reg(regno);
753 }
754 else {
755 regno = jit_get_reg(jit_class_fpr);
756 jit_movi_f(regno, u);
757 if (jit_arg_reg_p(v->u.w))
758 jit_movr_f_w(JIT_RA0 - v->u.w, regno);
759 else
760 jit_stxi_f(v->u.w, JIT_FP, regno);
761 jit_unget_reg(regno);
762 }
763 jit_dec_synth();
764 }
765
766 void
_jit_getarg_d(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)767 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
768 {
769 assert(v->code == jit_code_arg_d);
770 jit_inc_synth_wp(getarg_d, u, v);
771 if (jit_cpu.abi && !(_jitc->function->self.call & jit_call_varargs)) {
772 if (jit_arg_f_reg_p(v->u.w))
773 jit_movr_d(u, JIT_FA0 - v->u.w);
774 else
775 jit_ldxi_d(u, JIT_FP, v->u.w);
776 }
777 else if (jit_swf_p())
778 jit_ldxi_d(u, JIT_FP, arg_offset(v->u.w));
779 else {
780 if (jit_arg_reg_p(v->u.w))
781 jit_movr_ww_d(u, JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1));
782 else
783 jit_ldxi_d(u, JIT_FP, v->u.w);
784 }
785 jit_dec_synth();
786 }
787
788 void
_jit_putargr_d(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)789 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
790 {
791 assert(v->code == jit_code_arg_d);
792 jit_inc_synth_wp(putargr_d, u, v);
793 if (jit_cpu.abi) {
794 if (jit_arg_f_reg_p(v->u.w))
795 jit_movr_d(JIT_FA0 - v->u.w, u);
796 else
797 jit_stxi_d(v->u.w, JIT_FP, u);
798 }
799 else if (jit_swf_p())
800 jit_stxi_d(arg_offset(v->u.w), JIT_FP, u);
801 else {
802 if (jit_arg_reg_p(v->u.w))
803 jit_movr_d_ww(JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1), u);
804 else
805 jit_stxi_d(v->u.w, JIT_FP, u);
806 }
807 jit_dec_synth();
808 }
809
810 void
_jit_putargi_d(jit_state_t * _jit,jit_float64_t u,jit_node_t * v)811 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
812 {
813 jit_int32_t regno;
814 assert(v->code == jit_code_arg_d);
815 jit_inc_synth_dp(putargi_d, u, v);
816 if (jit_cpu.abi) {
817 if (jit_arg_f_reg_p(v->u.w))
818 jit_movi_d(JIT_FA0 - v->u.w, u);
819 else {
820 regno = jit_get_reg(jit_class_fpr);
821 jit_movi_d(regno, u);
822 jit_stxi_d(v->u.w, JIT_FP, regno);
823 jit_unget_reg(regno);
824 }
825 }
826 else if (jit_swf_p()) {
827 regno = jit_get_reg(jit_class_fpr);
828 jit_movi_d(regno, u);
829 jit_stxi_d(arg_offset(v->u.w), JIT_FP, regno);
830 jit_unget_reg(regno);
831 }
832 else {
833 regno = jit_get_reg(jit_class_fpr);
834 jit_movi_d(regno, u);
835 if (jit_arg_reg_p(v->u.w))
836 jit_movr_d_ww(JIT_RA0 - v->u.w, JIT_RA0 - (v->u.w + 1), regno);
837 else
838 jit_stxi_d(v->u.w, JIT_FP, regno);
839 jit_unget_reg(regno);
840 }
841 jit_dec_synth();
842 }
843
844 void
_jit_pushargr(jit_state_t * _jit,jit_int32_t u)845 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
846 {
847 assert(_jitc->function);
848 jit_inc_synth_w(pushargr, u);
849 jit_link_prepare();
850 if (jit_arg_reg_p(_jitc->function->call.argi)) {
851 jit_movr(JIT_RA0 - _jitc->function->call.argi, u);
852 ++_jitc->function->call.argi;
853 }
854 else {
855 jit_stxi(_jitc->function->call.size, JIT_SP, u);
856 _jitc->function->call.size += sizeof(jit_word_t);
857 }
858 jit_dec_synth();
859 }
860
861 void
_jit_pushargi(jit_state_t * _jit,jit_word_t u)862 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
863 {
864 jit_int32_t regno;
865 assert(_jitc->function);
866 jit_inc_synth_w(pushargi, u);
867 jit_link_prepare();
868 if (jit_arg_reg_p(_jitc->function->call.argi)) {
869 jit_movi(JIT_RA0 - _jitc->function->call.argi, u);
870 ++_jitc->function->call.argi;
871 }
872 else {
873 regno = jit_get_reg(jit_class_gpr);
874 jit_movi(regno, u);
875 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
876 jit_unget_reg(regno);
877 _jitc->function->call.size += sizeof(jit_word_t);
878 }
879 jit_dec_synth();
880 }
881
882 void
_jit_pushargr_f(jit_state_t * _jit,jit_int32_t u)883 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
884 {
885 assert(_jitc->function);
886 jit_inc_synth_w(pushargr_f, u);
887 jit_link_prepare();
888 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
889 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
890 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, u);
891 ++_jitc->function->call.argf;
892 goto done;
893 }
894 }
895 else {
896 if (jit_arg_reg_p(_jitc->function->call.argi)) {
897 jit_movr_f_w(JIT_RA0 - _jitc->function->call.argi, u);
898 ++_jitc->function->call.argi;
899 goto done;
900 }
901 }
902 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
903 _jitc->function->call.size += sizeof(jit_word_t);
904 done:
905 jit_dec_synth();
906 }
907
908 void
_jit_pushargi_f(jit_state_t * _jit,jit_float32_t u)909 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
910 {
911 jit_int32_t regno;
912 assert(_jitc->function);
913 jit_inc_synth_f(pushargi_f, u);
914 jit_link_prepare();
915 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
916 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
917 /* cannot jit_movi_f in the argument register because
918 * float arguments are packed, and that would cause
919 * either an assertion in debug mode, or overwritting
920 * two registers */
921 regno = jit_get_reg(jit_class_fpr);
922 jit_movi_f(regno, u);
923 jit_movr_f(JIT_FA0 - _jitc->function->call.argf, regno);
924 jit_unget_reg(regno);
925 ++_jitc->function->call.argf;
926 goto done;
927 }
928 }
929 else {
930 if (jit_arg_reg_p(_jitc->function->call.argi)) {
931 jit_movi_f_w(JIT_RA0 - _jitc->function->call.argi, u);
932 ++_jitc->function->call.argi;
933 goto done;
934 }
935 }
936 regno = jit_get_reg(jit_class_fpr);
937 jit_movi_f(regno, u);
938 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
939 jit_unget_reg(regno);
940 _jitc->function->call.size += sizeof(jit_word_t);
941 done:
942 jit_dec_synth();
943 }
944
945 void
_jit_pushargr_d(jit_state_t * _jit,jit_int32_t u)946 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
947 {
948 assert(_jitc->function);
949 jit_inc_synth_w(pushargr_d, u);
950 jit_link_prepare();
951 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
952 if (jit_arg_d_reg_p(_jitc->function->call.argf)) {
953 if (_jitc->function->call.argf & 1)
954 ++_jitc->function->call.argf;
955 jit_movr_d(JIT_FA0 - _jitc->function->call.argf, u);
956 _jitc->function->call.argf += 2;
957 goto done;
958 }
959 }
960 else {
961 if (_jitc->function->call.argi & 1)
962 ++_jitc->function->call.argi;
963 if (jit_arg_reg_p(_jitc->function->call.argi)) {
964 jit_movr_d_ww(JIT_RA0 - _jitc->function->call.argi,
965 JIT_RA0 - (_jitc->function->call.argi + 1),
966 u);
967 _jitc->function->call.argi += 2;
968 goto done;
969 }
970 }
971 if (_jitc->function->call.size & 7)
972 _jitc->function->call.size += 4;
973 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
974 _jitc->function->call.size += sizeof(jit_float64_t);
975 done:
976 jit_dec_synth();
977 }
978
979 void
_jit_pushargi_d(jit_state_t * _jit,jit_float64_t u)980 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
981 {
982 jit_int32_t regno;
983 assert(_jitc->function);
984 jit_inc_synth_d(pushargi_d, u);
985 jit_link_prepare();
986 if (jit_cpu.abi && !(_jitc->function->call.call & jit_call_varargs)) {
987 if (jit_arg_d_reg_p(_jitc->function->call.argf)) {
988 if (_jitc->function->call.argf & 1)
989 ++_jitc->function->call.argf;
990 jit_movi_d(JIT_FA0 - _jitc->function->call.argf, u);
991 _jitc->function->call.argf += 2;
992 goto done;
993 }
994 }
995 else {
996 if (_jitc->function->call.argi & 1)
997 ++_jitc->function->call.argi;
998 if (jit_arg_reg_p(_jitc->function->call.argi)) {
999 jit_movi_d_ww(JIT_RA0 - _jitc->function->call.argi,
1000 JIT_RA0 - (_jitc->function->call.argi + 1),
1001 u);
1002 _jitc->function->call.argi += 2;
1003 goto done;
1004 }
1005 }
1006 if (_jitc->function->call.size & 7)
1007 _jitc->function->call.size += 4;
1008 regno = jit_get_reg(jit_class_fpr);
1009 jit_movi_d(regno, u);
1010 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1011 jit_unget_reg(regno);
1012 _jitc->function->call.size += sizeof(jit_float64_t);
1013 done:
1014 jit_dec_synth();
1015 }
1016
1017 jit_bool_t
_jit_regarg_p(jit_state_t * _jit,jit_node_t * node,jit_int32_t regno)1018 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1019 {
1020 jit_int32_t spec;
1021 spec = jit_class(_rvs[regno].spec);
1022 if (spec & jit_class_arg) {
1023 regno = JIT_RA0 - regno;
1024 if (regno >= 0 && regno < node->v.w)
1025 return (1);
1026 if (jit_cpu.abi && spec & jit_class_fpr) {
1027 regno = JIT_FA0 - regno;
1028 if (regno >= 0 && regno < node->w.w)
1029 return (1);
1030 }
1031 }
1032
1033 return (0);
1034 }
1035
1036 void
_jit_finishr(jit_state_t * _jit,jit_int32_t r0)1037 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1038 {
1039 jit_node_t *node;
1040 assert(_jitc->function);
1041 jit_inc_synth_w(finishr, r0);
1042 if (_jitc->function->self.alen < _jitc->function->call.size)
1043 _jitc->function->self.alen = _jitc->function->call.size;
1044 node = jit_callr(r0);
1045 node->v.w = _jitc->function->self.argi;
1046 node->w.w = _jitc->function->call.argf;
1047 _jitc->function->call.argi = _jitc->function->call.argf =
1048 _jitc->function->call.size = 0;
1049 _jitc->prepare = 0;
1050 jit_dec_synth();
1051 }
1052
1053 jit_node_t *
_jit_finishi(jit_state_t * _jit,jit_pointer_t i0)1054 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1055 {
1056 jit_node_t *node;
1057 assert(_jitc->function);
1058 jit_inc_synth_w(finishi, (jit_word_t)i0);
1059 if (_jitc->function->self.alen < _jitc->function->call.size)
1060 _jitc->function->self.alen = _jitc->function->call.size;
1061 node = jit_calli(i0);
1062 node->v.w = _jitc->function->call.argi;
1063 node->w.w = _jitc->function->call.argf;
1064 _jitc->function->call.argi = _jitc->function->call.argf =
1065 _jitc->function->call.size = 0;
1066 _jitc->prepare = 0;
1067 jit_dec_synth();
1068 return (node);
1069 }
1070
1071 void
_jit_retval_c(jit_state_t * _jit,jit_int32_t r0)1072 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1073 {
1074 jit_inc_synth_w(retval_c, r0);
1075 jit_extr_c(r0, JIT_RET);
1076 jit_dec_synth();
1077 }
1078
1079 void
_jit_retval_uc(jit_state_t * _jit,jit_int32_t r0)1080 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1081 {
1082 jit_inc_synth_w(retval_uc, r0);
1083 jit_extr_uc(r0, JIT_RET);
1084 jit_dec_synth();
1085 }
1086
1087 void
_jit_retval_s(jit_state_t * _jit,jit_int32_t r0)1088 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1089 {
1090 jit_inc_synth_w(retval_s, r0);
1091 jit_extr_s(r0, JIT_RET);
1092 jit_dec_synth();
1093 }
1094
1095 void
_jit_retval_us(jit_state_t * _jit,jit_int32_t r0)1096 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1097 {
1098 jit_inc_synth_w(retval_us, r0);
1099 jit_extr_us(r0, JIT_RET);
1100 jit_dec_synth();
1101 }
1102
1103 void
_jit_retval_i(jit_state_t * _jit,jit_int32_t r0)1104 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1105 {
1106 jit_inc_synth_w(retval_i, r0);
1107 if (r0 != JIT_RET)
1108 jit_movr(r0, JIT_RET);
1109 jit_dec_synth();
1110 }
1111
1112 void
_jit_retval_f(jit_state_t * _jit,jit_int32_t r0)1113 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1114 {
1115 jit_inc_synth_w(retval_f, r0);
1116 if (jit_cpu.abi) {
1117 if (r0 != JIT_FRET)
1118 jit_movr_f(r0, JIT_FRET);
1119 }
1120 else if (r0 != JIT_RET)
1121 jit_movr_w_f(r0, JIT_RET);
1122 jit_dec_synth();
1123 }
1124
1125 void
_jit_retval_d(jit_state_t * _jit,jit_int32_t r0)1126 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1127 {
1128 jit_inc_synth_w(retval_d, r0);
1129 if (jit_cpu.abi) {
1130 if (r0 != JIT_FRET)
1131 jit_movr_d(r0, JIT_FRET);
1132 }
1133 else if (r0 != JIT_RET)
1134 jit_movr_ww_d(r0, JIT_RET, _R1);
1135 jit_dec_synth();
1136 }
1137
1138 jit_pointer_t
_emit_code(jit_state_t * _jit)1139 _emit_code(jit_state_t *_jit)
1140 {
1141 jit_node_t *node;
1142 jit_node_t *temp;
1143 jit_word_t word;
1144 jit_int32_t value;
1145 jit_int32_t offset;
1146 struct {
1147 jit_node_t *node;
1148 jit_uint8_t *data;
1149 jit_word_t word;
1150 #if DEVEL_DISASSEMBLER
1151 jit_word_t prevw;
1152 #endif
1153 jit_uword_t thumb;
1154 #if DISASSEMBLER
1155 jit_int32_t info_offset;
1156 #endif
1157 jit_int32_t const_offset;
1158 jit_int32_t patch_offset;
1159 } undo;
1160 #if DEVEL_DISASSEMBLER
1161 jit_word_t prevw;
1162 #endif
1163
1164 _jitc->function = NULL;
1165 _jitc->thumb = 0;
1166
1167 jit_reglive_setup();
1168
1169 _jitc->consts.data = NULL;
1170 _jitc->consts.offset = _jitc->consts.length = 0;
1171
1172 undo.word = 0;
1173 undo.node = NULL;
1174 undo.data = NULL;
1175 undo.thumb = 0;
1176 #if DISASSEMBLER
1177 undo.info_offset =
1178 #endif
1179 undo.const_offset = undo.patch_offset = 0;
1180 # define assert_data(node) /**/
1181 #define case_rr(name, type) \
1182 case jit_code_##name##r##type: \
1183 name##r##type(rn(node->u.w), rn(node->v.w)); \
1184 break
1185 #define case_rw(name, type) \
1186 case jit_code_##name##i##type: \
1187 name##i##type(rn(node->u.w), node->v.w); \
1188 break
1189 #define case_vv(name, type) \
1190 case jit_code_##name##r##type: \
1191 if (jit_swf_p()) \
1192 swf_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1193 else \
1194 vfp_##name##r##type(rn(node->u.w), rn(node->v.w)); \
1195 break
1196 #define case_vw(name, type) \
1197 case jit_code_##name##i##type: \
1198 if (jit_swf_p()) \
1199 swf_##name##i##type(rn(node->u.w), node->v.w); \
1200 else \
1201 vfp_##name##i##type(rn(node->u.w), node->v.w); \
1202 break
1203 #define case_wr(name, type) \
1204 case jit_code_##name##i##type: \
1205 name##i##type(node->u.w, rn(node->v.w)); \
1206 break
1207 #define case_wv(name, type) \
1208 case jit_code_##name##i##type: \
1209 if (jit_swf_p()) \
1210 swf_##name##i##type(node->u.w, rn(node->v.w)); \
1211 else \
1212 vfp_##name##i##type(node->u.w, rn(node->v.w)); \
1213 break
1214 #define case_rrr(name, type) \
1215 case jit_code_##name##r##type: \
1216 name##r##type(rn(node->u.w), \
1217 rn(node->v.w), rn(node->w.w)); \
1218 break
1219 #define case_rrrr(name, type) \
1220 case jit_code_##name##r##type: \
1221 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
1222 rn(node->v.w), rn(node->w.w)); \
1223 break
1224 #define case_vvv(name, type) \
1225 case jit_code_##name##r##type: \
1226 if (jit_swf_p()) \
1227 swf_##name##r##type(rn(node->u.w), \
1228 rn(node->v.w), rn(node->w.w)); \
1229 else \
1230 vfp_##name##r##type(rn(node->u.w), \
1231 rn(node->v.w), rn(node->w.w)); \
1232 break
1233 #define case_rrw(name, type) \
1234 case jit_code_##name##i##type: \
1235 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1236 break
1237 #define case_rrrw(name, type) \
1238 case jit_code_##name##i##type: \
1239 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
1240 rn(node->v.w), node->w.w); \
1241 break
1242 #define case_vvw(name, type) \
1243 case jit_code_##name##i##type: \
1244 if (jit_swf_p()) \
1245 swf_##name##i##type(rn(node->u.w), \
1246 rn(node->v.w), node->w.w); \
1247 else \
1248 vfp_##name##i##type(rn(node->u.w), \
1249 rn(node->v.w), node->w.w); \
1250 break
1251 #define case_vvf(name) \
1252 case jit_code_##name##i_f: \
1253 assert_data(node); \
1254 if (jit_swf_p()) \
1255 swf_##name##i_f(rn(node->u.w), rn(node->v.w), \
1256 node->w.f); \
1257 else \
1258 vfp_##name##i_f(rn(node->u.w), rn(node->v.w), \
1259 node->w.f); \
1260 break
1261 #define case_vvd(name) \
1262 case jit_code_##name##i_d: \
1263 assert_data(node); \
1264 if (jit_swf_p()) \
1265 swf_##name##i_d(rn(node->u.w), rn(node->v.w), \
1266 node->w.d); \
1267 else \
1268 vfp_##name##i_d(rn(node->u.w), rn(node->v.w), \
1269 node->w.d); \
1270 break
1271 #define case_wrr(name, type) \
1272 case jit_code_##name##i##type: \
1273 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1274 break
1275 #define case_wvv(name, type) \
1276 case jit_code_##name##i##type: \
1277 if (jit_swf_p()) \
1278 swf_##name##i##type(node->u.w, \
1279 rn(node->v.w), rn(node->w.w)); \
1280 else \
1281 vfp_##name##i##type(node->u.w, \
1282 rn(node->v.w), rn(node->w.w)); \
1283 break
1284 #define case_brr(name, type) \
1285 case jit_code_##name##r##type: \
1286 temp = node->u.n; \
1287 assert(temp->code == jit_code_label || \
1288 temp->code == jit_code_epilog); \
1289 if (temp->flag & jit_flag_patch) \
1290 name##r##type(temp->u.w, rn(node->v.w), \
1291 rn(node->w.w)); \
1292 else { \
1293 word = name##r##type(_jit->pc.w, \
1294 rn(node->v.w), rn(node->w.w)); \
1295 patch(word, node); \
1296 } \
1297 break
1298 #define case_bvv(name, type) \
1299 case jit_code_##name##r##type: \
1300 temp = node->u.n; \
1301 assert(temp->code == jit_code_label || \
1302 temp->code == jit_code_epilog); \
1303 if (temp->flag & jit_flag_patch) { \
1304 if (jit_swf_p()) \
1305 swf_##name##r##type(temp->u.w, rn(node->v.w), \
1306 rn(node->w.w)); \
1307 else \
1308 vfp_##name##r##type(temp->u.w, rn(node->v.w), \
1309 rn(node->w.w)); \
1310 } \
1311 else { \
1312 if (jit_swf_p()) \
1313 word = swf_##name##r##type(_jit->pc.w, \
1314 rn(node->v.w), \
1315 rn(node->w.w)); \
1316 else \
1317 word = vfp_##name##r##type(_jit->pc.w, \
1318 rn(node->v.w), \
1319 rn(node->w.w)); \
1320 patch(word, node); \
1321 } \
1322 break
1323 #define case_brw(name, type) \
1324 case jit_code_##name##i##type: \
1325 temp = node->u.n; \
1326 assert(temp->code == jit_code_label || \
1327 temp->code == jit_code_epilog); \
1328 if (temp->flag & jit_flag_patch) \
1329 name##i##type(temp->u.w, \
1330 rn(node->v.w), node->w.w); \
1331 else { \
1332 word = name##i##type(_jit->pc.w, \
1333 rn(node->v.w), node->w.w); \
1334 patch(word, node); \
1335 } \
1336 break;
1337 #define case_bvf(name) \
1338 case jit_code_##name##i_f: \
1339 temp = node->u.n; \
1340 assert(temp->code == jit_code_label || \
1341 temp->code == jit_code_epilog); \
1342 if (temp->flag & jit_flag_patch) { \
1343 if (jit_swf_p()) \
1344 swf_##name##i_f(temp->u.w, rn(node->v.w), \
1345 node->w.f); \
1346 else \
1347 vfp_##name##i_f(temp->u.w, rn(node->v.w), \
1348 node->w.f); \
1349 } \
1350 else { \
1351 if (jit_swf_p()) \
1352 word = swf_##name##i_f(_jit->pc.w, \
1353 rn(node->v.w), \
1354 node->w.f); \
1355 else \
1356 word = vfp_##name##i_f(_jit->pc.w, \
1357 rn(node->v.w), \
1358 node->w.f); \
1359 patch(word, node); \
1360 } \
1361 break
1362 #define case_bvd(name) \
1363 case jit_code_##name##i_d: \
1364 temp = node->u.n; \
1365 assert(temp->code == jit_code_label || \
1366 temp->code == jit_code_epilog); \
1367 if (temp->flag & jit_flag_patch) { \
1368 if (jit_swf_p()) \
1369 swf_##name##i_d(temp->u.w, rn(node->v.w), \
1370 node->w.d); \
1371 else \
1372 vfp_##name##i_d(temp->u.w, rn(node->v.w), \
1373 node->w.d); \
1374 } \
1375 else { \
1376 if (jit_swf_p()) \
1377 word = swf_##name##i_d(_jit->pc.w, \
1378 rn(node->v.w), \
1379 node->w.d); \
1380 else \
1381 word = vfp_##name##i_d(_jit->pc.w, \
1382 rn(node->v.w), \
1383 node->w.d); \
1384 patch(word, node); \
1385 } \
1386 break
1387 #if DEVEL_DISASSEMBLER
1388 prevw = _jit->pc.w;
1389 #endif
1390 for (node = _jitc->head; node; node = node->next) {
1391 if (_jit->pc.uc >= _jitc->code.end)
1392 return (NULL);
1393
1394 #if DEVEL_DISASSEMBLER
1395 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1396 prevw = _jit->pc.w;
1397 #endif
1398 value = jit_classify(node->code);
1399 jit_regarg_set(node, value);
1400 switch (node->code) {
1401 case jit_code_align:
1402 assert(!(node->u.w & (node->u.w - 1)) &&
1403 node->u.w <= sizeof(jit_word_t));
1404 if (node->u.w == sizeof(jit_word_t) &&
1405 (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1406 nop(sizeof(jit_word_t) - word);
1407 break;
1408 case jit_code_note: case jit_code_name:
1409 if (must_align_p(node->next))
1410 nop(2);
1411 node->u.w = _jit->pc.w;
1412 break;
1413 case jit_code_label:
1414 if (must_align_p(node->next))
1415 nop(2);
1416 /* remember label is defined */
1417 node->flag |= jit_flag_patch;
1418 node->u.w = _jit->pc.w;
1419 break;
1420 case_rrr(add,);
1421 case_rrw(add,);
1422 case_rrr(addc,);
1423 case_rrw(addc,);
1424 case_rrr(addx,);
1425 case_rrw(addx,);
1426 case_rrr(sub,);
1427 case_rrw(sub,);
1428 case_rrr(subc,);
1429 case_rrw(subc,);
1430 case_rrr(subx,);
1431 case_rrw(subx,);
1432 case_rrw(rsb,);
1433 case_rrr(mul,);
1434 case_rrw(mul,);
1435 case_rrrr(qmul,);
1436 case_rrrw(qmul,);
1437 case_rrrr(qmul, _u);
1438 case_rrrw(qmul, _u);
1439 case_rrr(div,);
1440 case_rrw(div,);
1441 case_rrr(div, _u);
1442 case_rrw(div, _u);
1443 case_rrrr(qdiv,);
1444 case_rrrw(qdiv,);
1445 case_rrrr(qdiv, _u);
1446 case_rrrw(qdiv, _u);
1447 case_rrr(rem,);
1448 case_rrw(rem,);
1449 case_rrr(rem, _u);
1450 case_rrw(rem, _u);
1451 case_rrr(lsh,);
1452 case_rrw(lsh,);
1453 case_rrr(rsh,);
1454 case_rrw(rsh,);
1455 case_rrr(rsh, _u);
1456 case_rrw(rsh, _u);
1457 case_rr(neg,);
1458 case_rr(com,);
1459 case_rrr(and,);
1460 case_rrw(and,);
1461 case_rrr(or,);
1462 case_rrw(or,);
1463 case_rrr(xor,);
1464 case_rrw(xor,);
1465 case_vv(trunc, _f_i);
1466 case_vv(trunc, _d_i);
1467 case_rr(ld, _c);
1468 case_rw(ld, _c);
1469 case_rr(ld, _uc);
1470 case_rw(ld, _uc);
1471 case_rr(ld, _s);
1472 case_rw(ld, _s);
1473 case_rr(ld, _us);
1474 case_rw(ld, _us);
1475 case_rr(ld, _i);
1476 case_rw(ld, _i);
1477 case_rrr(ldx, _c);
1478 case_rrw(ldx, _c);
1479 case_rrr(ldx, _uc);
1480 case_rrw(ldx, _uc);
1481 case_rrr(ldx, _s);
1482 case_rrw(ldx, _s);
1483 case_rrr(ldx, _us);
1484 case_rrw(ldx, _us);
1485 case_rrr(ldx, _i);
1486 case_rrw(ldx, _i);
1487 case_rr(st, _c);
1488 case_wr(st, _c);
1489 case_rr(st, _s);
1490 case_wr(st, _s);
1491 case_rr(st, _i);
1492 case_wr(st, _i);
1493 case_rrr(stx, _c);
1494 case_wrr(stx, _c);
1495 case_rrr(stx, _s);
1496 case_wrr(stx, _s);
1497 case_rrr(stx, _i);
1498 case_wrr(stx, _i);
1499 case_rr(hton, _us);
1500 case_rr(hton, _ui);
1501 case_rr(ext, _c);
1502 case_rr(ext, _uc);
1503 case_rr(ext, _s);
1504 case_rr(ext, _us);
1505 case_rr(mov,);
1506 case jit_code_movi:
1507 if (node->flag & jit_flag_node) {
1508 temp = node->v.n;
1509 if (temp->code == jit_code_data ||
1510 (temp->code == jit_code_label &&
1511 (temp->flag & jit_flag_patch)))
1512 movi(rn(node->u.w), temp->u.w);
1513 else {
1514 assert(temp->code == jit_code_label ||
1515 temp->code == jit_code_epilog);
1516 word = movi_p(rn(node->u.w), temp->u.w);
1517 patch(word, node);
1518 }
1519 }
1520 else
1521 movi(rn(node->u.w), node->v.w);
1522 break;
1523 case_rrr(lt,);
1524 case_rrw(lt,);
1525 case_rrr(lt, _u);
1526 case_rrw(lt, _u);
1527 case_rrr(le,);
1528 case_rrw(le,);
1529 case_rrr(le, _u);
1530 case_rrw(le, _u);
1531 case_rrr(eq,);
1532 case_rrw(eq,);
1533 case_rrr(ge,);
1534 case_rrw(ge,);
1535 case_rrr(ge, _u);
1536 case_rrw(ge, _u);
1537 case_rrr(gt,);
1538 case_rrw(gt,);
1539 case_rrr(gt, _u);
1540 case_rrw(gt, _u);
1541 case_rrr(ne,);
1542 case_rrw(ne,);
1543 case_brr(blt,);
1544 case_brw(blt,);
1545 case_brr(blt, _u);
1546 case_brw(blt, _u);
1547 case_brr(ble,);
1548 case_brw(ble,);
1549 case_brr(ble, _u);
1550 case_brw(ble, _u);
1551 case_brr(beq,);
1552 case_brw(beq,);
1553 case_brr(bge,);
1554 case_brw(bge,);
1555 case_brr(bge, _u);
1556 case_brw(bge, _u);
1557 case_brr(bgt,);
1558 case_brw(bgt,);
1559 case_brr(bgt, _u);
1560 case_brw(bgt, _u);
1561 case_brr(bne,);
1562 case_brw(bne,);
1563 case_brr(boadd,);
1564 case_brw(boadd,);
1565 case_brr(boadd, _u);
1566 case_brw(boadd, _u);
1567 case_brr(bxadd,);
1568 case_brw(bxadd,);
1569 case_brr(bxadd, _u);
1570 case_brw(bxadd, _u);
1571 case_brr(bosub,);
1572 case_brw(bosub,);
1573 case_brr(bosub, _u);
1574 case_brw(bosub, _u);
1575 case_brr(bxsub,);
1576 case_brw(bxsub,);
1577 case_brr(bxsub, _u);
1578 case_brw(bxsub, _u);
1579 case_brr(bms,);
1580 case_brw(bms,);
1581 case_brr(bmc,);
1582 case_brw(bmc,);
1583 case_vvv(add, _f);
1584 case_vvf(add);
1585 case_vvv(sub, _f);
1586 case_vvf(sub);
1587 case_vvf(rsb);
1588 case_vvv(mul, _f);
1589 case_vvf(mul);
1590 case_vvv(div, _f);
1591 case_vvf(div);
1592 case_vv(abs, _f);
1593 case_vv(neg, _f);
1594 case_vv(sqrt, _f);
1595 case_vv(ext, _f);
1596 case_vv(ld, _f);
1597 case_vw(ld, _f);
1598 case_vvv(ldx, _f);
1599 case_vvw(ldx, _f);
1600 case_vv(st, _f);
1601 case_wv(st, _f);
1602 case_vvv(stx, _f);
1603 case_wvv(stx, _f);
1604 case_vv(mov, _f);
1605 case jit_code_movi_f:
1606 assert_data(node);
1607 if (jit_swf_p())
1608 swf_movi_f(rn(node->u.w), node->v.f);
1609 else
1610 vfp_movi_f(rn(node->u.w), node->v.f);
1611 break;
1612 case_vv(ext, _d_f);
1613 case_vvv(lt, _f);
1614 case_vvf(lt);
1615 case_vvv(le, _f);
1616 case_vvf(le);
1617 case_vvv(eq, _f);
1618 case_vvf(eq);
1619 case_vvv(ge, _f);
1620 case_vvf(ge);
1621 case_vvv(gt, _f);
1622 case_vvf(gt);
1623 case_vvv(ne, _f);
1624 case_vvf(ne);
1625 case_vvv(unlt, _f);
1626 case_vvf(unlt);
1627 case_vvv(unle, _f);
1628 case_vvf(unle);
1629 case_vvv(uneq, _f);
1630 case_vvf(uneq);
1631 case_vvv(unge, _f);
1632 case_vvf(unge);
1633 case_vvv(ungt, _f);
1634 case_vvf(ungt);
1635 case_vvv(ltgt, _f);
1636 case_vvf(ltgt);
1637 case_vvv(ord, _f);
1638 case_vvf(ord);
1639 case_vvv(unord, _f);
1640 case_vvf(unord);
1641 case_bvv(blt, _f);
1642 case_bvf(blt);
1643 case_bvv(ble, _f);
1644 case_bvf(ble);
1645 case_bvv(beq, _f);
1646 case_bvf(beq);
1647 case_bvv(bge, _f);
1648 case_bvf(bge);
1649 case_bvv(bgt, _f);
1650 case_bvf(bgt);
1651 case_bvv(bne, _f);
1652 case_bvf(bne);
1653 case_bvv(bunlt, _f);
1654 case_bvf(bunlt);
1655 case_bvv(bunle, _f);
1656 case_bvf(bunle);
1657 case_bvv(buneq, _f);
1658 case_bvf(buneq);
1659 case_bvv(bunge, _f);
1660 case_bvf(bunge);
1661 case_bvv(bungt, _f);
1662 case_bvf(bungt);
1663 case_bvv(bltgt, _f);
1664 case_bvf(bltgt);
1665 case_bvv(bord, _f);
1666 case_bvf(bord);
1667 case_bvv(bunord, _f);
1668 case_bvf(bunord);
1669 case_vvv(add, _d);
1670 case_vvd(add);
1671 case_vvv(sub, _d);
1672 case_vvd(sub);
1673 case_vvd(rsb);
1674 case_vvv(mul, _d);
1675 case_vvd(mul);
1676 case_vvv(div, _d);
1677 case_vvd(div);
1678 case_vv(abs, _d);
1679 case_vv(neg, _d);
1680 case_vv(sqrt, _d);
1681 case_vv(ext, _d);
1682 case_vv(ld, _d);
1683 case_vw(ld, _d);
1684 case_vvv(ldx, _d);
1685 case_vvw(ldx, _d);
1686 case_vv(st, _d);
1687 case_wv(st, _d);
1688 case_vvv(stx, _d);
1689 case_wvv(stx, _d);
1690 case_vv(mov, _d);
1691 case jit_code_movi_d:
1692 assert_data(node);
1693 if (jit_swf_p())
1694 swf_movi_d(rn(node->u.w), node->v.d);
1695 else
1696 vfp_movi_d(rn(node->u.w), node->v.d);
1697 break;
1698 case_vv(ext, _f_d);
1699 case_vvv(lt, _d);
1700 case_vvd(lt);
1701 case_vvv(le, _d);
1702 case_vvd(le);
1703 case_vvv(eq, _d);
1704 case_vvd(eq);
1705 case_vvv(ge, _d);
1706 case_vvd(ge);
1707 case_vvv(gt, _d);
1708 case_vvd(gt);
1709 case_vvv(ne, _d);
1710 case_vvd(ne);
1711 case_vvv(unlt, _d);
1712 case_vvd(unlt);
1713 case_vvv(unle, _d);
1714 case_vvd(unle);
1715 case_vvv(uneq, _d);
1716 case_vvd(uneq);
1717 case_vvv(unge, _d);
1718 case_vvd(unge);
1719 case_vvv(ungt, _d);
1720 case_vvd(ungt);
1721 case_vvv(ltgt, _d);
1722 case_vvd(ltgt);
1723 case_vvv(ord, _d);
1724 case_vvd(ord);
1725 case_vvv(unord, _d);
1726 case_vvd(unord);
1727 case_bvv(blt, _d);
1728 case_bvd(blt);
1729 case_bvv(ble, _d);
1730 case_bvd(ble);
1731 case_bvv(beq, _d);
1732 case_bvd(beq);
1733 case_bvv(bge, _d);
1734 case_bvd(bge);
1735 case_bvv(bgt, _d);
1736 case_bvd(bgt);
1737 case_bvv(bne, _d);
1738 case_bvd(bne);
1739 case_bvv(bunlt, _d);
1740 case_bvd(bunlt);
1741 case_bvv(bunle, _d);
1742 case_bvd(bunle);
1743 case_bvv(buneq, _d);
1744 case_bvd(buneq);
1745 case_bvv(bunge, _d);
1746 case_bvd(bunge);
1747 case_bvv(bungt, _d);
1748 case_bvd(bungt);
1749 case_bvv(bltgt, _d);
1750 case_bvd(bltgt);
1751 case_bvv(bord, _d);
1752 case_bvd(bord);
1753 case_bvv(bunord, _d);
1754 case_bvd(bunord);
1755 case jit_code_jmpr:
1756 jmpr(rn(node->u.w));
1757 flush_consts();
1758 break;
1759 case jit_code_jmpi:
1760 if (node->flag & jit_flag_node) {
1761 temp = node->u.n;
1762 assert(temp->code == jit_code_label ||
1763 temp->code == jit_code_epilog);
1764 if (temp->flag & jit_flag_patch)
1765 jmpi(temp->u.w);
1766 else {
1767 word = jmpi_p(_jit->pc.w, 1);
1768 patch(word, node);
1769 }
1770 }
1771 else
1772 jmpi(node->u.w);
1773 flush_consts();
1774 break;
1775 case jit_code_callr:
1776 callr(rn(node->u.w));
1777 break;
1778 case jit_code_calli:
1779 if (node->flag & jit_flag_node) {
1780 temp = node->u.n;
1781 assert(temp->code == jit_code_label ||
1782 temp->code == jit_code_epilog);
1783 if (temp->flag & jit_flag_patch)
1784 calli(temp->u.w);
1785 else {
1786 word = calli_p(_jit->pc.w);
1787 patch(word, node);
1788 }
1789 }
1790 else
1791 calli(node->u.w);
1792 break;
1793 case jit_code_prolog:
1794 _jitc->function = _jitc->functions.ptr + node->w.w;
1795 undo.node = node;
1796 undo.word = _jit->pc.w;
1797 #if DEVEL_DISASSEMBLER
1798 undo.prevw = prevw;
1799 #endif
1800 undo.data = _jitc->consts.data;
1801 undo.thumb = _jitc->thumb;
1802 undo.const_offset = _jitc->consts.offset;
1803 undo.patch_offset = _jitc->patches.offset;
1804 #if DISASSEMBLER
1805 if (_jitc->data_info.ptr)
1806 undo.info_offset = _jitc->data_info.offset;
1807 #endif
1808 restart_function:
1809 _jitc->again = 0;
1810 prolog(node);
1811 break;
1812 case jit_code_epilog:
1813 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1814 if (_jitc->again) {
1815 for (temp = undo.node->next;
1816 temp != node; temp = temp->next) {
1817 if (temp->code == jit_code_label ||
1818 temp->code == jit_code_epilog)
1819 temp->flag &= ~jit_flag_patch;
1820 }
1821 temp->flag &= ~jit_flag_patch;
1822 node = undo.node;
1823 _jit->pc.w = undo.word;
1824 #if DEVEL_DISASSEMBLER
1825 prevw = undo.prevw;
1826 #endif
1827 invalidate_consts();
1828 _jitc->consts.data = undo.data;
1829 _jitc->thumb = undo.thumb;
1830 _jitc->consts.offset = undo.const_offset;
1831 _jitc->patches.offset = undo.patch_offset;
1832 #if DISASSEMBLER
1833 if (_jitc->data_info.ptr)
1834 _jitc->data_info.offset = undo.info_offset;
1835 #endif
1836 goto restart_function;
1837 }
1838 /* remember label is defined */
1839 node->flag |= jit_flag_patch;
1840 node->u.w = _jit->pc.w;
1841 epilog(node);
1842 _jitc->function = NULL;
1843 flush_consts();
1844 break;
1845 case jit_code_movr_w_f:
1846 if (jit_swf_p())
1847 swf_movr_f(rn(node->u.w), rn(node->v.w));
1848 else
1849 vfp_movr_f(rn(node->u.w), rn(node->v.w));
1850 break;
1851 case jit_code_movr_f_w:
1852 if (jit_swf_p())
1853 swf_movr_f(rn(node->u.w), rn(node->v.w));
1854 else
1855 vfp_movr_f(rn(node->u.w), rn(node->v.w));
1856 break;
1857 case jit_code_movi_f_w:
1858 assert_data(node);
1859 if (jit_swf_p())
1860 swf_movi_f(rn(node->u.w), node->v.f);
1861 else
1862 vfp_movi_f(rn(node->u.w), node->v.f);
1863 break;
1864 case jit_code_movr_ww_d:
1865 if (jit_swf_p())
1866 swf_movr_d(rn(node->u.w), rn(node->v.w));
1867 else
1868 vfp_movr_d(rn(node->u.w), rn(node->v.w));
1869 break;
1870 case jit_code_movr_d_ww:
1871 if (jit_swf_p())
1872 swf_movr_d(rn(node->u.w), rn(node->w.w));
1873 else
1874 vfp_movr_d(rn(node->u.w), rn(node->w.w));
1875 break;
1876 case jit_code_movi_d_ww:
1877 assert_data(node);
1878 if (jit_swf_p())
1879 swf_movi_d(rn(node->u.w), node->w.d);
1880 else
1881 vfp_movi_d(rn(node->u.w), node->w.d);
1882 break;
1883 case jit_code_va_start:
1884 vastart(rn(node->u.w));
1885 break;
1886 case jit_code_va_arg:
1887 vaarg(rn(node->u.w), rn(node->v.w));
1888 break;
1889 case jit_code_va_arg_d:
1890 if (jit_swf_p())
1891 swf_vaarg_d(rn(node->u.w), rn(node->v.w));
1892 else
1893 vfp_vaarg_d(rn(node->u.w), rn(node->v.w));
1894 break;
1895 case jit_code_live: case jit_code_ellipsis:
1896 case jit_code_va_push:
1897 case jit_code_allocai: case jit_code_allocar:
1898 case jit_code_arg:
1899 case jit_code_arg_f: case jit_code_arg_d:
1900 case jit_code_va_end:
1901 case jit_code_ret:
1902 case jit_code_retr: case jit_code_reti:
1903 case jit_code_retr_f: case jit_code_reti_f:
1904 case jit_code_retr_d: case jit_code_reti_d:
1905 case jit_code_getarg_c: case jit_code_getarg_uc:
1906 case jit_code_getarg_s: case jit_code_getarg_us:
1907 case jit_code_getarg_i:
1908 case jit_code_getarg_f: case jit_code_getarg_d:
1909 case jit_code_putargr: case jit_code_putargi:
1910 case jit_code_putargr_f: case jit_code_putargi_f:
1911 case jit_code_putargr_d: case jit_code_putargi_d:
1912 case jit_code_pushargr: case jit_code_pushargi:
1913 case jit_code_pushargr_f: case jit_code_pushargi_f:
1914 case jit_code_pushargr_d: case jit_code_pushargi_d:
1915 case jit_code_retval_c: case jit_code_retval_uc:
1916 case jit_code_retval_s: case jit_code_retval_us:
1917 case jit_code_retval_i:
1918 case jit_code_retval_f: case jit_code_retval_d:
1919 case jit_code_prepare:
1920 case jit_code_finishr: case jit_code_finishi:
1921 break;
1922 default:
1923 abort();
1924 }
1925 jit_regarg_clr(node, value);
1926 assert(_jitc->regarg == 0 && _jitc->synth == 0);
1927 /* update register live state */
1928 jit_reglive(node);
1929
1930 if (_jitc->consts.length &&
1931 (_jit->pc.uc - _jitc->consts.data >= 3968 ||
1932 (jit_uword_t)_jit->pc.uc -
1933 (jit_uword_t)_jitc->consts.patches[0] >= 3968)) {
1934 /* longest sequence should be 64 bytes, but preventively
1935 * do not let it go past 128 remaining bytes before a flush */
1936 if (node->next &&
1937 node->next->code != jit_code_jmpi &&
1938 node->next->code != jit_code_jmpr &&
1939 node->next->code != jit_code_epilog) {
1940 /* insert a jump, flush constants and continue */
1941 word = _jit->pc.w;
1942 assert(!jit_thumb_p());
1943 B(0);
1944 flush_consts();
1945 patch_at(arm_patch_jump, word, _jit->pc.w);
1946 }
1947 }
1948 }
1949 #undef case_bvd
1950 #undef case_bvf
1951 #undef case_brw
1952 #undef case_bvv
1953 #undef case_brr
1954 #undef case_wvv
1955 #undef case_wrr
1956 #undef case_vvd
1957 #undef case_vvf
1958 #undef case_vvw
1959 #undef case_rrw
1960 #undef case_vvv
1961 #undef case_rrr
1962 #undef case_wv
1963 #undef case_wr
1964 #undef case_vw
1965 #undef case_vv
1966 #undef case_rw
1967 #undef case_rr
1968
1969 flush_consts();
1970 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1971 assert(_jitc->patches.ptr[offset].kind & arm_patch_node);
1972 node = _jitc->patches.ptr[offset].node;
1973 word = _jitc->patches.ptr[offset].inst;
1974 if (!jit_thumb_p() &&
1975 (node->code == jit_code_movi || node->code == jit_code_calli)) {
1976 /* calculate where to patch word */
1977 value = *(jit_int32_t *)word;
1978 assert((value & 0x0f700000) == ARM_LDRI);
1979 /* offset may become negative (-4) if last instruction
1980 * before unconditional branch and data following
1981 * FIXME can this cause issues in the preprocessor prefetch
1982 * or something else? should not, as the constants are after
1983 * an unconditional jump */
1984 if (value & ARM_P) value = value & 0x00000fff;
1985 else value = -(value & 0x00000fff);
1986 word = word + 8 + value;
1987 }
1988 value = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1989 patch_at(_jitc->patches.ptr[offset].kind & ~arm_patch_node, word, value);
1990 }
1991
1992 jit_flush(_jit->code.ptr, _jit->pc.uc);
1993
1994 return (_jit->code.ptr);
1995 }
1996
1997 #define CODE 1
1998 # include "jit_rewind.c"
1999 # include "jit_arm-cpu.c"
2000 # include "jit_arm-swf.c"
2001 # include "jit_arm-vfp.c"
2002 #undef CODE
2003
2004 void
jit_flush(void * fptr,void * tptr)2005 jit_flush(void *fptr, void *tptr)
2006 {
2007 #if defined(__GNUC__)
2008 jit_uword_t i, f, t, s;
2009
2010 s = sysconf(_SC_PAGE_SIZE);
2011 f = (jit_uword_t)fptr & -s;
2012 t = (((jit_uword_t)tptr) + s - 1) & -s;
2013 for (i = f; i < t; i += s)
2014 __clear_cache((void *)i, (void *)(i + s));
2015 #endif
2016 }
2017
2018 void
_emit_ldxi(jit_state_t * _jit,jit_int32_t r0,jit_int32_t r1,jit_word_t i0)2019 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2020 {
2021 ldxi_i(rn(r0), rn(r1), i0);
2022 }
2023
2024 void
_emit_stxi(jit_state_t * _jit,jit_word_t i0,jit_int32_t r0,jit_int32_t r1)2025 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2026 {
2027 stxi_i(i0, rn(r0), rn(r1));
2028 }
2029
2030 void
_emit_ldxi_d(jit_state_t * _jit,jit_int32_t r0,jit_int32_t r1,jit_word_t i0)2031 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
2032 {
2033 if (jit_swf_p())
2034 swf_ldxi_d(rn(r0), rn(r1), i0);
2035 else
2036 vfp_ldxi_d(rn(r0), rn(r1), i0);
2037 }
2038
2039 void
_emit_stxi_d(jit_state_t * _jit,jit_word_t i0,jit_int32_t r0,jit_int32_t r1)2040 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
2041 {
2042 if (jit_swf_p())
2043 swf_stxi_d(i0, rn(r0), rn(r1));
2044 else
2045 vfp_stxi_d(i0, rn(r0), rn(r1));
2046 }
2047
2048 static jit_int32_t
_jit_get_reg_pair(jit_state_t * _jit)2049 _jit_get_reg_pair(jit_state_t *_jit)
2050 {
2051 /* bypass jit_get_reg() with argument or'ed with jit_class_chk
2052 * and try to find an consecutive, even free register pair, or
2053 * return JIT_NOREG if fail, as the cost of spills is greater
2054 * than splitting a double load/store in two operations. */
2055 if (jit_reg_free_p(_R0) && jit_reg_free_p(_R1)) {
2056 jit_regset_setbit(&_jitc->regarg, _R0);
2057 jit_regset_setbit(&_jitc->regarg, _R1);
2058 return (_R0);
2059 }
2060 if (jit_reg_free_p(_R2) && jit_reg_free_p(_R3)) {
2061 jit_regset_setbit(&_jitc->regarg, _R2);
2062 jit_regset_setbit(&_jitc->regarg, _R3);
2063 return (_R2);
2064 }
2065 if (jit_reg_free_p(_R4) && jit_reg_free_p(_R5)) {
2066 jit_regset_setbit(&_jitc->regarg, _R4);
2067 jit_regset_setbit(&_jitc->regarg, _R5);
2068 return (_R4);
2069 }
2070 if (jit_reg_free_p(_R6) && jit_reg_free_p(_R7)) {
2071 jit_regset_setbit(&_jitc->regarg, _R6);
2072 jit_regset_setbit(&_jitc->regarg, _R7);
2073 return (_R6);
2074 }
2075 if (jit_reg_free_p(_R8) && jit_reg_free_p(_R9)) {
2076 jit_regset_setbit(&_jitc->regarg, _R8);
2077 jit_regset_setbit(&_jitc->regarg, _R9);
2078 return (_R8);
2079 }
2080 return (JIT_NOREG);
2081 }
2082
2083 static void
_jit_unget_reg_pair(jit_state_t * _jit,jit_int32_t reg)2084 _jit_unget_reg_pair(jit_state_t *_jit, jit_int32_t reg)
2085 {
2086 jit_unget_reg(reg);
2087 switch (reg) {
2088 case _R0: jit_unget_reg(_R1); break;
2089 case _R2: jit_unget_reg(_R3); break;
2090 case _R4: jit_unget_reg(_R5); break;
2091 case _R6: jit_unget_reg(_R7); break;
2092 case _R8: jit_unget_reg(_R9); break;
2093 default: abort();
2094 }
2095 }
2096
2097 /* A prolog must be aligned at mod 4 bytes boundary.
2098 * This condition was not being required to be tested by
2099 * accident previously, but with the jit_frame and jit_tramp
2100 * code it is required */
2101 static jit_bool_t
_must_align_p(jit_state_t * _jit,jit_node_t * node)2102 _must_align_p(jit_state_t *_jit, jit_node_t *node)
2103 {
2104 if (jit_thumb_p() && (_jit->pc.w & 3)) {
2105 for (; node; node = node->next) {
2106 switch (node->code) {
2107 case jit_code_note:
2108 case jit_code_name:
2109 case jit_code_label:
2110 break;
2111 case jit_code_prolog:
2112 return (1);
2113 default:
2114 return (0);
2115 }
2116 }
2117 }
2118 return (0);
2119 }
2120
2121 static void
_load_const(jit_state_t * _jit,jit_bool_t uniq,jit_int32_t r0,jit_word_t i0)2122 _load_const(jit_state_t *_jit, jit_bool_t uniq, jit_int32_t r0, jit_word_t i0)
2123 {
2124 jit_word_t w;
2125 jit_word_t d;
2126 jit_word_t base;
2127 jit_int32_t *data;
2128 jit_int32_t size;
2129 jit_int32_t offset;
2130
2131 assert(!jit_thumb_p());
2132 if (!uniq) {
2133 /* use zero, a valid directly encoded immediate, to avoid the
2134 * need of a bitmask to know what offsets will be patched, so
2135 * that comparison will always fail for constants that cannot
2136 * be encoded */
2137 assert(i0 != 0);
2138
2139 /* Actually, code is (currently at least) not self modifying,
2140 * so, any value reachable backwards is valid as a constant. */
2141
2142 /* FIXME a quickly updateable/mutable hash table could be
2143 * better here, but most times only a few comparisons
2144 * should be done
2145 */
2146
2147 /* search in previous constant pool */
2148 if ((data = (jit_int32_t *)_jitc->consts.data)) {
2149 w = (jit_word_t)data;
2150 /* maximum backwards offset */
2151 base = (_jit->pc.w + 8) - 4092;
2152 if (base <= w)
2153 /* can scan all possible available backward constants */
2154 base = 0;
2155 else
2156 base = (base - w) >> 2;
2157 size = _jitc->consts.size >> 2;
2158 for (offset = size - 1; offset >= base; offset--) {
2159 if (data[offset] == i0) {
2160 w = (jit_word_t)(data + offset);
2161 d = (_jit->pc.w + 8) - w;
2162 LDRIN(r0, _R15_REGNO, d);
2163 return;
2164 }
2165 }
2166 }
2167 }
2168 else
2169 assert(i0 == 0);
2170
2171 _jitc->consts.patches[_jitc->consts.offset++] = _jit->pc.w;
2172 /* (probably) positive forward offset */
2173 LDRI(r0, _R15_REGNO, 0);
2174
2175 if (!uniq) {
2176 /* search already requested values */
2177 for (offset = 0; offset < _jitc->consts.length; offset++) {
2178 if (_jitc->consts.values[offset] == i0) {
2179 _jitc->consts.patches[_jitc->consts.offset++] = offset;
2180 return;
2181 }
2182 }
2183 }
2184
2185 #if DEBUG
2186 /* cannot run out of space because of limited range
2187 * but assert anyway to catch logic errors */
2188 assert(_jitc->consts.length < 1024);
2189 assert(_jitc->consts.offset < 2048);
2190 #endif
2191 _jitc->consts.patches[_jitc->consts.offset++] = _jitc->consts.length;
2192 _jitc->consts.values[_jitc->consts.length++] = i0;
2193 }
2194
2195 static void
_flush_consts(jit_state_t * _jit)2196 _flush_consts(jit_state_t *_jit)
2197 {
2198 jit_word_t word;
2199 jit_int32_t offset;
2200
2201 /* if no forward constants */
2202 if (!_jitc->consts.length)
2203 return;
2204 assert(!jit_thumb_p());
2205 word = _jit->pc.w;
2206 _jitc->consts.data = _jit->pc.uc;
2207 _jitc->consts.size = _jitc->consts.length << 2;
2208 /* FIXME check will not overrun, otherwise, need to reallocate
2209 * code buffer and start over */
2210 jit_memcpy(_jitc->consts.data, _jitc->consts.values, _jitc->consts.size);
2211 _jit->pc.w += _jitc->consts.size;
2212
2213 #if DISASSEMBLER
2214 if (_jitc->data_info.ptr) {
2215 if (_jitc->data_info.offset >= _jitc->data_info.length) {
2216 jit_realloc((jit_pointer_t *)&_jitc->data_info.ptr,
2217 _jitc->data_info.length * sizeof(jit_data_info_t),
2218 (_jitc->data_info.length + 1024) *
2219 sizeof(jit_data_info_t));
2220 _jitc->data_info.length += 1024;
2221 }
2222 _jitc->data_info.ptr[_jitc->data_info.offset].code = word;
2223 _jitc->data_info.ptr[_jitc->data_info.offset].length = _jitc->consts.size;
2224 ++_jitc->data_info.offset;
2225 }
2226 #endif
2227
2228 for (offset = 0; offset < _jitc->consts.offset; offset += 2)
2229 patch_at(arm_patch_load, _jitc->consts.patches[offset],
2230 word + (_jitc->consts.patches[offset + 1] << 2));
2231 _jitc->consts.length = _jitc->consts.offset = 0;
2232 }
2233
2234 /* to be called if needing to start over a function */
2235 static void
_invalidate_consts(jit_state_t * _jit)2236 _invalidate_consts(jit_state_t *_jit)
2237 {
2238 /* if no forward constants */
2239 if (_jitc->consts.length)
2240 _jitc->consts.length = _jitc->consts.offset = 0;
2241 }
2242
2243 static void
_patch(jit_state_t * _jit,jit_word_t instr,jit_node_t * node)2244 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
2245 {
2246 jit_int32_t flag;
2247 jit_int32_t kind;
2248
2249 assert(node->flag & jit_flag_node);
2250 if (node->code == jit_code_movi) {
2251 flag = node->v.n->flag;
2252 kind = arm_patch_word;
2253 }
2254 else {
2255 flag = node->u.n->flag;
2256 if (node->code == jit_code_calli ||
2257 (node->code == jit_code_jmpi && !(node->flag & jit_flag_node)))
2258 kind = arm_patch_word;
2259 else
2260 kind = arm_patch_jump;
2261 }
2262 assert(!(flag & jit_flag_patch));
2263 kind |= arm_patch_node;
2264 if (_jitc->patches.offset >= _jitc->patches.length) {
2265 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
2266 _jitc->patches.length * sizeof(jit_patch_t),
2267 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
2268 _jitc->patches.length += 1024;
2269 }
2270 _jitc->patches.ptr[_jitc->patches.offset].kind = kind;
2271 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
2272 _jitc->patches.ptr[_jitc->patches.offset].node = node;
2273 ++_jitc->patches.offset;
2274 }
2275