1 /*
2 * Copyright (C) 2012-2019 Free Software Foundation, Inc.
3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20 #if defined(__linux__)
21 # include <sys/cachectl.h>
22 #endif
23
24 #if NEW_ABI
25 # define NUM_WORD_ARGS 8
26 # define STACK_SLOT 8
27 # define STACK_SHIFT 3
28 #else
29 # define NUM_WORD_ARGS 4
30 # define STACK_SLOT 4
31 # define STACK_SHIFT 2
32 #endif
33 #if NEW_ABI && __BYTE_ORDER == __BIG_ENDIAN && __WORDSIZE == 32
34 # define WORD_ADJUST 4
35 #else
36 # define WORD_ADJUST 0
37 #endif
38 #define jit_arg_reg_p(i) ((i) >= 0 && (i) < NUM_WORD_ARGS)
39 #if __BYTE_ORDER == __LITTLE_ENDIAN
40 # define C_DISP 0
41 # define S_DISP 0
42 # define I_DISP 0
43 #else
44 # define C_DISP STACK_SLOT - sizeof(jit_int8_t)
45 # define S_DISP STACK_SLOT - sizeof(jit_int16_t)
46 # define I_DISP STACK_SLOT - sizeof(jit_int32_t)
47 #endif
48
49 /*
50 * Types
51 */
52 typedef struct jit_pointer_t jit_va_list_t;
53
54 /*
55 * Prototypes
56 */
57 #define jit_make_arg(node) _jit_make_arg(_jit,node)
58 static jit_node_t *_jit_make_arg(jit_state_t*,jit_node_t*);
59 #define jit_make_arg_f(node) _jit_make_arg_f(_jit,node)
60 static jit_node_t *_jit_make_arg_f(jit_state_t*,jit_node_t*);
61 #define jit_make_arg_d(node) _jit_make_arg_d(_jit,node)
62 static jit_node_t *_jit_make_arg_d(jit_state_t*,jit_node_t*);
63 #define patch(instr, node) _patch(_jit, instr, node)
64 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
65
66 #define PROTO 1
67 # include "jit_rewind.c"
68 # include "jit_mips-cpu.c"
69 # include "jit_mips-fpu.c"
70 #undef PROTO
71
72 /*
73 * Initialization
74 */
75 jit_register_t _rvs[] = {
76 { rc(gpr) | 0x01, "at" },
77 { rc(gpr) | 0x02, "v0" },
78 { rc(gpr) | 0x03, "v1" },
79 #if !NEW_ABI
80 { rc(gpr) | 0x08, "t0" },
81 { rc(gpr) | 0x09, "t1" },
82 { rc(gpr) | 0x0a, "t2" },
83 { rc(gpr) | 0x0b, "t3" },
84 #endif
85 { rc(gpr) | 0x0c, "t4" },
86 { rc(gpr) | 0x0d, "t5" },
87 { rc(gpr) | 0x0e, "t6" },
88 { rc(gpr) | 0x0f, "t7" },
89 { rc(gpr) | 0x18, "t8" },
90 { rc(gpr) | 0x19, "t9" },
91 { rc(sav) | rc(gpr) | 0x10, "s0" },
92 { rc(sav) | rc(gpr) | 0x11, "s1" },
93 { rc(sav) | rc(gpr) | 0x12, "s2" },
94 { rc(sav) | rc(gpr) | 0x13, "s3" },
95 { rc(sav) | rc(gpr) | 0x14, "s4" },
96 { rc(sav) | rc(gpr) | 0x15, "s5" },
97 { rc(sav) | rc(gpr) | 0x16, "s6" },
98 { rc(sav) | rc(gpr) | 0x17, "s7" },
99 { 0x00, "zero" },
100 { 0x1a, "k0" },
101 { 0x1b, "k1" },
102 { rc(sav) | 0x1f, "ra" },
103 { rc(sav) | 0x1c, "gp" },
104 { rc(sav) | 0x1d, "sp" },
105 { rc(sav) | 0x1e, "fp" },
106 #if NEW_ABI
107 { rc(gpr) | 0x0b, "a7" },
108 { rc(gpr) | 0x0a, "a6" },
109 { rc(gpr) | 0x09, "a5" },
110 { rc(gpr) | 0x08, "a4" },
111 #endif
112 { rc(arg) | rc(gpr) | 0x07, "a3" },
113 { rc(arg) | rc(gpr) | 0x06, "a2" },
114 { rc(arg) | rc(gpr) | 0x05, "a1" },
115 { rc(arg) | rc(gpr) | 0x04, "a0" },
116 { rc(fpr) | 0x00, "$f0" },
117 { rc(fpr) | 0x02, "$f2" },
118 { rc(fpr) | 0x04, "$f4" },
119 { rc(fpr) | 0x06, "$f6" },
120 { rc(fpr) | 0x08, "$f8" },
121 { rc(fpr) | 0x0a, "$f10" },
122 #if !NEW_ABI
123 { rc(sav) | rc(fpr) | 0x10, "$f16" },
124 { rc(sav) | rc(fpr) | 0x12, "$f18" },
125 #endif
126 { rc(sav) | rc(fpr) | 0x14, "$f20" },
127 { rc(sav) | rc(fpr) | 0x16, "$f22" },
128 { rc(sav) | rc(fpr) | 0x18, "$f24" },
129 { rc(sav) | rc(fpr) | 0x1a, "$f26" },
130 { rc(sav) | rc(fpr) | 0x1c, "$f28" },
131 { rc(sav) | rc(fpr) | 0x1e, "$f30" },
132 #if NEW_ABI
133 { rc(arg) | rc(fpr) | 0x13, "$f19" },
134 { rc(arg) | rc(fpr) | 0x12, "$f18" },
135 { rc(arg) | rc(fpr) | 0x11, "$f17" },
136 { rc(arg) | rc(fpr) | 0x10, "$f16" },
137 { rc(arg) | rc(fpr) | 0x0f, "$f15" },
138 { rc(arg) | rc(fpr) | 0x0e, "$f14" },
139 { rc(arg) | rc(fpr) | 0x0d, "$f13" },
140 { rc(arg) | rc(fpr) | 0x0c, "$f12" },
141 #else
142 { rc(arg) | rc(fpr) | 0x0e, "$f14" },
143 { rc(arg) | rc(fpr) | 0x0c, "$f12" },
144 #endif
145 { _NOREG, "<none>" },
146 };
147
148 /*
149 * Implementation
150 */
151 void
jit_get_cpu(void)152 jit_get_cpu(void)
153 {
154 }
155
156 void
_jit_init(jit_state_t * _jit)157 _jit_init(jit_state_t *_jit)
158 {
159 _jitc->reglen = jit_size(_rvs) - 1;
160 /* Could also:
161 * o reserve a register for carry (overkill)
162 * o use MTLO/MFLO (performance hit)
163 * So, keep a register allocated after setting carry, and implicitly
164 * deallocate it if it can no longer be tracked
165 */
166 jit_carry = _NOREG;
167 }
168
169 void
_jit_prolog(jit_state_t * _jit)170 _jit_prolog(jit_state_t *_jit)
171 {
172 jit_int32_t offset;
173
174 if (_jitc->function)
175 jit_epilog();
176 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
177 jit_regset_set_ui(&_jitc->regsav, 0);
178 offset = _jitc->functions.offset;
179 if (offset >= _jitc->functions.length) {
180 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
181 _jitc->functions.length * sizeof(jit_function_t),
182 (_jitc->functions.length + 16) * sizeof(jit_function_t));
183 _jitc->functions.length += 16;
184 }
185 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
186 _jitc->function->self.size = stack_framesize;
187 _jitc->function->self.argi = _jitc->function->self.argf =
188 _jitc->function->self.aoff = _jitc->function->self.alen = 0;
189 _jitc->function->self.call = jit_call_default;
190 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
191 _jitc->reglen * sizeof(jit_int32_t));
192
193 /* _no_link here does not mean the jit_link() call can be removed
194 * by rewriting as:
195 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
196 */
197 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
198 jit_link(_jitc->function->prolog);
199 _jitc->function->prolog->w.w = offset;
200 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
201 /* u: label value
202 * v: offset in blocks vector
203 * w: offset in functions vector
204 */
205 _jitc->function->epilog->w.w = offset;
206
207 jit_regset_new(&_jitc->function->regset);
208 }
209
210 jit_int32_t
_jit_allocai(jit_state_t * _jit,jit_int32_t length)211 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
212 {
213 assert(_jitc->function);
214 switch (length) {
215 case 0: case 1: break;
216 case 2: _jitc->function->self.aoff &= -2; break;
217 case 3: case 4: _jitc->function->self.aoff &= -4; break;
218 default: _jitc->function->self.aoff &= -8; break;
219 }
220 _jitc->function->self.aoff -= length;
221 if (!_jitc->realize) {
222 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
223 jit_dec_synth();
224 }
225 return (_jitc->function->self.aoff);
226 }
227
228 void
_jit_allocar(jit_state_t * _jit,jit_int32_t u,jit_int32_t v)229 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
230 {
231 jit_int32_t reg;
232 assert(_jitc->function);
233 jit_inc_synth_ww(allocar, u, v);
234 if (!_jitc->function->allocar) {
235 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
236 _jitc->function->allocar = 1;
237 }
238 reg = jit_get_reg(jit_class_gpr);
239 jit_negr(reg, v);
240 jit_andi(reg, reg, -8);
241 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
242 jit_addr(u, u, reg);
243 jit_addr(JIT_SP, JIT_SP, reg);
244 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
245 jit_unget_reg(reg);
246 jit_dec_synth();
247 }
248
249 void
_jit_ret(jit_state_t * _jit)250 _jit_ret(jit_state_t *_jit)
251 {
252 jit_node_t *instr;
253 assert(_jitc->function);
254 jit_inc_synth(ret);
255 /* jump to epilog */
256 instr = jit_jmpi();
257 jit_patch_at(instr, _jitc->function->epilog);
258 jit_dec_synth();
259 }
260
261 void
_jit_retr(jit_state_t * _jit,jit_int32_t u)262 _jit_retr(jit_state_t *_jit, jit_int32_t u)
263 {
264 jit_inc_synth_w(retr, u);
265 if (JIT_RET != u)
266 jit_movr(JIT_RET, u);
267 jit_live(JIT_RET);
268 jit_ret();
269 jit_dec_synth();
270 }
271
272 void
_jit_reti(jit_state_t * _jit,jit_word_t u)273 _jit_reti(jit_state_t *_jit, jit_word_t u)
274 {
275 jit_inc_synth_w(reti, u);
276 jit_movi(JIT_RET, u);
277 jit_ret();
278 jit_dec_synth();
279 }
280
281 void
_jit_retr_f(jit_state_t * _jit,jit_int32_t u)282 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
283 {
284 jit_inc_synth_w(retr_f, u);
285 if (JIT_FRET != u)
286 jit_movr_f(JIT_FRET, u);
287 else
288 jit_live(JIT_FRET);
289 jit_ret();
290 jit_dec_synth();
291 }
292
293 void
_jit_reti_f(jit_state_t * _jit,jit_float32_t u)294 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
295 {
296 jit_inc_synth_f(reti_f, u);
297 jit_movi_f(JIT_FRET, u);
298 jit_ret();
299 jit_dec_synth();
300 }
301
302 void
_jit_retr_d(jit_state_t * _jit,jit_int32_t u)303 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
304 {
305 jit_inc_synth_w(retr_d, u);
306 if (JIT_FRET != u)
307 jit_movr_d(JIT_FRET, u);
308 else
309 jit_live(JIT_FRET);
310 jit_ret();
311 jit_dec_synth();
312 }
313
314 void
_jit_reti_d(jit_state_t * _jit,jit_float64_t u)315 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
316 {
317 jit_inc_synth_d(reti_d, u);
318 jit_movi_d(JIT_FRET, u);
319 jit_ret();
320 jit_dec_synth();
321 }
322
323 void
_jit_epilog(jit_state_t * _jit)324 _jit_epilog(jit_state_t *_jit)
325 {
326 assert(_jitc->function);
327 assert(_jitc->function->epilog->next == NULL);
328 jit_link(_jitc->function->epilog);
329 _jitc->function = NULL;
330 }
331
332 jit_bool_t
_jit_arg_register_p(jit_state_t * _jit,jit_node_t * u)333 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
334 {
335 if (u->code == jit_code_arg)
336 return (jit_arg_reg_p(u->u.w));
337 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
338 #if NEW_ABI
339 return (jit_arg_reg_p(u->u.w));
340 #else
341 return (u->u.w < 8);
342 #endif
343 }
344
345 static jit_node_t *
_jit_make_arg(jit_state_t * _jit,jit_node_t * node)346 _jit_make_arg(jit_state_t *_jit, jit_node_t *node)
347 {
348 jit_int32_t offset;
349 #if NEW_ABI
350 if (jit_arg_reg_p(_jitc->function->self.argi))
351 offset = _jitc->function->self.argi++;
352 else {
353 offset = _jitc->function->self.size;
354 _jitc->function->self.size += STACK_SLOT;
355 }
356 #else
357 offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
358 _jitc->function->self.argi = 1;
359 if (offset >= 4)
360 offset = _jitc->function->self.size;
361 _jitc->function->self.size += STACK_SLOT;
362 #endif
363 if (node == (jit_node_t *)0)
364 node = jit_new_node(jit_code_arg);
365 else
366 link_node(node);
367 node->u.w = offset;
368 node->v.w = ++_jitc->function->self.argn;
369 jit_link_prolog();
370 return (node);
371 }
372
373 static jit_node_t *
_jit_make_arg_f(jit_state_t * _jit,jit_node_t * node)374 _jit_make_arg_f(jit_state_t *_jit, jit_node_t *node)
375 {
376 jit_int32_t offset;
377 #if NEW_ABI
378 if (jit_arg_reg_p(_jitc->function->self.argi)) {
379 offset = _jitc->function->self.argi++;
380 if (_jitc->function->self.call & jit_call_varargs)
381 offset += 8;
382 }
383 else {
384 offset = _jitc->function->self.size;
385 _jitc->function->self.size += STACK_SLOT;
386 }
387 #else
388 offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
389 if (offset < NUM_WORD_ARGS) {
390 if (!_jitc->function->self.argi &&
391 !(_jitc->function->self.call & jit_call_varargs)) {
392 if (offset == 0)
393 offset = 4;
394 else {
395 offset = 6;
396 _jitc->function->self.argi = 1;
397 }
398 /* Use as flag to rewind in case of varargs function */
399 ++_jitc->function->self.argf;
400 }
401 }
402 else
403 offset = _jitc->function->self.size;
404 _jitc->function->self.size += STACK_SLOT;
405 #endif
406 if (node == (jit_node_t *)0)
407 node = jit_new_node(jit_code_arg_f);
408 else
409 link_node(node);
410 node->u.w = offset;
411 node->v.w = ++_jitc->function->self.argn;
412 jit_link_prolog();
413 return (node);
414 }
415
416 static jit_node_t *
_jit_make_arg_d(jit_state_t * _jit,jit_node_t * node)417 _jit_make_arg_d(jit_state_t *_jit, jit_node_t *node)
418 {
419 jit_int32_t offset;
420 #if NEW_ABI
421 if (jit_arg_reg_p(_jitc->function->self.argi)) {
422 offset = _jitc->function->self.argi++;
423 if (_jitc->function->self.call & jit_call_varargs)
424 offset += 8;
425 }
426 else {
427 offset = _jitc->function->self.size;
428 _jitc->function->self.size += STACK_SLOT;
429 }
430 #else
431 if (_jitc->function->self.size & 7) {
432 _jitc->function->self.size += 4;
433 _jitc->function->self.argi = 1;
434 }
435 offset = (_jitc->function->self.size - stack_framesize) >> STACK_SHIFT;
436 if (offset < NUM_WORD_ARGS) {
437 if (!_jitc->function->self.argi &&
438 !(_jitc->function->self.call & jit_call_varargs)) {
439 offset += 4;
440 /* Use as flag to rewind in case of varargs function */
441 ++_jitc->function->self.argf;
442 }
443 }
444 else
445 offset = _jitc->function->self.size;
446 _jitc->function->self.size += sizeof(jit_float64_t);
447 #endif
448 if (node == (jit_node_t *)0)
449 node = jit_new_node(jit_code_arg_d);
450 else
451 link_node(node);
452 node->u.w = offset;
453 node->v.w = ++_jitc->function->self.argn;
454 jit_link_prolog();
455 return (node);
456 }
457
458 void
_jit_ellipsis(jit_state_t * _jit)459 _jit_ellipsis(jit_state_t *_jit)
460 {
461 if (_jitc->prepare) {
462 assert(!(_jitc->function->call.call & jit_call_varargs));
463 _jitc->function->call.call |= jit_call_varargs;
464 #if !NEW_ABI
465 if (_jitc->function->call.argf)
466 rewind_prepare();
467 #endif
468 }
469 else {
470 assert(!(_jitc->function->self.call & jit_call_varargs));
471 #if NEW_ABI
472 /* If varargs start in a register, allocate extra 64 bytes. */
473 if (jit_arg_reg_p(_jitc->function->self.argi))
474 rewind_prolog();
475 /* Do not set during possible rewind. */
476 _jitc->function->self.call |= jit_call_varargs;
477 #else
478 _jitc->function->self.call |= jit_call_varargs;
479 if (_jitc->function->self.argf)
480 rewind_prolog();
481 #endif
482 _jitc->function->vagp = _jitc->function->self.argi;
483 }
484 jit_inc_synth(ellipsis);
485 if (_jitc->prepare)
486 jit_link_prepare();
487 else
488 jit_link_prolog();
489 jit_dec_synth();
490 }
491
492 void
_jit_va_push(jit_state_t * _jit,jit_int32_t u)493 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
494 {
495 jit_inc_synth_w(va_push, u);
496 jit_pushargr(u);
497 jit_dec_synth();
498 }
499
500 jit_node_t *
_jit_arg(jit_state_t * _jit)501 _jit_arg(jit_state_t *_jit)
502 {
503 assert(_jitc->function);
504 return (jit_make_arg((jit_node_t*)0));
505 }
506
507 jit_node_t *
_jit_arg_f(jit_state_t * _jit)508 _jit_arg_f(jit_state_t *_jit)
509 {
510 assert(_jitc->function);
511 return (jit_make_arg_f((jit_node_t*)0));
512 }
513
514 jit_node_t *
_jit_arg_d(jit_state_t * _jit)515 _jit_arg_d(jit_state_t *_jit)
516 {
517 assert(_jitc->function);
518 return (jit_make_arg_d((jit_node_t*)0));
519 }
520
521 void
_jit_getarg_c(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)522 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
523 {
524 assert(v->code == jit_code_arg);
525 jit_inc_synth_wp(getarg_c, u, v);
526 if (jit_arg_reg_p(v->u.w))
527 jit_extr_c(u, _A0 - v->u.w);
528 else
529 jit_ldxi_c(u, _FP, v->u.w + C_DISP);
530 jit_dec_synth();
531 }
532
533 void
_jit_getarg_uc(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)534 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
535 {
536 assert(v->code == jit_code_arg);
537 jit_inc_synth_wp(getarg_uc, u, v);
538 if (jit_arg_reg_p(v->u.w))
539 jit_extr_uc(u, _A0 - v->u.w);
540 else
541 jit_ldxi_uc(u, _FP, v->u.w + C_DISP);
542 jit_dec_synth();
543 }
544
545 void
_jit_getarg_s(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)546 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
547 {
548 assert(v->code == jit_code_arg);
549 jit_inc_synth_wp(getarg_s, u, v);
550 if (jit_arg_reg_p(v->u.w))
551 jit_extr_s(u, _A0 - v->u.w);
552 else
553 jit_ldxi_s(u, _FP, v->u.w + S_DISP);
554 jit_dec_synth();
555 }
556
557 void
_jit_getarg_us(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)558 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
559 {
560 assert(v->code == jit_code_arg);
561 jit_inc_synth_wp(getarg_us, u, v);
562 if (jit_arg_reg_p(v->u.w))
563 jit_extr_us(u, _A0 - v->u.w);
564 else
565 jit_ldxi_us(u, _FP, v->u.w + S_DISP);
566 jit_dec_synth();
567 }
568
569 void
_jit_getarg_i(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)570 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
571 {
572 assert(v->code == jit_code_arg);
573 jit_inc_synth_wp(getarg_i, u, v);
574 if (jit_arg_reg_p(v->u.w)) {
575 #if __WORDSIZE == 64
576 jit_extr_i(u, _A0 - v->u.w);
577 #else
578 jit_movr(u, _A0 - v->u.w);
579 #endif
580 }
581 else
582 jit_ldxi_i(u, _FP, v->u.w + I_DISP);
583 jit_dec_synth();
584 }
585
586 #if __WORDSIZE == 64
587 void
_jit_getarg_ui(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)588 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
589 {
590 assert(v->code == jit_code_arg);
591 jit_inc_synth_wp(getarg_ui, u, v);
592 if (jit_arg_reg_p(v->u.w))
593 jit_extr_ui(u, _A0 - v->u.w);
594 else
595 jit_ldxi_ui(u, _FP, v->u.w + I_DISP);
596 jit_dec_synth();
597 }
598
599 void
_jit_getarg_l(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)600 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
601 {
602 assert(v->code == jit_code_arg);
603 jit_inc_synth_wp(getarg_l, u, v);
604 if (jit_arg_reg_p(v->u.w))
605 jit_movr(u, _A0 - v->u.w);
606 else
607 jit_ldxi_l(u, _FP, v->u.w);
608 jit_dec_synth();
609 }
610 #endif
611
612 void
_jit_putargr(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)613 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
614 {
615 jit_inc_synth_wp(putargr, u, v);
616 assert(v->code == jit_code_arg);
617 if (jit_arg_reg_p(v->u.w))
618 jit_movr(_A0 - v->u.w, u);
619 else
620 jit_stxi(v->u.w + WORD_ADJUST, _FP, u);
621 jit_dec_synth();
622 }
623
624 void
_jit_putargi(jit_state_t * _jit,jit_word_t u,jit_node_t * v)625 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
626 {
627 jit_int32_t regno;
628 assert(v->code == jit_code_arg);
629 jit_inc_synth_wp(putargi, u, v);
630 if (jit_arg_reg_p(v->u.w))
631 jit_movi(_A0 - v->u.w, u);
632 else {
633 regno = jit_get_reg(jit_class_gpr);
634 jit_movi(regno, u);
635 jit_stxi(v->u.w + WORD_ADJUST, _FP, regno);
636 jit_unget_reg(regno);
637 }
638 jit_dec_synth();
639 }
640
641 void
_jit_getarg_f(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)642 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
643 {
644 assert(v->code == jit_code_arg_f);
645 jit_inc_synth_wp(getarg_f, u, v);
646 #if NEW_ABI
647 if (jit_arg_reg_p(v->u.w))
648 jit_movr_f(u, _F12 - v->u.w);
649 else if (jit_arg_reg_p(v->u.w - 8))
650 jit_movr_w_f(u, _A0 - v->u.w - 8);
651 #else
652 if (v->u.w < 4)
653 jit_movr_w_f(u, _A0 - v->u.w);
654 else if (v->u.w < 8)
655 jit_movr_f(u, _F12 - ((v->u.w - 4) >> 1));
656 #endif
657 else
658 jit_ldxi_f(u, _FP, v->u.w);
659 jit_dec_synth();
660 }
661
662 void
_jit_putargr_f(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)663 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
664 {
665 assert(v->code == jit_code_arg_f);
666 jit_inc_synth_wp(putargr_f, u, v);
667 #if NEW_ABI
668 if (jit_arg_reg_p(v->u.w))
669 jit_movr_f(_F12 - v->u.w, u);
670 else if (jit_arg_reg_p(v->u.w - 8))
671 jit_movr_f_w(_A0 - v->u.w - 8, u);
672 #else
673 if (v->u.w < 4)
674 jit_movr_f_w(_A0 - v->u.w, u);
675 else if (v->u.w < 8)
676 jit_movr_f(_F12 - ((v->u.w - 4) >> 1), u);
677 #endif
678 else
679 jit_stxi_f(v->u.w, _FP, u);
680 jit_dec_synth();
681 }
682
683 void
_jit_putargi_f(jit_state_t * _jit,jit_float32_t u,jit_node_t * v)684 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
685 {
686 jit_int32_t regno;
687 assert(v->code == jit_code_arg_f);
688 jit_inc_synth_fp(putargi_f, u, v);
689 #if NEW_ABI
690 if (jit_arg_reg_p(v->u.w))
691 jit_movi_f(_F12 - v->u.w, u);
692 else if (jit_arg_reg_p(v->u.w - 8)) {
693 regno = jit_get_reg(jit_class_fpr);
694 jit_movi_f(regno, u);
695 jit_movr_f_w(_A0 - v->u.w - 8, u);
696 jit_unget_reg(regno);
697 }
698 #else
699 if (v->u.w < 4) {
700 regno = jit_get_reg(jit_class_fpr);
701 jit_movi_f(regno, u);
702 jit_movr_f_w(_A0 - ((v->u.w - 4) >> 1), regno);
703 jit_unget_reg(regno);
704 }
705 else if (v->u.w < 8)
706 jit_movi_f(_F12 - ((v->u.w - 4) >> 1), u);
707 #endif
708 else {
709 regno = jit_get_reg(jit_class_fpr);
710 jit_movi_f(regno, u);
711 jit_stxi_f(v->u.w, _FP, regno);
712 jit_unget_reg(regno);
713 }
714 jit_dec_synth();
715 }
716
717 void
_jit_getarg_d(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)718 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
719 {
720 assert(v->code == jit_code_arg_d);
721 jit_inc_synth_wp(getarg_d, u, v);
722 #if NEW_ABI
723 if (jit_arg_reg_p(v->u.w))
724 jit_movr_d(u, _F12 - v->u.w);
725 else if (jit_arg_reg_p(v->u.w - 8))
726 jit_movr_d_w(_A0 - v->u.w - 8, u);
727 #else
728 if (v->u.w < 4)
729 jit_movr_ww_d(u, _A0 - v->u.w, _A0 - (v->u.w + 1));
730 else if (v->u.w < 8)
731 jit_movr_d(u, _F12 - ((v->u.w - 4) >> 1));
732 #endif
733 else
734 jit_ldxi_d(u, _FP, v->u.w);
735 jit_dec_synth();
736 }
737
738 void
_jit_putargr_d(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)739 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
740 {
741 assert(v->code == jit_code_arg_d);
742 jit_inc_synth_wp(putargr_d, u, v);
743 #if NEW_ABI
744 if (jit_arg_reg_p(v->u.w))
745 jit_movr_d(_F12 - v->u.w, u);
746 else if (jit_arg_reg_p(v->u.w - 8))
747 jit_movr_d_w(_A0 - v->u.w - 8, u);
748 #else
749 if (v->u.w < 4)
750 jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), u);
751 else if (v->u.w < 8)
752 jit_movr_d(_F12 - ((v->u.w - 4) >> 1), u);
753 #endif
754 else
755 jit_stxi_d(v->u.w, _FP, u);
756 jit_dec_synth();
757 }
758
759 void
_jit_putargi_d(jit_state_t * _jit,jit_float64_t u,jit_node_t * v)760 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
761 {
762 jit_int32_t regno;
763 assert(v->code == jit_code_arg_d);
764 jit_inc_synth_dp(putargi_d, u, v);
765 #if NEW_ABI
766 if (jit_arg_reg_p(v->u.w))
767 jit_movi_d(_F12 - v->u.w, u);
768 else if (jit_arg_reg_p(v->u.w - 8)) {
769 regno = jit_get_reg(jit_class_fpr);
770 jit_movi_d(regno, u);
771 jit_movr_d_w(_A0 - v->u.w - 8, u);
772 jit_unget_reg(regno);
773 }
774 #else
775 if (v->u.w < 4) {
776 regno = jit_get_reg(jit_class_fpr);
777 jit_movi_d(regno, u);
778 jit_movr_d_ww(_A0 - v->u.w, _A0 - (v->u.w + 1), regno);
779 jit_unget_reg(regno);
780 }
781 else if (v->u.w < 8)
782 jit_movi_d(_F12 - ((v->u.w - 4) >> 1), u);
783 #endif
784 else {
785 regno = jit_get_reg(jit_class_fpr);
786 jit_movi_d(regno, u);
787 jit_stxi_d(v->u.w, _FP, regno);
788 jit_unget_reg(regno);
789 }
790 jit_dec_synth();
791 }
792
793 void
_jit_pushargr(jit_state_t * _jit,jit_int32_t u)794 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
795 {
796 jit_inc_synth_w(pushargr, u);
797 jit_link_prepare();
798 #if NEW_ABI
799 assert(_jitc->function);
800 if (jit_arg_reg_p(_jitc->function->call.argi)) {
801 jit_movr(_A0 - _jitc->function->call.argi, u);
802 ++_jitc->function->call.argi;
803 }
804 else {
805 jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, u);
806 _jitc->function->call.size += STACK_SLOT;
807 }
808 #else
809 jit_word_t offset;
810 assert(_jitc->function);
811 offset = _jitc->function->call.size >> STACK_SHIFT;
812 _jitc->function->call.argi = 1;
813 if (jit_arg_reg_p(offset))
814 jit_movr(_A0 - offset, u);
815 else
816 jit_stxi(_jitc->function->call.size, JIT_SP, u);
817 _jitc->function->call.size += STACK_SLOT;
818 #endif
819 jit_dec_synth();
820 }
821
822 void
_jit_pushargi(jit_state_t * _jit,jit_word_t u)823 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
824 {
825 jit_int32_t regno;
826 #if !NEW_ABI
827 jit_word_t offset;
828 #endif
829 assert(_jitc->function);
830 jit_inc_synth_w(pushargi, u);
831 jit_link_prepare();
832 #if NEW_ABI
833 if (jit_arg_reg_p(_jitc->function->call.argi)) {
834 jit_movi(_A0 - _jitc->function->call.argi, u);
835 ++_jitc->function->call.argi;
836 }
837 else {
838 regno = jit_get_reg(jit_class_gpr);
839 jit_movi(regno, u);
840 jit_stxi(_jitc->function->call.size + WORD_ADJUST, JIT_SP, regno);
841 _jitc->function->call.size += STACK_SLOT;
842 jit_unget_reg(regno);
843 }
844 #else
845 offset = _jitc->function->call.size >> STACK_SHIFT;
846 ++_jitc->function->call.argi;
847 if (jit_arg_reg_p(offset))
848 jit_movi(_A0 - offset, u);
849 else {
850 regno = jit_get_reg(jit_class_gpr);
851 jit_movi(regno, u);
852 jit_stxi(_jitc->function->call.size, JIT_SP, regno);
853 jit_unget_reg(regno);
854 }
855 _jitc->function->call.size += STACK_SLOT;
856 #endif
857 jit_dec_synth();
858 }
859
860 void
_jit_pushargr_f(jit_state_t * _jit,jit_int32_t u)861 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
862 {
863 #if !NEW_ABI
864 jit_word_t offset;
865 #endif
866 assert(_jitc->function);
867 jit_inc_synth_w(pushargr_f, u);
868 jit_link_prepare();
869 #if NEW_ABI
870 if (jit_arg_reg_p(_jitc->function->call.argi)) {
871 if (!(_jitc->function->call.call & jit_call_varargs))
872 jit_movr_f(_F12 - _jitc->function->call.argi, u);
873 else
874 jit_movr_f_w(_A0 - _jitc->function->call.argi, u);
875 ++_jitc->function->call.argi;
876 }
877 else {
878 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
879 _jitc->function->call.size += STACK_SLOT;
880 }
881 #else
882 offset = _jitc->function->call.size >> STACK_SHIFT;
883 if (offset < 2 && !_jitc->function->call.argi &&
884 !(_jitc->function->call.call & jit_call_varargs)) {
885 ++_jitc->function->call.argf;
886 jit_movr_f(_F12 - offset, u);
887 }
888 else if (offset < 4) {
889 ++_jitc->function->call.argi;
890 jit_movr_f_w(_A0 - offset, u);
891 }
892 else
893 jit_stxi_f(_jitc->function->call.size, JIT_SP, u);
894 _jitc->function->call.size += STACK_SLOT;
895 #endif
896 jit_dec_synth();
897 }
898
899 void
_jit_pushargi_f(jit_state_t * _jit,jit_float32_t u)900 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
901 {
902 jit_int32_t regno;
903 #if !NEW_ABI
904 jit_word_t offset;
905 #endif
906 assert(_jitc->function);
907 jit_inc_synth_f(pushargi_f, u);
908 jit_link_prepare();
909 #if NEW_ABI
910 if (jit_arg_reg_p(_jitc->function->call.argi)) {
911 if (!(_jitc->function->call.call & jit_call_varargs))
912 jit_movi_f(_F12 - _jitc->function->call.argi, u);
913 else
914 jit_movi_f_w(_A0 - _jitc->function->call.argi, u);
915 ++_jitc->function->call.argi;
916 }
917 else {
918 regno = jit_get_reg(jit_class_fpr);
919 jit_movi_f(regno, u);
920 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
921 _jitc->function->call.size += STACK_SLOT;
922 jit_unget_reg(regno);
923 }
924 #else
925 offset = _jitc->function->call.size >> STACK_SHIFT;
926 if (offset < 2 && !_jitc->function->call.argi &&
927 !(_jitc->function->call.call & jit_call_varargs)) {
928 ++_jitc->function->call.argf;
929 jit_movi_f(_F12 - offset, u);
930 }
931 else if (offset < 4) {
932 ++_jitc->function->call.argi;
933 jit_movi_f_w(_A0 - offset, u);
934 }
935 else {
936 regno = jit_get_reg(jit_class_fpr);
937 jit_movi_f(regno, u);
938 jit_stxi_f(_jitc->function->call.size, JIT_SP, regno);
939 jit_unget_reg(regno);
940 }
941 _jitc->function->call.size += STACK_SLOT;
942 #endif
943 jit_dec_synth();
944 }
945
946 void
_jit_pushargr_d(jit_state_t * _jit,jit_int32_t u)947 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
948 {
949 #if !NEW_ABI
950 jit_bool_t adjust;
951 jit_word_t offset;
952 #endif
953 assert(_jitc->function);
954 jit_inc_synth_w(pushargr_d, u);
955 jit_link_prepare();
956 #if NEW_ABI
957 if (jit_arg_reg_p(_jitc->function->call.argi)) {
958 if (!(_jitc->function->call.call & jit_call_varargs))
959 jit_movr_d(_F12 - _jitc->function->call.argi, u);
960 else
961 jit_movr_d_w(_A0 - _jitc->function->call.argi, u);
962 ++_jitc->function->call.argi;
963 }
964 else {
965 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
966 _jitc->function->call.size += STACK_SLOT;
967 }
968 #else
969 adjust = !!_jitc->function->call.argi;
970 if (_jitc->function->call.size & 7) {
971 _jitc->function->call.size += 4;
972 adjust = 1;
973 }
974 offset = _jitc->function->call.size >> STACK_SHIFT;
975 if (offset < 3) {
976 if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
977 jit_movr_d_ww(_A0 - offset, _A0 - (offset + 1), u);
978 _jitc->function->call.argi += 2;
979 }
980 else {
981 jit_movr_d(_F12 - (offset >> 1), u);
982 ++_jitc->function->call.argf;
983 }
984 }
985 else
986 jit_stxi_d(_jitc->function->call.size, JIT_SP, u);
987 _jitc->function->call.size += sizeof(jit_float64_t);
988 #endif
989 jit_dec_synth();
990 }
991
992 void
_jit_pushargi_d(jit_state_t * _jit,jit_float64_t u)993 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
994 {
995 jit_int32_t regno;
996 #if !NEW_ABI
997 jit_bool_t adjust;
998 jit_word_t offset;
999 #endif
1000 assert(_jitc->function);
1001 jit_inc_synth_d(pushargi_d, u);
1002 jit_link_prepare();
1003 #if NEW_ABI
1004 if (jit_arg_reg_p(_jitc->function->call.argi)) {
1005 if (!(_jitc->function->call.call & jit_call_varargs))
1006 jit_movi_d(_F12 - _jitc->function->call.argi, u);
1007 else
1008 jit_movi_d_w(_A0 - _jitc->function->call.argi, u);
1009 ++_jitc->function->call.argi;
1010 }
1011 else {
1012 regno = jit_get_reg(jit_class_fpr);
1013 jit_movi_d(regno, u);
1014 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1015 _jitc->function->call.size += STACK_SLOT;
1016 jit_unget_reg(regno);
1017 }
1018 #else
1019 adjust = !!_jitc->function->call.argi;
1020 if (_jitc->function->call.size & 7) {
1021 _jitc->function->call.size += 4;
1022 adjust = 1;
1023 }
1024 offset = _jitc->function->call.size >> STACK_SHIFT;
1025 if (offset < 3) {
1026 if (adjust || (_jitc->function->call.call & jit_call_varargs)) {
1027 jit_movi_d_ww(_A0 - offset, _A0 - (offset + 1), u);
1028 _jitc->function->call.argi += 2;
1029 }
1030 else {
1031 jit_movi_d(_F12 - (offset >> 1), u);
1032 ++_jitc->function->call.argf;
1033 }
1034 }
1035 else {
1036 regno = jit_get_reg(jit_class_fpr);
1037 jit_movi_d(regno, u);
1038 jit_stxi_d(_jitc->function->call.size, JIT_SP, regno);
1039 jit_unget_reg(regno);
1040 }
1041 _jitc->function->call.size += sizeof(jit_float64_t);
1042 #endif
1043 jit_dec_synth();
1044 }
1045
1046 jit_bool_t
_jit_regarg_p(jit_state_t * _jit,jit_node_t * node,jit_int32_t regno)1047 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
1048 {
1049 jit_int32_t spec;
1050
1051 spec = jit_class(_rvs[regno].spec);
1052 if (spec & jit_class_arg) {
1053 if (spec & jit_class_gpr) {
1054 regno = _A0 - regno;
1055 if (regno >= 0 && regno < node->v.w)
1056 return (1);
1057 }
1058 else if (spec & jit_class_fpr) {
1059 regno = _F12 - regno;
1060 if (regno >= 0 && regno < node->w.w)
1061 return (1);
1062 }
1063 }
1064
1065 return (0);
1066 }
1067
1068 void
_jit_finishr(jit_state_t * _jit,jit_int32_t r0)1069 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
1070 {
1071 jit_node_t *call;
1072 assert(_jitc->function);
1073 jit_inc_synth_w(finishr, r0);
1074 if (_jitc->function->self.alen < _jitc->function->call.size)
1075 _jitc->function->self.alen = _jitc->function->call.size;
1076 jit_movr(_T9, r0);
1077 call = jit_callr(_T9);
1078 call->v.w = _jitc->function->self.argi;
1079 #if NEW_ABI
1080 call->w.w = call->v.w;
1081 #else
1082 call->w.w = _jitc->function->self.argf;
1083 #endif
1084 _jitc->function->call.argi = _jitc->function->call.argf =
1085 _jitc->function->call.size = 0;
1086 _jitc->prepare = 0;
1087 jit_dec_synth();
1088 }
1089
1090 jit_node_t *
_jit_finishi(jit_state_t * _jit,jit_pointer_t i0)1091 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
1092 {
1093 jit_node_t *call;
1094 jit_node_t *node;
1095 assert(_jitc->function);
1096 jit_inc_synth_w(finishi, (jit_word_t)i0);
1097 if (_jitc->function->self.alen < _jitc->function->call.size)
1098 _jitc->function->self.alen = _jitc->function->call.size;
1099 node = jit_movi(_T9, (jit_word_t)i0);
1100 call = jit_callr(_T9);
1101 call->v.w = _jitc->function->call.argi;
1102 #if NEW_ABI
1103 call->w.w = call->v.w;
1104 #else
1105 call->w.w = _jitc->function->call.argf;
1106 #endif
1107 _jitc->function->call.argi = _jitc->function->call.argf =
1108 _jitc->function->call.size = 0;
1109 _jitc->prepare = 0;
1110 jit_dec_synth();
1111 return (node);
1112 }
1113
1114 void
_jit_retval_c(jit_state_t * _jit,jit_int32_t r0)1115 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
1116 {
1117 jit_extr_c(r0, JIT_RET);
1118 }
1119
1120 void
_jit_retval_uc(jit_state_t * _jit,jit_int32_t r0)1121 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
1122 {
1123 jit_extr_uc(r0, JIT_RET);
1124 }
1125
1126 void
_jit_retval_s(jit_state_t * _jit,jit_int32_t r0)1127 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
1128 {
1129 jit_extr_s(r0, JIT_RET);
1130 }
1131
1132 void
_jit_retval_us(jit_state_t * _jit,jit_int32_t r0)1133 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
1134 {
1135 jit_extr_us(r0, JIT_RET);
1136 }
1137
1138 void
_jit_retval_i(jit_state_t * _jit,jit_int32_t r0)1139 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
1140 {
1141 #if __WORDSIZE == 32
1142 if (r0 != JIT_RET)
1143 jit_movr(r0, JIT_RET);
1144 #else
1145 jit_extr_i(r0, JIT_RET);
1146 #endif
1147 }
1148
1149 #if __WORDSIZE == 64
1150 void
_jit_retval_ui(jit_state_t * _jit,jit_int32_t r0)1151 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
1152 {
1153 jit_extr_ui(r0, JIT_RET);
1154 }
1155
1156 void
_jit_retval_l(jit_state_t * _jit,jit_int32_t r0)1157 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
1158 {
1159 if (r0 != JIT_RET)
1160 jit_movr(r0, JIT_RET);
1161 }
1162 #endif
1163
1164 void
_jit_retval_f(jit_state_t * _jit,jit_int32_t r0)1165 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
1166 {
1167 if (r0 != JIT_FRET)
1168 jit_movr_f(r0, JIT_FRET);
1169 }
1170
1171 void
_jit_retval_d(jit_state_t * _jit,jit_int32_t r0)1172 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
1173 {
1174 if (r0 != JIT_FRET)
1175 jit_movr_d(r0, JIT_FRET);
1176 }
1177
1178 jit_pointer_t
_emit_code(jit_state_t * _jit)1179 _emit_code(jit_state_t *_jit)
1180 {
1181 jit_node_t *node;
1182 jit_node_t *temp;
1183 jit_word_t word;
1184 jit_int32_t value;
1185 jit_int32_t offset;
1186 struct {
1187 jit_node_t *node;
1188 jit_word_t word;
1189 #if DEVEL_DISASSEMBLER
1190 jit_word_t prevw;
1191 #endif
1192 jit_int32_t patch_offset;
1193 } undo;
1194 #if DEVEL_DISASSEMBLER
1195 jit_word_t prevw;
1196 #endif
1197
1198 _jitc->function = NULL;
1199
1200 jit_reglive_setup();
1201
1202 undo.word = 0;
1203 undo.node = NULL;
1204 undo.patch_offset = 0;
1205 #define case_rr(name, type) \
1206 case jit_code_##name##r##type: \
1207 name##r##type(rn(node->u.w), rn(node->v.w)); \
1208 break
1209 #define case_rw(name, type) \
1210 case jit_code_##name##i##type: \
1211 name##i##type(rn(node->u.w), node->v.w); \
1212 break
1213 #define case_wr(name, type) \
1214 case jit_code_##name##i##type: \
1215 name##i##type(node->u.w, rn(node->v.w)); \
1216 break
1217 #define case_rrr(name, type) \
1218 case jit_code_##name##r##type: \
1219 name##r##type(rn(node->u.w), \
1220 rn(node->v.w), rn(node->w.w)); \
1221 break
1222 #define case_rrw(name, type) \
1223 case jit_code_##name##i##type: \
1224 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
1225 break
1226 #define case_rrrr(name, type) \
1227 case jit_code_##name##r##type: \
1228 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
1229 rn(node->v.w), rn(node->w.w)); \
1230 break
1231 #define case_rrrw(name, type) \
1232 case jit_code_##name##i##type: \
1233 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
1234 rn(node->v.w), node->w.w); \
1235 break
1236 #define case_rrf(name, type, size) \
1237 case jit_code_##name##i##type: \
1238 assert(node->flag & jit_flag_data); \
1239 name##i##type(rn(node->u.w), rn(node->v.w), \
1240 (jit_float##size##_t *)node->w.n->u.w); \
1241 break
1242 #define case_wrr(name, type) \
1243 case jit_code_##name##i##type: \
1244 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
1245 break
1246 #define case_brr(name, type) \
1247 case jit_code_##name##r##type: \
1248 temp = node->u.n; \
1249 assert(temp->code == jit_code_label || \
1250 temp->code == jit_code_epilog); \
1251 if (temp->flag & jit_flag_patch) \
1252 name##r##type(temp->u.w, rn(node->v.w), \
1253 rn(node->w.w)); \
1254 else { \
1255 word = name##r##type(_jit->pc.w, \
1256 rn(node->v.w), rn(node->w.w)); \
1257 patch(word, node); \
1258 } \
1259 break
1260 #define case_brw(name, type) \
1261 case jit_code_##name##i##type: \
1262 temp = node->u.n; \
1263 assert(temp->code == jit_code_label || \
1264 temp->code == jit_code_epilog); \
1265 if (temp->flag & jit_flag_patch) \
1266 name##i##type(temp->u.w, \
1267 rn(node->v.w), node->w.w); \
1268 else { \
1269 word = name##i##type(_jit->pc.w, \
1270 rn(node->v.w), node->w.w); \
1271 patch(word, node); \
1272 } \
1273 break
1274 #define case_brf(name, type, size) \
1275 case jit_code_##name##i##type: \
1276 temp = node->u.n; \
1277 assert(temp->code == jit_code_label || \
1278 temp->code == jit_code_epilog); \
1279 if (temp->flag & jit_flag_patch) \
1280 name##i##type(temp->u.w, rn(node->v.w), \
1281 (jit_float##size##_t *)node->w.n->u.w); \
1282 else { \
1283 word = name##i##type(_jit->pc.w, rn(node->v.w), \
1284 (jit_float##size##_t *)node->w.n->u.w); \
1285 patch(word, node); \
1286 } \
1287 break
1288 #if DEVEL_DISASSEMBLER
1289 prevw = _jit->pc.w;
1290 #endif
1291 for (node = _jitc->head; node; node = node->next) {
1292 if (_jit->pc.uc >= _jitc->code.end)
1293 return (NULL);
1294
1295 #if DEVEL_DISASSEMBLER
1296 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1297 prevw = _jit->pc.w;
1298 #endif
1299 value = jit_classify(node->code);
1300 jit_regarg_set(node, value);
1301 switch (node->code) {
1302 case jit_code_align:
1303 assert(!(node->u.w & (node->u.w - 1)) &&
1304 node->u.w <= sizeof(jit_word_t));
1305 if (node->u.w == sizeof(jit_word_t) &&
1306 (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1307 nop(sizeof(jit_word_t) - word);
1308 break;
1309 case jit_code_note: case jit_code_name:
1310 node->u.w = _jit->pc.w;
1311 break;
1312 case jit_code_label:
1313 /* remember label is defined */
1314 node->flag |= jit_flag_patch;
1315 node->u.w = _jit->pc.w;
1316 break;
1317 case_rrr(add,);
1318 case_rrw(add,);
1319 case_rrr(addc,);
1320 case_rrw(addc,);
1321 case_rrr(addx,);
1322 case_rrw(addx,);
1323 case_rrr(sub,);
1324 case_rrw(sub,);
1325 case_rrr(subc,);
1326 case_rrw(subc,);
1327 case_rrr(subx,);
1328 case_rrw(subx,);
1329 case_rrw(rsb,);
1330 case_rrr(mul,);
1331 case_rrw(mul,);
1332 case_rrrr(qmul,);
1333 case_rrrw(qmul,);
1334 case_rrrr(qmul, _u);
1335 case_rrrw(qmul, _u);
1336 case_rrr(div,);
1337 case_rrw(div,);
1338 case_rrr(div, _u);
1339 case_rrw(div, _u);
1340 case_rrrr(qdiv,);
1341 case_rrrw(qdiv,);
1342 case_rrrr(qdiv, _u);
1343 case_rrrw(qdiv, _u);
1344 case_rrr(rem,);
1345 case_rrw(rem,);
1346 case_rrr(rem, _u);
1347 case_rrw(rem, _u);
1348 case_rrr(lsh,);
1349 case_rrw(lsh,);
1350 case_rrr(rsh,);
1351 case_rrw(rsh,);
1352 case_rrr(rsh, _u);
1353 case_rrw(rsh, _u);
1354 case_rrr(and,);
1355 case_rrw(and,);
1356 case_rrr(or,);
1357 case_rrw(or,);
1358 case_rrr(xor,);
1359 case_rrw(xor,);
1360 case_rr(trunc, _f_i);
1361 case_rr(trunc, _d_i);
1362 #if __WORDSIZE == 64
1363 case_rr(trunc, _f_l);
1364 case_rr(trunc, _d_l);
1365 #endif
1366 case_rr(ld, _c);
1367 case_rw(ld, _c);
1368 case_rr(ld, _uc);
1369 case_rw(ld, _uc);
1370 case_rr(ld, _s);
1371 case_rw(ld, _s);
1372 case_rr(ld, _us);
1373 case_rw(ld, _us);
1374 case_rr(ld, _i);
1375 case_rw(ld, _i);
1376 #if __WORDSIZE == 64
1377 case_rr(ld, _ui);
1378 case_rw(ld, _ui);
1379 case_rr(ld, _l);
1380 case_rw(ld, _l);
1381 #endif
1382 case_rrr(ldx, _c);
1383 case_rrw(ldx, _c);
1384 case_rrr(ldx, _uc);
1385 case_rrw(ldx, _uc);
1386 case_rrr(ldx, _s);
1387 case_rrw(ldx, _s);
1388 case_rrr(ldx, _us);
1389 case_rrw(ldx, _us);
1390 case_rrr(ldx, _i);
1391 case_rrw(ldx, _i);
1392 #if __WORDSIZE == 64
1393 case_rrr(ldx, _ui);
1394 case_rrw(ldx, _ui);
1395 case_rrr(ldx, _l);
1396 case_rrw(ldx, _l);
1397 #endif
1398 case_rr(st, _c);
1399 case_wr(st, _c);
1400 case_rr(st, _s);
1401 case_wr(st, _s);
1402 case_rr(st, _i);
1403 case_wr(st, _i);
1404 #if __WORDSIZE == 64
1405 case_rr(st, _l);
1406 case_wr(st, _l);
1407 #endif
1408 case_rrr(stx, _c);
1409 case_wrr(stx, _c);
1410 case_rrr(stx, _s);
1411 case_wrr(stx, _s);
1412 case_rrr(stx, _i);
1413 case_wrr(stx, _i);
1414 #if __WORDSIZE == 64
1415 case_rrr(stx, _l);
1416 case_wrr(stx, _l);
1417 #endif
1418 case_rr(hton, _us);
1419 case_rr(hton, _ui);
1420 #if __WORDSIZE == 64
1421 case_rr(hton, _ul);
1422 #endif
1423 case_rr(ext, _c);
1424 case_rr(ext, _uc);
1425 case_rr(ext, _s);
1426 case_rr(ext, _us);
1427 #if __WORDSIZE == 64
1428 case_rr(ext, _i);
1429 case_rr(ext, _ui);
1430 #endif
1431 case_rr(mov,);
1432 case jit_code_movi:
1433 if (node->flag & jit_flag_node) {
1434 temp = node->v.n;
1435 if (temp->code == jit_code_data ||
1436 (temp->code == jit_code_label &&
1437 (temp->flag & jit_flag_patch)))
1438 movi(rn(node->u.w), temp->u.w);
1439 else {
1440 assert(temp->code == jit_code_label ||
1441 temp->code == jit_code_epilog);
1442 word = movi_p(rn(node->u.w), node->v.w);
1443 patch(word, node);
1444 }
1445 }
1446 else
1447 movi(rn(node->u.w), node->v.w);
1448 break;
1449 case_rr(neg,);
1450 case_rr(com,);
1451 case_rrr(lt,);
1452 case_rrw(lt,);
1453 case_rrr(lt, _u);
1454 case_rrw(lt, _u);
1455 case_rrr(le,);
1456 case_rrw(le,);
1457 case_rrr(le, _u);
1458 case_rrw(le, _u);
1459 case_rrr(eq,);
1460 case_rrw(eq,);
1461 case_rrr(ge,);
1462 case_rrw(ge,);
1463 case_rrr(ge, _u);
1464 case_rrw(ge, _u);
1465 case_rrr(gt,);
1466 case_rrw(gt,);
1467 case_rrr(gt, _u);
1468 case_rrw(gt, _u);
1469 case_rrr(ne,);
1470 case_rrw(ne,);
1471 case_brr(blt,);
1472 case_brw(blt,);
1473 case_brr(blt, _u);
1474 case_brw(blt, _u);
1475 case_brr(ble,);
1476 case_brw(ble,);
1477 case_brr(ble, _u);
1478 case_brw(ble, _u);
1479 case_brr(beq,);
1480 case_brw(beq,);
1481 case_brr(bge,);
1482 case_brw(bge,);
1483 case_brr(bge, _u);
1484 case_brw(bge, _u);
1485 case_brr(bgt,);
1486 case_brw(bgt,);
1487 case_brr(bgt, _u);
1488 case_brw(bgt, _u);
1489 case_brr(bne,);
1490 case_brw(bne,);
1491 case_brr(boadd,);
1492 case_brw(boadd,);
1493 case_brr(boadd, _u);
1494 case_brw(boadd, _u);
1495 case_brr(bxadd,);
1496 case_brw(bxadd,);
1497 case_brr(bxadd, _u);
1498 case_brw(bxadd, _u);
1499 case_brr(bosub,);
1500 case_brw(bosub,);
1501 case_brr(bosub, _u);
1502 case_brw(bosub, _u);
1503 case_brr(bxsub,);
1504 case_brw(bxsub,);
1505 case_brr(bxsub, _u);
1506 case_brw(bxsub, _u);
1507 case_brr(bms,);
1508 case_brw(bms,);
1509 case_brr(bmc,);
1510 case_brw(bmc,);
1511 case_rrr(add, _f);
1512 case_rrf(add, _f, 32);
1513 case_rrr(sub, _f);
1514 case_rrf(sub, _f, 32);
1515 case_rrf(rsb, _f, 32);
1516 case_rrr(mul, _f);
1517 case_rrf(mul, _f, 32);
1518 case_rrr(div, _f);
1519 case_rrf(div, _f, 32);
1520 case_rr(abs, _f);
1521 case_rr(neg, _f);
1522 case_rr(sqrt, _f);
1523 case_rr(ext, _f);
1524 case_rr(ld, _f);
1525 case_rw(ld, _f);
1526 case_rrr(ldx, _f);
1527 case_rrw(ldx, _f);
1528 case_rr(st, _f);
1529 case_wr(st, _f);
1530 case_rrr(stx, _f);
1531 case_wrr(stx, _f);
1532 case_rr(mov, _f);
1533 case jit_code_movi_f:
1534 assert(node->flag & jit_flag_data);
1535 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1536 break;
1537 case_rr(ext, _d_f);
1538 case_rrr(lt, _f);
1539 case_rrf(lt, _f, 32);
1540 case_rrr(le, _f);
1541 case_rrf(le, _f, 32);
1542 case_rrr(eq, _f);
1543 case_rrf(eq, _f, 32);
1544 case_rrr(ge, _f);
1545 case_rrf(ge, _f, 32);
1546 case_rrr(gt, _f);
1547 case_rrf(gt, _f, 32);
1548 case_rrr(ne, _f);
1549 case_rrf(ne, _f, 32);
1550 case_rrr(unlt, _f);
1551 case_rrf(unlt, _f, 32);
1552 case_rrr(unle, _f);
1553 case_rrf(unle, _f, 32);
1554 case_rrr(uneq, _f);
1555 case_rrf(uneq, _f, 32);
1556 case_rrr(unge, _f);
1557 case_rrf(unge, _f, 32);
1558 case_rrr(ungt, _f);
1559 case_rrf(ungt, _f, 32);
1560 case_rrr(ltgt, _f);
1561 case_rrf(ltgt, _f, 32);
1562 case_rrr(ord, _f);
1563 case_rrf(ord, _f, 32);
1564 case_rrr(unord, _f);
1565 case_rrf(unord, _f, 32);
1566 case_brr(blt, _f);
1567 case_brf(blt, _f, 32);
1568 case_brr(ble, _f);
1569 case_brf(ble, _f, 32);
1570 case_brr(beq, _f);
1571 case_brf(beq, _f, 32);
1572 case_brr(bge, _f);
1573 case_brf(bge, _f, 32);
1574 case_brr(bgt, _f);
1575 case_brf(bgt, _f, 32);
1576 case_brr(bne, _f);
1577 case_brf(bne, _f, 32);
1578 case_brr(bunlt, _f);
1579 case_brf(bunlt, _f, 32);
1580 case_brr(bunle, _f);
1581 case_brf(bunle, _f, 32);
1582 case_brr(buneq, _f);
1583 case_brf(buneq, _f, 32);
1584 case_brr(bunge, _f);
1585 case_brf(bunge, _f, 32);
1586 case_brr(bungt, _f);
1587 case_brf(bungt, _f, 32);
1588 case_brr(bltgt, _f);
1589 case_brf(bltgt, _f, 32);
1590 case_brr(bord, _f);
1591 case_brf(bord, _f, 32);
1592 case_brr(bunord, _f);
1593 case_brf(bunord, _f, 32);
1594 case_rrr(add, _d);
1595 case_rrf(add, _d, 64);
1596 case_rrr(sub, _d);
1597 case_rrf(sub, _d, 64);
1598 case_rrf(rsb, _d, 64);
1599 case_rrr(mul, _d);
1600 case_rrf(mul, _d, 64);
1601 case_rrr(div, _d);
1602 case_rrf(div, _d, 64);
1603 case_rr(abs, _d);
1604 case_rr(neg, _d);
1605 case_rr(sqrt, _d);
1606 case_rr(ext, _d);
1607 case_rr(ld, _d);
1608 case_rw(ld, _d);
1609 case_rrr(ldx, _d);
1610 case_rrw(ldx, _d);
1611 case_rr(st, _d);
1612 case_wr(st, _d);
1613 case_rrr(stx, _d);
1614 case_wrr(stx, _d);
1615 case_rr(mov, _d);
1616 case jit_code_movi_d:
1617 assert(node->flag & jit_flag_data);
1618 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1619 break;
1620 case_rr(ext, _f_d);
1621 case_rrr(lt, _d);
1622 case_rrf(lt, _d, 64);
1623 case_rrr(le, _d);
1624 case_rrf(le, _d, 64);
1625 case_rrr(eq, _d);
1626 case_rrf(eq, _d, 64);
1627 case_rrr(ge, _d);
1628 case_rrf(ge, _d, 64);
1629 case_rrr(gt, _d);
1630 case_rrf(gt, _d, 64);
1631 case_rrr(ne, _d);
1632 case_rrf(ne, _d, 64);
1633 case_rrr(unlt, _d);
1634 case_rrf(unlt, _d, 64);
1635 case_rrr(unle, _d);
1636 case_rrf(unle, _d, 64);
1637 case_rrr(uneq, _d);
1638 case_rrf(uneq, _d, 64);
1639 case_rrr(unge, _d);
1640 case_rrf(unge, _d, 64);
1641 case_rrr(ungt, _d);
1642 case_rrf(ungt, _d, 64);
1643 case_rrr(ltgt, _d);
1644 case_rrf(ltgt, _d, 64);
1645 case_rrr(ord, _d);
1646 case_rrf(ord, _d, 64);
1647 case_rrr(unord, _d);
1648 case_rrf(unord, _d, 64);
1649 case_brr(blt, _d);
1650 case_brf(blt, _d, 64);
1651 case_brr(ble, _d);
1652 case_brf(ble, _d, 64);
1653 case_brr(beq, _d);
1654 case_brf(beq, _d, 64);
1655 case_brr(bge, _d);
1656 case_brf(bge, _d, 64);
1657 case_brr(bgt, _d);
1658 case_brf(bgt, _d, 64);
1659 case_brr(bne, _d);
1660 case_brf(bne, _d, 64);
1661 case_brr(bunlt, _d);
1662 case_brf(bunlt, _d, 64);
1663 case_brr(bunle, _d);
1664 case_brf(bunle, _d, 64);
1665 case_brr(buneq, _d);
1666 case_brf(buneq, _d, 64);
1667 case_brr(bunge, _d);
1668 case_brf(bunge, _d, 64);
1669 case_brr(bungt, _d);
1670 case_brf(bungt, _d, 64);
1671 case_brr(bltgt, _d);
1672 case_brf(bltgt, _d, 64);
1673 case_brr(bord, _d);
1674 case_brf(bord, _d, 64);
1675 case_brr(bunord, _d);
1676 case_brf(bunord, _d, 64);
1677 case jit_code_jmpr:
1678 jmpr(rn(node->u.w));
1679 break;
1680 case jit_code_jmpi:
1681 if (node->flag & jit_flag_node) {
1682 temp = node->u.n;
1683 assert(temp->code == jit_code_label ||
1684 temp->code == jit_code_epilog);
1685 if (temp->flag & jit_flag_patch)
1686 jmpi(temp->u.w);
1687 else {
1688 word = jmpi(_jit->pc.w);
1689 patch(word, node);
1690 }
1691 }
1692 else
1693 jmpi(node->u.w);
1694 break;
1695 case jit_code_callr:
1696 callr(rn(node->u.w));
1697 break;
1698 case jit_code_calli:
1699 if (node->flag & jit_flag_node) {
1700 temp = node->u.n;
1701 assert(temp->code == jit_code_label ||
1702 temp->code == jit_code_epilog);
1703 word = calli_p(temp->u.w);
1704 if (!(temp->flag & jit_flag_patch))
1705 patch(word, node);
1706 }
1707 else
1708 calli(node->u.w);
1709 break;
1710 case jit_code_prolog:
1711 _jitc->function = _jitc->functions.ptr + node->w.w;
1712 undo.node = node;
1713 undo.word = _jit->pc.w;
1714 #if DEVEL_DISASSEMBLER
1715 undo.prevw = prevw;
1716 #endif
1717 undo.patch_offset = _jitc->patches.offset;
1718 restart_function:
1719 _jitc->again = 0;
1720 prolog(node);
1721 break;
1722 case jit_code_epilog:
1723 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1724 if (_jitc->again) {
1725 for (temp = undo.node->next;
1726 temp != node; temp = temp->next) {
1727 if (temp->code == jit_code_label ||
1728 temp->code == jit_code_epilog)
1729 temp->flag &= ~jit_flag_patch;
1730 }
1731 temp->flag &= ~jit_flag_patch;
1732 node = undo.node;
1733 _jit->pc.w = undo.word;
1734 #if DEVEL_DISASSEMBLER
1735 prevw = undo.prevw;
1736 #endif
1737 _jitc->patches.offset = undo.patch_offset;
1738 goto restart_function;
1739 }
1740 /* remember label is defined */
1741 node->flag |= jit_flag_patch;
1742 node->u.w = _jit->pc.w;
1743 epilog(node);
1744 _jitc->function = NULL;
1745 break;
1746 #if !NEW_ABI
1747 case jit_code_movr_w_f:
1748 movr_w_f(rn(node->u.w), rn(node->v.w));
1749 break;
1750 #endif
1751 case jit_code_movr_f_w:
1752 movr_f_w(rn(node->u.w), rn(node->v.w));
1753 break;
1754 case jit_code_movi_f_w:
1755 assert(node->flag & jit_flag_data);
1756 movi_f_w(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1757 break;
1758 #if NEW_ABI
1759 case jit_code_movr_d_w:
1760 movr_d_w(rn(node->u.w), rn(node->v.w));
1761 break;
1762 case jit_code_movi_d_w:
1763 assert(node->flag & jit_flag_data);
1764 movi_d_w(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1765 break;
1766 #else
1767 case jit_code_movr_ww_d:
1768 movr_ww_d(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1769 break;
1770 case jit_code_movr_d_ww:
1771 movr_d_ww(rn(node->u.w), rn(node->v.w), rn(node->w.w));
1772 break;
1773 case jit_code_movi_d_ww:
1774 assert(node->flag & jit_flag_data);
1775 movi_d_ww(rn(node->u.w), rn(node->v.w),
1776 (jit_float64_t *)node->w.n->u.w);
1777 break;
1778 #endif
1779 case jit_code_va_start:
1780 vastart(rn(node->u.w));
1781 break;
1782 case jit_code_va_arg:
1783 vaarg(rn(node->u.w), rn(node->v.w));
1784 break;
1785 case jit_code_va_arg_d:
1786 vaarg_d(rn(node->u.w), rn(node->v.w));
1787 break;
1788 case jit_code_live:
1789 case jit_code_arg: case jit_code_ellipsis:
1790 case jit_code_va_push:
1791 case jit_code_allocai: case jit_code_allocar:
1792 case jit_code_arg_f: case jit_code_arg_d:
1793 case jit_code_va_end:
1794 case jit_code_ret:
1795 case jit_code_retr: case jit_code_reti:
1796 case jit_code_retr_f: case jit_code_reti_f:
1797 case jit_code_retr_d: case jit_code_reti_d:
1798 case jit_code_getarg_c: case jit_code_getarg_uc:
1799 case jit_code_getarg_s: case jit_code_getarg_us:
1800 case jit_code_getarg_i:
1801 #if __WORDSIZE == 64
1802 case jit_code_getarg_ui: case jit_code_getarg_l:
1803 #endif
1804 case jit_code_getarg_f: case jit_code_getarg_d:
1805 case jit_code_putargr: case jit_code_putargi:
1806 case jit_code_putargr_f: case jit_code_putargi_f:
1807 case jit_code_putargr_d: case jit_code_putargi_d:
1808 case jit_code_pushargr: case jit_code_pushargi:
1809 case jit_code_pushargr_f: case jit_code_pushargi_f:
1810 case jit_code_pushargr_d: case jit_code_pushargi_d:
1811 case jit_code_retval_c: case jit_code_retval_uc:
1812 case jit_code_retval_s: case jit_code_retval_us:
1813 case jit_code_retval_i:
1814 #if __WORDSIZE == 64
1815 case jit_code_retval_ui: case jit_code_retval_l:
1816 #endif
1817 case jit_code_retval_f: case jit_code_retval_d:
1818 case jit_code_prepare:
1819 case jit_code_finishr: case jit_code_finishi:
1820 break;
1821 default:
1822 abort();
1823 }
1824 if (jit_carry != _NOREG) {
1825 switch (node->code) {
1826 case jit_code_note:
1827 case jit_code_addcr: case jit_code_addci:
1828 case jit_code_addxr: case jit_code_addxi:
1829 case jit_code_subcr: case jit_code_subci:
1830 case jit_code_subxr: case jit_code_subxi:
1831 break;
1832 default:
1833 jit_unget_reg(jit_carry);
1834 jit_carry = _NOREG;
1835 break;
1836 }
1837 }
1838 jit_regarg_clr(node, value);
1839 assert(_jitc->regarg == 0 ||
1840 (jit_carry != _NOREG && _jitc->regarg == (1 << jit_carry)));
1841 assert(_jitc->synth == 0);
1842 /* update register live state */
1843 jit_reglive(node);
1844 }
1845 #undef case_brf
1846 #undef case_brw
1847 #undef case_brr
1848 #undef case_wrr
1849 #undef case_rrf
1850 #undef case_rrw
1851 #undef case_rrr
1852 #undef case_wr
1853 #undef case_rw
1854 #undef case_rr
1855
1856 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1857 node = _jitc->patches.ptr[offset].node;
1858 word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1859 patch_at(_jitc->patches.ptr[offset].inst, word);
1860 }
1861
1862 jit_flush(_jit->code.ptr, _jit->pc.uc);
1863
1864 return (_jit->code.ptr);
1865 }
1866
1867 #define CODE 1
1868 # include "jit_rewind.c"
1869 # include "jit_mips-cpu.c"
1870 # include "jit_mips-fpu.c"
1871 #undef CODE
1872
1873 void
jit_flush(void * fptr,void * tptr)1874 jit_flush(void *fptr, void *tptr)
1875 {
1876 #if defined(__linux__)
1877 jit_word_t f, t, s;
1878
1879 s = sysconf(_SC_PAGE_SIZE);
1880 f = (jit_word_t)fptr & -s;
1881 t = (((jit_word_t)tptr) + s - 1) & -s;
1882 _flush_cache((void *)f, t - f, ICACHE);
1883 #endif
1884 }
1885
1886 void
_emit_ldxi(jit_state_t * _jit,jit_int32_t r0,jit_int32_t r1,jit_word_t i0)1887 _emit_ldxi(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1888 {
1889 ldxi(rn(r0), rn(r1), i0);
1890 }
1891
1892 void
_emit_stxi(jit_state_t * _jit,jit_word_t i0,jit_int32_t r0,jit_int32_t r1)1893 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1894 {
1895 stxi(i0, rn(r0), rn(r1));
1896 }
1897
1898 void
_emit_ldxi_d(jit_state_t * _jit,jit_int32_t r0,jit_int32_t r1,jit_word_t i0)1899 _emit_ldxi_d(jit_state_t *_jit, jit_int32_t r0, jit_int32_t r1, jit_word_t i0)
1900 {
1901 ldxi_d(rn(r0), rn(r1), i0);
1902 }
1903
1904 void
_emit_stxi_d(jit_state_t * _jit,jit_word_t i0,jit_int32_t r0,jit_int32_t r1)1905 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_int32_t r0, jit_int32_t r1)
1906 {
1907 stxi_d(i0, rn(r0), rn(r1));
1908 }
1909
1910 static void
_patch(jit_state_t * _jit,jit_word_t instr,jit_node_t * node)1911 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1912 {
1913 jit_int32_t flag;
1914
1915 assert(node->flag & jit_flag_node);
1916 if (node->code == jit_code_movi)
1917 flag = node->v.n->flag;
1918 else
1919 flag = node->u.n->flag;
1920 assert(!(flag & jit_flag_patch));
1921 if (_jitc->patches.offset >= _jitc->patches.length) {
1922 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1923 _jitc->patches.length * sizeof(jit_patch_t),
1924 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1925 _jitc->patches.length += 1024;
1926 }
1927 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1928 _jitc->patches.ptr[_jitc->patches.offset].node = node;
1929 ++_jitc->patches.offset;
1930 }
1931