1 /*
2 * Copyright (C) 2013-2019 Free Software Foundation, Inc.
3 *
4 * This file is part of GNU lightning.
5 *
6 * GNU lightning is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; either version 3, or (at your option)
9 * any later version.
10 *
11 * GNU lightning is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 * License for more details.
15 *
16 * Authors:
17 * Paulo Cesar Pereira de Andrade
18 */
19
20 #include <lightning.h>
21 #include <lightning/jit_private.h>
22
23 #if __WORDSIZE == 32
24 # define NUM_FLOAT_REG_ARGS 2
25 #else
26 # define NUM_FLOAT_REG_ARGS 4
27 #endif
28 #define jit_arg_reg_p(i) ((i) >= 0 && (i) < 5)
29 #define jit_arg_f_reg_p(i) ((i) >= 0 && (i) < NUM_FLOAT_REG_ARGS)
30
31 /*
32 * Types
33 */
34 typedef struct jit_va_list {
35 /* The offsets are "1" based, as addresses are fixed in the
36 * standard stack frame format. */
37 jit_word_t gpoff;
38 jit_word_t fpoff;
39
40 /* Easier when there is an explicitly defined type...
41 (gdb) ptype ap
42 type = struct __va_list_tag {
43 long __gpr;
44 long __fpr;
45 void *__overflow_arg_area;
46 void *__reg_save_area;
47
48 Note that gopff (__gpr) and fpoff (__fpr) are jit_word_t equivalent
49 and, again, "1" (unit) based, so must be adjusted at va_arg time.
50 */
51 jit_pointer_t over;
52 jit_pointer_t save;
53
54 /* For variadic functions, always allocate space to save callee
55 * save fpr registers.
56 * Note that s390 has a standard stack frame format that lightning
57 * does not fully comply with, but for variadic functions it must,
58 * for those (variadic) do not use the "empty" spaces for any
59 * callee save fpr register, but save them after the va_list
60 * space; and use the standard stack frame format, as required
61 * by variadic functions (and have a compatible va_list pointer). */
62 jit_float64_t f8;
63 jit_float64_t f9;
64 jit_float64_t f10;
65 jit_float64_t f11;
66 jit_float64_t f12;
67 jit_float64_t f13;
68 jit_float64_t f14;
69 jit_float64_t f15;
70 } jit_va_list_t;
71
72 /*
73 * Prototypes
74 */
75 #define jit_get_reg_pair() _jit_get_reg_pair(_jit)
76 static jit_int32_t _jit_get_reg_pair(jit_state_t*);
77 #define jit_unget_reg_pair(regno) _jit_unget_reg_pair(_jit,regno)
78 static void _jit_unget_reg_pair(jit_state_t*,jit_int32_t);
79 #define jit_get_reg_but_zero(flags) _jit_get_reg_but_zero(_jit,flags)
80 static jit_int32_t _jit_get_reg_but_zero(jit_state_t*,jit_int32_t);
81 #define jit_unget_reg_but_zero(reg) jit_unget_reg(reg)
82 #define patch(instr, node) _patch(_jit, instr, node)
83 static void _patch(jit_state_t*,jit_word_t,jit_node_t*);
84
85 /* libgcc */
86 extern void __clear_cache(void *, void *);
87
88 #define PROTO 1
89 # include "jit_s390-cpu.c"
90 # include "jit_s390-fpu.c"
91 #undef PROTO
92
93 /*
94 * Initialization
95 */
96 jit_register_t _rvs[] = {
97 { rc(gpr) | 0x0, "%r0" },
98 { rc(gpr) | 0x1, "%r1" },
99 { rc(gpr) | rc(sav) | 0xc, "%r12" },
100 { rc(gpr) | rc(sav) | 0xb, "%r11" },
101 { rc(gpr) | rc(sav) | 0xa, "%r10" },
102 { rc(gpr) | rc(sav) | 0x9, "%r9" },
103 { rc(gpr) | rc(sav) | 0x8, "%r8" },
104 { rc(gpr) | rc(sav) | 0x7, "%r7" },
105 { rc(gpr) | rc(arg) | rc(sav) | 0x6,"%r6" },
106 { rc(gpr) | rc(arg) | 0x5, "%r5" },
107 { rc(gpr) | rc(arg) | 0x4, "%r4" },
108 { rc(gpr) | rc(arg) | 0x3, "%r3" },
109 { rc(gpr) | rc(arg) | 0x2, "%r2" },
110 { rc(sav) | 0xd, "%r13" }, /* used as JIT_FP */
111 { 0xe, "%r14" },
112 { rc(sav) | 0xf, "%r15" },
113 { rc(fpr) | 0x1, "%f1" },
114 { rc(fpr) | 0x3, "%f3" },
115 { rc(fpr) | 0x5, "%f5" },
116 { rc(fpr) | 0x7, "%f7" },
117 { rc(fpr) | rc(sav) | 0xe, "%f14" },
118 /* Do not use as temporary to simplify stack layout */
119 { 0xf, "%f15" },
120 { rc(fpr) | rc(sav) | 0x8, "%f8" },
121 { rc(fpr) | rc(sav) | 0x9, "%f9" },
122 { rc(fpr) | rc(sav) | 0xa, "%f10" },
123 { rc(fpr) | rc(sav) | 0xb, "%f11" },
124 { rc(fpr) | rc(sav) | 0xc, "%f12" },
125 { rc(fpr) | rc(sav) | 0xd, "%f13" },
126 { rc(fpr) | rc(arg) | 0x6, "%f6" },
127 { rc(fpr) | rc(arg) | 0x4, "%f4" },
128 { rc(fpr) | rc(arg) | 0x2, "%f2" },
129 { rc(fpr) | rc(arg) | 0x0, "%f0" },
130 { _NOREG, "<none>" },
131 };
132
133 /*
134 * Implementation
135 */
136 void
jit_get_cpu(void)137 jit_get_cpu(void)
138 {
139 }
140
141 void
_jit_init(jit_state_t * _jit)142 _jit_init(jit_state_t *_jit)
143 {
144 _jitc->reglen = jit_size(_rvs) - 1;
145 }
146
147 void
_jit_prolog(jit_state_t * _jit)148 _jit_prolog(jit_state_t *_jit)
149 {
150 jit_int32_t offset;
151
152 if (_jitc->function)
153 jit_epilog();
154 assert(jit_regset_cmp_ui(&_jitc->regarg, 0) == 0);
155 jit_regset_set_ui(&_jitc->regsav, 0);
156 offset = _jitc->functions.offset;
157 if (offset >= _jitc->functions.length) {
158 jit_realloc((jit_pointer_t *)&_jitc->functions.ptr,
159 _jitc->functions.length * sizeof(jit_function_t),
160 (_jitc->functions.length + 16) * sizeof(jit_function_t));
161 _jitc->functions.length += 16;
162 }
163 _jitc->function = _jitc->functions.ptr + _jitc->functions.offset++;
164 _jitc->function->self.size = stack_framesize;
165 _jitc->function->self.argi = _jitc->function->self.argf =
166 _jitc->function->self.aoff = _jitc->function->self.alen = 0;
167 /* preallocate 8 bytes if not using a constant data buffer */
168 if (_jitc->no_data)
169 _jitc->function->self.aoff = -8;
170 _jitc->function->self.call = jit_call_default;
171 jit_alloc((jit_pointer_t *)&_jitc->function->regoff,
172 _jitc->reglen * sizeof(jit_int32_t));
173
174 /* _no_link here does not mean the jit_link() call can be removed
175 * by rewriting as:
176 * _jitc->function->prolog = jit_new_node(jit_code_prolog);
177 */
178 _jitc->function->prolog = jit_new_node_no_link(jit_code_prolog);
179 jit_link(_jitc->function->prolog);
180 _jitc->function->prolog->w.w = offset;
181 _jitc->function->epilog = jit_new_node_no_link(jit_code_epilog);
182 /* u: label value
183 * v: offset in blocks vector
184 * w: offset in functions vector
185 */
186 _jitc->function->epilog->w.w = offset;
187
188 jit_regset_new(&_jitc->function->regset);
189 }
190
191 jit_int32_t
_jit_allocai(jit_state_t * _jit,jit_int32_t length)192 _jit_allocai(jit_state_t *_jit, jit_int32_t length)
193 {
194 assert(_jitc->function);
195 switch (length) {
196 case 0: case 1: break;
197 case 2: _jitc->function->self.aoff &= -2; break;
198 case 3: case 4: _jitc->function->self.aoff &= -4; break;
199 default: _jitc->function->self.aoff &= -8; break;
200 }
201 _jitc->function->self.aoff -= length;
202 if (!_jitc->realize) {
203 jit_inc_synth_ww(allocai, _jitc->function->self.aoff, length);
204 jit_dec_synth();
205 }
206 return (_jitc->function->self.aoff);
207 }
208
209 void
_jit_allocar(jit_state_t * _jit,jit_int32_t u,jit_int32_t v)210 _jit_allocar(jit_state_t *_jit, jit_int32_t u, jit_int32_t v)
211 {
212 jit_int32_t reg;
213 assert(_jitc->function);
214 jit_inc_synth_ww(allocar, u, v);
215 if (!_jitc->function->allocar) {
216 _jitc->function->aoffoff = jit_allocai(sizeof(jit_int32_t));
217 _jitc->function->allocar = 1;
218 }
219 reg = jit_get_reg(jit_class_gpr);
220 jit_negr(reg, v);
221 jit_andi(reg, reg, -8);
222 jit_ldxi_i(u, JIT_FP, _jitc->function->aoffoff);
223 jit_addr(u, u, reg);
224 jit_addr(JIT_SP, JIT_SP, reg);
225 jit_stxi_i(_jitc->function->aoffoff, JIT_FP, u);
226 jit_unget_reg(reg);
227 jit_dec_synth();
228 }
229
230 void
_jit_ret(jit_state_t * _jit)231 _jit_ret(jit_state_t *_jit)
232 {
233 jit_node_t *instr;
234 assert(_jitc->function);
235 jit_inc_synth(ret);
236 /* jump to epilog */
237 instr = jit_jmpi();
238 jit_patch_at(instr, _jitc->function->epilog);
239 jit_dec_synth();
240 }
241
242 void
_jit_retr(jit_state_t * _jit,jit_int32_t u)243 _jit_retr(jit_state_t *_jit, jit_int32_t u)
244 {
245 jit_inc_synth_w(retr, u);
246 jit_movr(JIT_RET, u);
247 jit_ret();
248 jit_dec_synth();
249 }
250
251 void
_jit_reti(jit_state_t * _jit,jit_word_t u)252 _jit_reti(jit_state_t *_jit, jit_word_t u)
253 {
254 jit_inc_synth_w(reti, u);
255 jit_movi(JIT_RET, u);
256 jit_ret();
257 jit_dec_synth();
258 }
259
260 void
_jit_retr_f(jit_state_t * _jit,jit_int32_t u)261 _jit_retr_f(jit_state_t *_jit, jit_int32_t u)
262 {
263 jit_inc_synth_w(retr_f, u);
264 jit_movr_f(JIT_FRET, u);
265 jit_ret();
266 jit_dec_synth();
267 }
268
269 void
_jit_reti_f(jit_state_t * _jit,jit_float32_t u)270 _jit_reti_f(jit_state_t *_jit, jit_float32_t u)
271 {
272 jit_inc_synth_f(reti_f, u);
273 jit_movi_f(JIT_FRET, u);
274 jit_ret();
275 jit_dec_synth();
276 }
277
278 void
_jit_retr_d(jit_state_t * _jit,jit_int32_t u)279 _jit_retr_d(jit_state_t *_jit, jit_int32_t u)
280 {
281 jit_inc_synth_w(retr_d, u);
282 jit_movr_d(JIT_FRET, u);
283 jit_ret();
284 jit_dec_synth();
285 }
286
287 void
_jit_reti_d(jit_state_t * _jit,jit_float64_t u)288 _jit_reti_d(jit_state_t *_jit, jit_float64_t u)
289 {
290 jit_inc_synth_d(reti_d, u);
291 jit_movi_d(JIT_FRET, u);
292 jit_ret();
293 jit_dec_synth();
294 }
295
296 void
_jit_epilog(jit_state_t * _jit)297 _jit_epilog(jit_state_t *_jit)
298 {
299 assert(_jitc->function);
300 assert(_jitc->function->epilog->next == NULL);
301 jit_link(_jitc->function->epilog);
302 _jitc->function = NULL;
303 }
304
305 jit_bool_t
_jit_arg_register_p(jit_state_t * _jit,jit_node_t * u)306 _jit_arg_register_p(jit_state_t *_jit, jit_node_t *u)
307 {
308 if (u->code == jit_code_arg)
309 return (jit_arg_reg_p(u->u.w));
310 assert(u->code == jit_code_arg_f || u->code == jit_code_arg_d);
311 return (jit_arg_f_reg_p(u->u.w));
312 }
313
314 void
_jit_ellipsis(jit_state_t * _jit)315 _jit_ellipsis(jit_state_t *_jit)
316 {
317 jit_inc_synth(ellipsis);
318 if (_jitc->prepare) {
319 jit_link_prepare();
320 assert(!(_jitc->function->call.call & jit_call_varargs));
321 _jitc->function->call.call |= jit_call_varargs;
322 }
323 else {
324 jit_link_prolog();
325 assert(!(_jitc->function->self.call & jit_call_varargs));
326 _jitc->function->self.call |= jit_call_varargs;
327
328 /* Allocate va_list like object in the stack. */
329 _jitc->function->vaoff = jit_allocai(sizeof(jit_va_list_t));
330
331 /* Initialize gp offset in save area. */
332 if (jit_arg_reg_p(_jitc->function->self.argi))
333 _jitc->function->vagp = _jitc->function->self.argi;
334 else
335 _jitc->function->vagp = 5;
336
337 /* Initialize fp offset in save area. */
338 if (jit_arg_f_reg_p(_jitc->function->self.argf))
339 _jitc->function->vafp = _jitc->function->self.argf;
340 else
341 _jitc->function->vafp = NUM_FLOAT_REG_ARGS;
342 }
343 jit_dec_synth();
344 }
345
346 void
_jit_va_push(jit_state_t * _jit,jit_int32_t u)347 _jit_va_push(jit_state_t *_jit, jit_int32_t u)
348 {
349 jit_inc_synth_w(va_push, u);
350 jit_pushargr(u);
351 jit_dec_synth();
352 }
353
354 jit_node_t *
_jit_arg(jit_state_t * _jit)355 _jit_arg(jit_state_t *_jit)
356 {
357 jit_node_t *node;
358 jit_int32_t offset;
359 assert(_jitc->function);
360 if (jit_arg_reg_p(_jitc->function->self.argi))
361 offset = _jitc->function->self.argi++;
362 else {
363 offset = _jitc->function->self.size;
364 _jitc->function->self.size += sizeof(jit_word_t);
365 }
366 node = jit_new_node_ww(jit_code_arg, offset,
367 ++_jitc->function->self.argn);
368 jit_link_prolog();
369 return (node);
370 }
371
372 jit_node_t *
_jit_arg_f(jit_state_t * _jit)373 _jit_arg_f(jit_state_t *_jit)
374 {
375 jit_node_t *node;
376 jit_int32_t offset;
377 assert(_jitc->function);
378 if (jit_arg_f_reg_p(_jitc->function->self.argf))
379 offset = _jitc->function->self.argf++;
380 else {
381 offset = _jitc->function->self.size;
382 _jitc->function->self.size += sizeof(jit_word_t);
383 }
384 node = jit_new_node_ww(jit_code_arg_f, offset,
385 ++_jitc->function->self.argn);
386 jit_link_prolog();
387 return (node);
388 }
389
390 jit_node_t *
_jit_arg_d(jit_state_t * _jit)391 _jit_arg_d(jit_state_t *_jit)
392 {
393 jit_node_t *node;
394 jit_int32_t offset;
395 assert(_jitc->function);
396 if (jit_arg_f_reg_p(_jitc->function->self.argf))
397 offset = _jitc->function->self.argf++;
398 else {
399 offset = _jitc->function->self.size;
400 _jitc->function->self.size += sizeof(jit_float64_t);
401 }
402 node = jit_new_node_ww(jit_code_arg_d, offset,
403 ++_jitc->function->self.argn);
404 jit_link_prolog();
405 return (node);
406 }
407
408 void
_jit_getarg_c(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)409 _jit_getarg_c(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
410 {
411 assert(v->code == jit_code_arg);
412 jit_inc_synth_wp(getarg_c, u, v);
413 if (jit_arg_reg_p(v->u.w))
414 jit_extr_c(u, _R2 - v->u.w);
415 else
416 jit_ldxi_c(u, JIT_FP,
417 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int8_t));
418 jit_dec_synth();
419 }
420
421 void
_jit_getarg_uc(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)422 _jit_getarg_uc(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
423 {
424 assert(v->code == jit_code_arg);
425 jit_inc_synth_wp(getarg_uc, u, v);
426 if (jit_arg_reg_p(v->u.w))
427 jit_extr_uc(u, _R2 - v->u.w);
428 else
429 jit_ldxi_uc(u, JIT_FP,
430 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint8_t));
431 jit_dec_synth();
432 }
433
434 void
_jit_getarg_s(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)435 _jit_getarg_s(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
436 {
437 assert(v->code == jit_code_arg);
438 jit_inc_synth_wp(getarg_s, u, v);
439 if (jit_arg_reg_p(v->u.w))
440 jit_extr_s(u, _R2 - v->u.w);
441 else
442 jit_ldxi_s(u, JIT_FP,
443 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int16_t));
444 jit_dec_synth();
445 }
446
447 void
_jit_getarg_us(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)448 _jit_getarg_us(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
449 {
450 assert(v->code == jit_code_arg);
451 jit_inc_synth_wp(getarg_us, u, v);
452 if (jit_arg_reg_p(v->u.w))
453 jit_extr_us(u, _R2 - v->u.w);
454 else
455 jit_ldxi_us(u, JIT_FP,
456 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint16_t));
457 jit_dec_synth();
458 }
459
460 void
_jit_getarg_i(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)461 _jit_getarg_i(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
462 {
463 assert(v->code == jit_code_arg);
464 jit_inc_synth_wp(getarg_i, u, v);
465 if (jit_arg_reg_p(v->u.w)) {
466 #if __WORDSIZE == 32
467 jit_movr(u, _R2 - v->u.w);
468 #else
469 jit_extr_i(u, _R2 - v->u.w);
470 #endif
471 }
472 else
473 jit_ldxi_i(u, JIT_FP,
474 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_int32_t));
475 jit_dec_synth();
476 }
477
478 #if __WORDSIZE == 64
479 void
_jit_getarg_ui(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)480 _jit_getarg_ui(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
481 {
482 assert(v->code == jit_code_arg);
483 jit_inc_synth_wp(getarg_ui, u, v);
484 if (jit_arg_reg_p(v->u.w))
485 jit_extr_ui(u, _R2 - v->u.w);
486 else
487 jit_ldxi_ui(u, JIT_FP,
488 v->u.w + (__WORDSIZE >> 3) - sizeof(jit_uint32_t));
489 jit_dec_synth();
490 }
491
492 void
_jit_getarg_l(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)493 _jit_getarg_l(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
494 {
495 assert(v->code == jit_code_arg);
496 jit_inc_synth_wp(getarg_l, u, v);
497 if (jit_arg_reg_p(v->u.w))
498 jit_movr(u, _R2 - v->u.w);
499 else
500 jit_ldxi_l(u, JIT_FP, v->u.w);
501 jit_dec_synth();
502 }
503 #endif
504
505 void
_jit_putargr(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)506 _jit_putargr(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
507 {
508 assert(v->code == jit_code_arg);
509 jit_inc_synth_wp(putargr, u, v);
510 if (jit_arg_reg_p(v->u.w))
511 jit_movr(_R2 - v->u.w, u);
512 else
513 jit_stxi(v->u.w, JIT_FP, u);
514 jit_dec_synth();
515 }
516
517 void
_jit_putargi(jit_state_t * _jit,jit_word_t u,jit_node_t * v)518 _jit_putargi(jit_state_t *_jit, jit_word_t u, jit_node_t *v)
519 {
520 jit_int32_t regno;
521 assert(v->code == jit_code_arg);
522 jit_inc_synth_wp(putargi, u, v);
523 if (jit_arg_reg_p(v->u.w))
524 jit_movi(_R2 - v->u.w, u);
525 else {
526 regno = jit_get_reg(jit_class_gpr);
527 jit_movi(regno, u);
528 jit_stxi(v->u.w, JIT_FP, regno);
529 jit_unget_reg(regno);
530 }
531 jit_dec_synth();
532 }
533
534 void
_jit_getarg_f(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)535 _jit_getarg_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
536 {
537 assert(v->code == jit_code_arg_f);
538 jit_inc_synth_wp(getarg_f, u, v);
539 if (jit_arg_f_reg_p(v->u.w))
540 jit_movr_f(u, _F0 - v->u.w);
541 else
542 jit_ldxi_f(u, JIT_FP,
543 v->u.w
544 #if __WORDSIZE == 64
545 + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
546 #endif
547 );
548 jit_dec_synth();
549 }
550
551 void
_jit_putargr_f(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)552 _jit_putargr_f(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
553 {
554 assert(v->code == jit_code_arg_f);
555 jit_inc_synth_wp(putargr_f, u, v);
556 if (jit_arg_f_reg_p(v->u.w))
557 jit_movr_f(_F0 - v->u.w, u);
558 else
559 jit_stxi_f(v->u.w
560 #if __WORDSIZE == 64
561 + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
562 #endif
563 , JIT_FP, u);
564 jit_dec_synth();
565 }
566
567 void
_jit_putargi_f(jit_state_t * _jit,jit_float32_t u,jit_node_t * v)568 _jit_putargi_f(jit_state_t *_jit, jit_float32_t u, jit_node_t *v)
569 {
570 jit_int32_t regno;
571 assert(v->code == jit_code_arg_f);
572 jit_inc_synth_fp(putargi_f, u, v);
573 if (jit_arg_f_reg_p(v->u.w))
574 jit_movi_f(_F0 - v->u.w, u);
575 else {
576 regno = jit_get_reg(jit_class_fpr);
577 jit_movi_f(regno, u);
578 jit_stxi_f(v->u.w
579 #if __WORDSIZE == 64
580 + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
581 #endif
582 , JIT_FP, regno);
583 jit_unget_reg(regno);
584 }
585 jit_dec_synth();
586 }
587
588 void
_jit_getarg_d(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)589 _jit_getarg_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
590 {
591 assert(v->code == jit_code_arg_d);
592 jit_inc_synth_wp(getarg_d, u, v);
593 if (jit_arg_f_reg_p(v->u.w))
594 jit_movr_d(u, _F0 - v->u.w);
595 else
596 jit_ldxi_d(u, JIT_FP, v->u.w);
597 jit_dec_synth();
598 }
599
600 void
_jit_putargr_d(jit_state_t * _jit,jit_int32_t u,jit_node_t * v)601 _jit_putargr_d(jit_state_t *_jit, jit_int32_t u, jit_node_t *v)
602 {
603 assert(v->code == jit_code_arg_d);
604 jit_inc_synth_wp(putargr_d, u, v);
605 if (jit_arg_f_reg_p(v->u.w))
606 jit_movr_d(_F0 - v->u.w, u);
607 else
608 jit_stxi_d(v->u.w, JIT_FP, u);
609 jit_dec_synth();
610 }
611
612 void
_jit_putargi_d(jit_state_t * _jit,jit_float64_t u,jit_node_t * v)613 _jit_putargi_d(jit_state_t *_jit, jit_float64_t u, jit_node_t *v)
614 {
615 jit_int32_t regno;
616 assert(v->code == jit_code_arg_d);
617 jit_inc_synth_dp(putargi_d, u, v);
618 if (jit_arg_f_reg_p(v->u.w))
619 jit_movi_d(_F0 - v->u.w, u);
620 else {
621 regno = jit_get_reg(jit_class_fpr);
622 jit_movi_d(regno, u);
623 jit_stxi_d(v->u.w, JIT_FP, regno);
624 jit_unget_reg(regno);
625 }
626 jit_dec_synth();
627 }
628
629 void
_jit_pushargr(jit_state_t * _jit,jit_int32_t u)630 _jit_pushargr(jit_state_t *_jit, jit_int32_t u)
631 {
632 assert(_jitc->function);
633 jit_inc_synth_w(pushargr, u);
634 jit_link_prepare();
635 if (jit_arg_reg_p(_jitc->function->call.argi)) {
636 jit_movr(_R2 - _jitc->function->call.argi, u);
637 ++_jitc->function->call.argi;
638 }
639 else {
640 jit_stxi(_jitc->function->call.size + stack_framesize, JIT_SP, u);
641 _jitc->function->call.size += sizeof(jit_word_t);
642 }
643 jit_dec_synth();
644 }
645
646 void
_jit_pushargi(jit_state_t * _jit,jit_word_t u)647 _jit_pushargi(jit_state_t *_jit, jit_word_t u)
648 {
649 jit_int32_t regno;
650 assert(_jitc->function);
651 jit_inc_synth_w(pushargi, u);
652 jit_link_prepare();
653 if (jit_arg_reg_p(_jitc->function->call.argi)) {
654 jit_movi(_R2 - _jitc->function->call.argi, u);
655 ++_jitc->function->call.argi;
656 }
657 else {
658 regno = jit_get_reg(jit_class_gpr);
659 jit_movi(regno, u);
660 jit_stxi(_jitc->function->call.size + stack_framesize, JIT_SP, regno);
661 jit_unget_reg(regno);
662 _jitc->function->call.size += sizeof(jit_word_t);
663 }
664 jit_dec_synth();
665 }
666
667 void
_jit_pushargr_f(jit_state_t * _jit,jit_int32_t u)668 _jit_pushargr_f(jit_state_t *_jit, jit_int32_t u)
669 {
670 assert(_jitc->function);
671 jit_inc_synth_w(pushargr_f, u);
672 jit_link_prepare();
673 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
674 jit_movr_f(_F0 - _jitc->function->call.argf, u);
675 ++_jitc->function->call.argf;
676 }
677 else {
678 jit_stxi_f(_jitc->function->call.size + stack_framesize
679 #if __WORDSIZE == 64
680 + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
681 #endif
682 , JIT_SP, u);
683 _jitc->function->call.size += sizeof(jit_word_t);
684 }
685 jit_dec_synth();
686 }
687
688 void
_jit_pushargi_f(jit_state_t * _jit,jit_float32_t u)689 _jit_pushargi_f(jit_state_t *_jit, jit_float32_t u)
690 {
691 jit_int32_t regno;
692 assert(_jitc->function);
693 jit_inc_synth_f(pushargi_f, u);
694 jit_link_prepare();
695 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
696 jit_movi_f(_F0 - _jitc->function->call.argf, u);
697 ++_jitc->function->call.argf;
698 }
699 else {
700 regno = jit_get_reg(jit_class_fpr);
701 jit_movi_f(regno, u);
702 jit_stxi_f(_jitc->function->call.size + stack_framesize
703 #if __WORDSIZE == 64
704 + (__WORDSIZE >> 3) - sizeof(jit_float32_t)
705 #endif
706 , JIT_SP, regno);
707 jit_unget_reg(regno);
708 _jitc->function->call.size += sizeof(jit_word_t);
709 }
710 jit_dec_synth();
711 }
712
713 void
_jit_pushargr_d(jit_state_t * _jit,jit_int32_t u)714 _jit_pushargr_d(jit_state_t *_jit, jit_int32_t u)
715 {
716 assert(_jitc->function);
717 jit_inc_synth_w(pushargr_d, u);
718 jit_link_prepare();
719 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
720 jit_movr_d(_F0 - _jitc->function->call.argf, u);
721 ++_jitc->function->call.argf;
722 }
723 else {
724 jit_stxi_d(_jitc->function->call.size + stack_framesize, JIT_SP, u);
725 _jitc->function->call.size += sizeof(jit_float64_t);
726 }
727 jit_dec_synth();
728 }
729
730 void
_jit_pushargi_d(jit_state_t * _jit,jit_float64_t u)731 _jit_pushargi_d(jit_state_t *_jit, jit_float64_t u)
732 {
733 jit_int32_t regno;
734 assert(_jitc->function);
735 jit_inc_synth_d(pushargi_d, u);
736 jit_link_prepare();
737 if (jit_arg_f_reg_p(_jitc->function->call.argf)) {
738 jit_movi_d(_F0 - _jitc->function->call.argf, u);
739 ++_jitc->function->call.argf;
740 }
741 else {
742 regno = jit_get_reg(jit_class_fpr);
743 jit_movi_d(regno, u);
744 jit_stxi_d(_jitc->function->call.size + stack_framesize, JIT_SP, regno);
745 jit_unget_reg(regno);
746 _jitc->function->call.size += sizeof(jit_float64_t);
747 }
748 jit_dec_synth();
749 }
750
751 jit_bool_t
_jit_regarg_p(jit_state_t * _jit,jit_node_t * node,jit_int32_t regno)752 _jit_regarg_p(jit_state_t *_jit, jit_node_t *node, jit_int32_t regno)
753 {
754 jit_int32_t spec;
755 spec = jit_class(_rvs[regno].spec);
756 if (spec & jit_class_arg) {
757 regno = _R2 - regno;
758 if (regno >= 0 && regno < node->v.w)
759 return (1);
760 if (spec & jit_class_fpr) {
761 regno = _F0 - regno;
762 if (regno >= 0 && regno < node->w.w)
763 return (1);
764 }
765 }
766 return (0);
767 }
768
769 void
_jit_finishr(jit_state_t * _jit,jit_int32_t r0)770 _jit_finishr(jit_state_t *_jit, jit_int32_t r0)
771 {
772 jit_node_t *call;
773 assert(_jitc->function);
774 jit_inc_synth_w(finishr, r0);
775 if (_jitc->function->self.alen < _jitc->function->call.size)
776 _jitc->function->self.alen = _jitc->function->call.size;
777 call = jit_callr(r0);
778 call->v.w = _jitc->function->call.argi;
779 call->w.w = _jitc->function->call.argf;
780 _jitc->function->call.argi = _jitc->function->call.argf =
781 _jitc->function->call.size = 0;
782 _jitc->prepare = 0;
783 jit_dec_synth();
784 }
785
786 jit_node_t *
_jit_finishi(jit_state_t * _jit,jit_pointer_t i0)787 _jit_finishi(jit_state_t *_jit, jit_pointer_t i0)
788 {
789 jit_node_t *node;
790 assert(_jitc->function);
791 jit_inc_synth_w(finishi, (jit_word_t)i0);
792 if (_jitc->function->self.alen < _jitc->function->call.size)
793 _jitc->function->self.alen = _jitc->function->call.size;
794 node = jit_calli(i0);
795 node->v.w = _jitc->function->call.argi;
796 node->w.w = _jitc->function->call.argf;
797 _jitc->function->call.argi = _jitc->function->call.argf =
798 _jitc->function->call.size = 0;
799 _jitc->prepare = 0;
800 jit_dec_synth();
801 return (node);
802 }
803
804 void
_jit_retval_c(jit_state_t * _jit,jit_int32_t r0)805 _jit_retval_c(jit_state_t *_jit, jit_int32_t r0)
806 {
807 jit_inc_synth_w(retval_c, r0);
808 jit_extr_c(r0, JIT_RET);
809 jit_dec_synth();
810 }
811
812 void
_jit_retval_uc(jit_state_t * _jit,jit_int32_t r0)813 _jit_retval_uc(jit_state_t *_jit, jit_int32_t r0)
814 {
815 jit_inc_synth_w(retval_uc, r0);
816 jit_extr_uc(r0, JIT_RET);
817 jit_dec_synth();
818 }
819
820 void
_jit_retval_s(jit_state_t * _jit,jit_int32_t r0)821 _jit_retval_s(jit_state_t *_jit, jit_int32_t r0)
822 {
823 jit_inc_synth_w(retval_s, r0);
824 jit_extr_s(r0, JIT_RET);
825 jit_dec_synth();
826 }
827
828 void
_jit_retval_us(jit_state_t * _jit,jit_int32_t r0)829 _jit_retval_us(jit_state_t *_jit, jit_int32_t r0)
830 {
831 jit_inc_synth_w(retval_us, r0);
832 jit_extr_us(r0, JIT_RET);
833 jit_dec_synth();
834 }
835
836 void
_jit_retval_i(jit_state_t * _jit,jit_int32_t r0)837 _jit_retval_i(jit_state_t *_jit, jit_int32_t r0)
838 {
839 jit_inc_synth_w(retval_i, r0);
840 #if __WORDSIZE == 64
841 jit_extr_i(r0, JIT_RET);
842 #else
843 jit_movr(r0, JIT_RET);
844 #endif
845 jit_dec_synth();
846 }
847
848 #if __WORDSIZE == 64
849 void
_jit_retval_ui(jit_state_t * _jit,jit_int32_t r0)850 _jit_retval_ui(jit_state_t *_jit, jit_int32_t r0)
851 {
852 jit_inc_synth_w(retval_ui, r0);
853 jit_extr_ui(r0, JIT_RET);
854 jit_dec_synth();
855 }
856
857 void
_jit_retval_l(jit_state_t * _jit,jit_int32_t r0)858 _jit_retval_l(jit_state_t *_jit, jit_int32_t r0)
859 {
860 jit_inc_synth_w(retval_l, r0);
861 jit_movr(r0, JIT_RET);
862 jit_dec_synth();
863 }
864 #endif
865
866 void
_jit_retval_f(jit_state_t * _jit,jit_int32_t r0)867 _jit_retval_f(jit_state_t *_jit, jit_int32_t r0)
868 {
869 jit_inc_synth_w(retval_f, r0);
870 jit_movr_f(r0, JIT_FRET);
871 jit_dec_synth();
872 }
873
874 void
_jit_retval_d(jit_state_t * _jit,jit_int32_t r0)875 _jit_retval_d(jit_state_t *_jit, jit_int32_t r0)
876 {
877 jit_inc_synth_w(retval_d, r0);
878 jit_movr_d(r0, JIT_FRET);
879 jit_dec_synth();
880 }
881
882 jit_pointer_t
_emit_code(jit_state_t * _jit)883 _emit_code(jit_state_t *_jit)
884 {
885 jit_node_t *node;
886 jit_node_t *temp;
887 jit_word_t word;
888 jit_int32_t value;
889 jit_int32_t offset;
890 struct {
891 jit_node_t *node;
892 jit_word_t word;
893 #if DEVEL_DISASSEMBLER
894 jit_word_t prevw;
895 #endif
896 jit_int32_t patch_offset;
897 } undo;
898 #if DEVEL_DISASSEMBLER
899 jit_word_t prevw;
900 #endif
901
902 _jitc->function = NULL;
903
904 jit_reglive_setup();
905
906 undo.word = 0;
907 undo.node = NULL;
908 undo.patch_offset = 0;
909
910 #define assert_data(node) /**/
911 #define case_rr(name, type) \
912 case jit_code_##name##r##type: \
913 name##r##type(rn(node->u.w), rn(node->v.w)); \
914 break
915 #define case_rw(name, type) \
916 case jit_code_##name##i##type: \
917 name##i##type(rn(node->u.w), node->v.w); \
918 break
919 #define case_wr(name, type) \
920 case jit_code_##name##i##type: \
921 name##i##type(node->u.w, rn(node->v.w)); \
922 break
923 #define case_rrr(name, type) \
924 case jit_code_##name##r##type: \
925 name##r##type(rn(node->u.w), \
926 rn(node->v.w), rn(node->w.w)); \
927 break
928 #define case_rrrr(name, type) \
929 case jit_code_##name##r##type: \
930 name##r##type(rn(node->u.q.l), rn(node->u.q.h), \
931 rn(node->v.w), rn(node->w.w)); \
932 break
933 #define case_rrw(name, type) \
934 case jit_code_##name##i##type: \
935 name##i##type(rn(node->u.w), rn(node->v.w), node->w.w); \
936 break
937 #define case_rrrw(name, type) \
938 case jit_code_##name##i##type: \
939 name##i##type(rn(node->u.q.l), rn(node->u.q.h), \
940 rn(node->v.w), node->w.w); \
941 break
942 #define case_rrf(name) \
943 case jit_code_##name##i_f: \
944 assert_data(node); \
945 name##i_f(rn(node->u.w), rn(node->v.w), \
946 (jit_float32_t *)node->w.n->u.w); \
947 break
948 #define case_rrd(name) \
949 case jit_code_##name##i_d: \
950 assert_data(node); \
951 name##i_d(rn(node->u.w), rn(node->v.w), \
952 (jit_float64_t *)node->w.n->u.w); \
953 break
954 #define case_wrr(name, type) \
955 case jit_code_##name##i##type: \
956 name##i##type(node->u.w, rn(node->v.w), rn(node->w.w)); \
957 break
958 #define case_brr(name, type) \
959 case jit_code_##name##r##type: \
960 temp = node->u.n; \
961 assert(temp->code == jit_code_label || \
962 temp->code == jit_code_epilog); \
963 if (temp->flag & jit_flag_patch) \
964 name##r##type(temp->u.w, rn(node->v.w), \
965 rn(node->w.w)); \
966 else { \
967 word = name##r##type##_p(_jit->pc.w, \
968 rn(node->v.w), \
969 rn(node->w.w)); \
970 patch(word, node); \
971 } \
972 break
973 #define case_brw(name, type) \
974 case jit_code_##name##i##type: \
975 temp = node->u.n; \
976 assert(temp->code == jit_code_label || \
977 temp->code == jit_code_epilog); \
978 if (temp->flag & jit_flag_patch) \
979 name##i##type(temp->u.w, \
980 rn(node->v.w), node->w.w); \
981 else { \
982 word = name##i##type##_p(_jit->pc.w, \
983 rn(node->v.w), node->w.w); \
984 patch(word, node); \
985 } \
986 break;
987 #define case_brf(name) \
988 case jit_code_##name##i_f: \
989 temp = node->u.n; \
990 assert(temp->code == jit_code_label || \
991 temp->code == jit_code_epilog); \
992 if (temp->flag & jit_flag_patch) \
993 name##i_f(temp->u.w, rn(node->v.w), \
994 (jit_float32_t *)node->w.n->u.w); \
995 else { \
996 word = name##i_f_p(_jit->pc.w, rn(node->v.w), \
997 (jit_float32_t *)node->w.n->u.w);\
998 patch(word, node); \
999 } \
1000 break
1001 #define case_brd(name) \
1002 case jit_code_##name##i_d: \
1003 temp = node->u.n; \
1004 assert(temp->code == jit_code_label || \
1005 temp->code == jit_code_epilog); \
1006 if (temp->flag & jit_flag_patch) \
1007 name##i_d(temp->u.w, rn(node->v.w), \
1008 (jit_float64_t *)node->w.n->u.w); \
1009 else { \
1010 word = name##i_d_p(_jit->pc.w, rn(node->v.w), \
1011 (jit_float64_t *)node->w.n->u.w);\
1012 patch(word, node); \
1013 } \
1014 break
1015 #if DEVEL_DISASSEMBLER
1016 prevw = _jit->pc.w;
1017 #endif
1018 for (node = _jitc->head; node; node = node->next) {
1019 if (_jit->pc.uc >= _jitc->code.end)
1020 return (NULL);
1021
1022 #if DEVEL_DISASSEMBLER
1023 node->offset = (jit_uword_t)_jit->pc.w - (jit_uword_t)prevw;
1024 prevw = _jit->pc.w;
1025 #endif
1026 value = jit_classify(node->code);
1027 jit_regarg_set(node, value);
1028 switch (node->code) {
1029 case jit_code_align:
1030 assert(!(node->u.w & (node->u.w - 1)) &&
1031 node->u.w <= sizeof(jit_word_t));
1032 if (node->u.w == sizeof(jit_word_t) &&
1033 (word = _jit->pc.w & (sizeof(jit_word_t) - 1)))
1034 nop(sizeof(jit_word_t) - word);
1035 break;
1036 case jit_code_note: case jit_code_name:
1037 node->u.w = _jit->pc.w;
1038 break;
1039 case jit_code_label:
1040 if ((node->link || (node->flag & jit_flag_use)) &&
1041 (word = _jit->pc.w & 3))
1042 nop(4 - word);
1043 /* remember label is defined */
1044 node->flag |= jit_flag_patch;
1045 node->u.w = _jit->pc.w;
1046 break;
1047 case_rrr(add,);
1048 case_rrw(add,);
1049 case_rrr(addc,);
1050 case_rrw(addc,);
1051 case_rrr(addx,);
1052 case_rrw(addx,);
1053 case_rrr(sub,);
1054 case_rrw(sub,);
1055 case_rrr(subc,);
1056 case_rrw(subc,);
1057 case_rrr(subx,);
1058 case_rrw(subx,);
1059 case_rrw(rsb,);
1060 case_rrr(mul,);
1061 case_rrw(mul,);
1062 case_rrrr(qmul,);
1063 case_rrrw(qmul,);
1064 case_rrrr(qmul, _u);
1065 case_rrrw(qmul, _u);
1066 case_rrr(div,);
1067 case_rrw(div,);
1068 case_rrr(div, _u);
1069 case_rrw(div, _u);
1070 case_rrr(rem,);
1071 case_rrw(rem,);
1072 case_rrr(rem, _u);
1073 case_rrw(rem, _u);
1074 case_rrrr(qdiv,);
1075 case_rrrw(qdiv,);
1076 case_rrrr(qdiv, _u);
1077 case_rrrw(qdiv, _u);
1078 case_rrr(lsh,);
1079 case_rrw(lsh,);
1080 case_rrr(rsh,);
1081 case_rrw(rsh,);
1082 case_rrr(rsh, _u);
1083 case_rrw(rsh, _u);
1084 case_rr(neg,);
1085 case_rr(com,);
1086 case_rrr(and,);
1087 case_rrw(and,);
1088 case_rrr(or,);
1089 case_rrw(or,);
1090 case_rrr(xor,);
1091 case_rrw(xor,);
1092 case_rr(trunc, _f_i);
1093 case_rr(trunc, _d_i);
1094 #if __WORDSIZE == 64
1095 case_rr(trunc, _f_l);
1096 case_rr(trunc, _d_l);
1097 #endif
1098 case_rr(ld, _c);
1099 case_rw(ld, _c);
1100 case_rr(ld, _uc);
1101 case_rw(ld, _uc);
1102 case_rr(ld, _s);
1103 case_rw(ld, _s);
1104 case_rr(ld, _us);
1105 case_rw(ld, _us);
1106 case_rr(ld, _i);
1107 case_rw(ld, _i);
1108 #if __WORDSIZE == 64
1109 case_rr(ld, _ui);
1110 case_rw(ld, _ui);
1111 case_rr(ld, _l);
1112 case_rw(ld, _l);
1113 #endif
1114 case_rrr(ldx, _c);
1115 case_rrw(ldx, _c);
1116 case_rrr(ldx, _uc);
1117 case_rrw(ldx, _uc);
1118 case_rrr(ldx, _s);
1119 case_rrw(ldx, _s);
1120 case_rrr(ldx, _us);
1121 case_rrw(ldx, _us);
1122 case_rrr(ldx, _i);
1123 case_rrw(ldx, _i);
1124 #if __WORDSIZE == 64
1125 case_rrr(ldx, _ui);
1126 case_rrw(ldx, _ui);
1127 case_rrr(ldx, _l);
1128 case_rrw(ldx, _l);
1129 #endif
1130 case_rr(st, _c);
1131 case_wr(st, _c);
1132 case_rr(st, _s);
1133 case_wr(st, _s);
1134 case_rr(st, _i);
1135 case_wr(st, _i);
1136 #if __WORDSIZE == 64
1137 case_rr(st, _l);
1138 case_wr(st, _l);
1139 #endif
1140 case_rrr(stx, _c);
1141 case_wrr(stx, _c);
1142 case_rrr(stx, _s);
1143 case_wrr(stx, _s);
1144 case_rrr(stx, _i);
1145 case_wrr(stx, _i);
1146 #if __WORDSIZE == 64
1147 case_rrr(stx, _l);
1148 case_wrr(stx, _l);
1149 #endif
1150 case_rr(hton, _us);
1151 case_rr(hton, _ui);
1152 #if __WORDSIZE == 64
1153 case_rr(hton, _ul);
1154 #endif
1155 case_rr(ext, _c);
1156 case_rr(ext, _uc);
1157 case_rr(ext, _s);
1158 case_rr(ext, _us);
1159 #if __WORDSIZE == 64
1160 case_rr(ext, _i);
1161 case_rr(ext, _ui);
1162 #endif
1163 case_rr(mov,);
1164 case jit_code_movi:
1165 if (node->flag & jit_flag_node) {
1166 temp = node->v.n;
1167 if (temp->code == jit_code_data ||
1168 (temp->code == jit_code_label &&
1169 (temp->flag & jit_flag_patch)))
1170 movi(rn(node->u.w), temp->u.w);
1171 else {
1172 assert(temp->code == jit_code_label ||
1173 temp->code == jit_code_epilog);
1174 word = movi_p(rn(node->u.w), temp->u.w);
1175 patch(word, node);
1176 }
1177 }
1178 else
1179 movi(rn(node->u.w), node->v.w);
1180 break;
1181 case_rrr(lt,);
1182 case_rrw(lt,);
1183 case_rrr(lt, _u);
1184 case_rrw(lt, _u);
1185 case_rrr(le,);
1186 case_rrw(le,);
1187 case_rrr(le, _u);
1188 case_rrw(le, _u);
1189 case_rrr(eq,);
1190 case_rrw(eq,);
1191 case_rrr(ge,);
1192 case_rrw(ge,);
1193 case_rrr(ge, _u);
1194 case_rrw(ge, _u);
1195 case_rrr(gt,);
1196 case_rrw(gt,);
1197 case_rrr(gt, _u);
1198 case_rrw(gt, _u);
1199 case_rrr(ne,);
1200 case_rrw(ne,);
1201 case_brr(blt,);
1202 case_brw(blt,);
1203 case_brr(blt, _u);
1204 case_brw(blt, _u);
1205 case_brr(ble,);
1206 case_brw(ble,);
1207 case_brr(ble, _u);
1208 case_brw(ble, _u);
1209 case_brr(beq,);
1210 case_brw(beq,);
1211 case_brr(bge,);
1212 case_brw(bge,);
1213 case_brr(bge, _u);
1214 case_brw(bge, _u);
1215 case_brr(bgt,);
1216 case_brw(bgt,);
1217 case_brr(bgt, _u);
1218 case_brw(bgt, _u);
1219 case_brr(bne,);
1220 case_brw(bne,);
1221 case_brr(boadd,);
1222 case_brw(boadd,);
1223 case_brr(boadd, _u);
1224 case_brw(boadd, _u);
1225 case_brr(bxadd,);
1226 case_brw(bxadd,);
1227 case_brr(bxadd, _u);
1228 case_brw(bxadd, _u);
1229 case_brr(bosub,);
1230 case_brw(bosub,);
1231 case_brr(bosub, _u);
1232 case_brw(bosub, _u);
1233 case_brr(bxsub,);
1234 case_brw(bxsub,);
1235 case_brr(bxsub, _u);
1236 case_brw(bxsub, _u);
1237 case_brr(bms,);
1238 case_brw(bms,);
1239 case_brr(bmc,);
1240 case_brw(bmc,);
1241 case_rrr(add, _f);
1242 case_rrf(add);
1243 case_rrr(sub, _f);
1244 case_rrf(sub);
1245 case_rrf(rsb);
1246 case_rrr(mul, _f);
1247 case_rrf(mul);
1248 case_rrr(div, _f);
1249 case_rrf(div);
1250 case_rr(abs, _f);
1251 case_rr(neg, _f);
1252 case_rr(sqrt, _f);
1253 case_rr(ext, _f);
1254 case_rr(ld, _f);
1255 case_rw(ld, _f);
1256 case_rrr(ldx, _f);
1257 case_rrw(ldx, _f);
1258 case_rr(st, _f);
1259 case_wr(st, _f);
1260 case_rrr(stx, _f);
1261 case_wrr(stx, _f);
1262 case_rr(mov, _f);
1263 case jit_code_movi_f:
1264 assert_data(node);
1265 movi_f(rn(node->u.w), (jit_float32_t *)node->v.n->u.w);
1266 break;
1267 case_rr(ext, _d_f);
1268 case_rrr(lt, _f);
1269 case_rrf(lt);
1270 case_rrr(le, _f);
1271 case_rrf(le);
1272 case_rrr(eq, _f);
1273 case_rrf(eq);
1274 case_rrr(ge, _f);
1275 case_rrf(ge);
1276 case_rrr(gt, _f);
1277 case_rrf(gt);
1278 case_rrr(ne, _f);
1279 case_rrf(ne);
1280 case_rrr(unlt, _f);
1281 case_rrf(unlt);
1282 case_rrr(unle, _f);
1283 case_rrf(unle);
1284 case_rrr(uneq, _f);
1285 case_rrf(uneq);
1286 case_rrr(unge, _f);
1287 case_rrf(unge);
1288 case_rrr(ungt, _f);
1289 case_rrf(ungt);
1290 case_rrr(ltgt, _f);
1291 case_rrf(ltgt);
1292 case_rrr(ord, _f);
1293 case_rrf(ord);
1294 case_rrr(unord, _f);
1295 case_rrf(unord);
1296 case_brr(blt, _f);
1297 case_brf(blt);
1298 case_brr(ble, _f);
1299 case_brf(ble);
1300 case_brr(beq, _f);
1301 case_brf(beq);
1302 case_brr(bge, _f);
1303 case_brf(bge);
1304 case_brr(bgt, _f);
1305 case_brf(bgt);
1306 case_brr(bne, _f);
1307 case_brf(bne);
1308 case_brr(bunlt, _f);
1309 case_brf(bunlt);
1310 case_brr(bunle, _f);
1311 case_brf(bunle);
1312 case_brr(buneq, _f);
1313 case_brf(buneq);
1314 case_brr(bunge, _f);
1315 case_brf(bunge);
1316 case_brr(bungt, _f);
1317 case_brf(bungt);
1318 case_brr(bltgt, _f);
1319 case_brf(bltgt);
1320 case_brr(bord, _f);
1321 case_brf(bord);
1322 case_brr(bunord, _f);
1323 case_brf(bunord);
1324 case_rrr(add, _d);
1325 case_rrd(add);
1326 case_rrr(sub, _d);
1327 case_rrd(sub);
1328 case_rrd(rsb);
1329 case_rrr(mul, _d);
1330 case_rrd(mul);
1331 case_rrr(div, _d);
1332 case_rrd(div);
1333 case_rr(abs, _d);
1334 case_rr(neg, _d);
1335 case_rr(sqrt, _d);
1336 case_rr(ext, _d);
1337 case_rr(ld, _d);
1338 case_rw(ld, _d);
1339 case_rrr(ldx, _d);
1340 case_rrw(ldx, _d);
1341 case_rr(st, _d);
1342 case_wr(st, _d);
1343 case_rrr(stx, _d);
1344 case_wrr(stx, _d);
1345 case_rr(mov, _d);
1346 case jit_code_movi_d:
1347 assert_data(node);
1348 movi_d(rn(node->u.w), (jit_float64_t *)node->v.n->u.w);
1349 break;
1350 case_rr(ext, _f_d);
1351 case_rrr(lt, _d);
1352 case_rrd(lt);
1353 case_rrr(le, _d);
1354 case_rrd(le);
1355 case_rrr(eq, _d);
1356 case_rrd(eq);
1357 case_rrr(ge, _d);
1358 case_rrd(ge);
1359 case_rrr(gt, _d);
1360 case_rrd(gt);
1361 case_rrr(ne, _d);
1362 case_rrd(ne);
1363 case_rrr(unlt, _d);
1364 case_rrd(unlt);
1365 case_rrr(unle, _d);
1366 case_rrd(unle);
1367 case_rrr(uneq, _d);
1368 case_rrd(uneq);
1369 case_rrr(unge, _d);
1370 case_rrd(unge);
1371 case_rrr(ungt, _d);
1372 case_rrd(ungt);
1373 case_rrr(ltgt, _d);
1374 case_rrd(ltgt);
1375 case_rrr(ord, _d);
1376 case_rrd(ord);
1377 case_rrr(unord, _d);
1378 case_rrd(unord);
1379 case_brr(blt, _d);
1380 case_brd(blt);
1381 case_brr(ble, _d);
1382 case_brd(ble);
1383 case_brr(beq, _d);
1384 case_brd(beq);
1385 case_brr(bge, _d);
1386 case_brd(bge);
1387 case_brr(bgt, _d);
1388 case_brd(bgt);
1389 case_brr(bne, _d);
1390 case_brd(bne);
1391 case_brr(bunlt, _d);
1392 case_brd(bunlt);
1393 case_brr(bunle, _d);
1394 case_brd(bunle);
1395 case_brr(buneq, _d);
1396 case_brd(buneq);
1397 case_brr(bunge, _d);
1398 case_brd(bunge);
1399 case_brr(bungt, _d);
1400 case_brd(bungt);
1401 case_brr(bltgt, _d);
1402 case_brd(bltgt);
1403 case_brr(bord, _d);
1404 case_brd(bord);
1405 case_brr(bunord, _d);
1406 case_brd(bunord);
1407 case jit_code_jmpr:
1408 jmpr(rn(node->u.w));
1409 break;
1410 case jit_code_jmpi:
1411 if (node->flag & jit_flag_node) {
1412 temp = node->u.n;
1413 assert(temp->code == jit_code_label ||
1414 temp->code == jit_code_epilog);
1415 if (temp->flag & jit_flag_patch)
1416 jmpi(temp->u.w);
1417 else {
1418 word = jmpi_p(_jit->pc.w);
1419 patch(word, node);
1420 }
1421 }
1422 else
1423 jmpi(node->u.w);
1424 break;
1425 case jit_code_callr:
1426 callr(rn(node->u.w));
1427 break;
1428 case jit_code_calli:
1429 if (node->flag & jit_flag_node) {
1430 temp = node->u.n;
1431 assert(temp->code == jit_code_label ||
1432 temp->code == jit_code_epilog);
1433 if (temp->flag & jit_flag_patch)
1434 calli(temp->u.w);
1435 else {
1436 word = calli_p(_jit->pc.w);
1437 patch(word, node);
1438 }
1439 }
1440 else
1441 calli(node->u.w);
1442 break;
1443 case jit_code_prolog:
1444 _jitc->function = _jitc->functions.ptr + node->w.w;
1445 undo.node = node;
1446 undo.word = _jit->pc.w;
1447 #if DEVEL_DISASSEMBLER
1448 undo.prevw = prevw;
1449 #endif
1450 undo.patch_offset = _jitc->patches.offset;
1451 restart_function:
1452 _jitc->again = 0;
1453 prolog(node);
1454 break;
1455 case jit_code_epilog:
1456 assert(_jitc->function == _jitc->functions.ptr + node->w.w);
1457 if (_jitc->again) {
1458 for (temp = undo.node->next;
1459 temp != node; temp = temp->next) {
1460 if (temp->code == jit_code_label ||
1461 temp->code == jit_code_epilog)
1462 temp->flag &= ~jit_flag_patch;
1463 }
1464 temp->flag &= ~jit_flag_patch;
1465 node = undo.node;
1466 _jit->pc.w = undo.word;
1467 #if DEVEL_DISASSEMBLER
1468 prevw = undo.prevw;
1469 #endif
1470 _jitc->patches.offset = undo.patch_offset;
1471 goto restart_function;
1472 }
1473 if (node->link && (word = _jit->pc.w & 3))
1474 nop(4 - word);
1475 /* remember label is defined */
1476 node->flag |= jit_flag_patch;
1477 node->u.w = _jit->pc.w;
1478 epilog(node);
1479 _jitc->function = NULL;
1480 break;
1481 case jit_code_va_start:
1482 vastart(rn(node->u.w));
1483 break;
1484 case jit_code_va_arg:
1485 vaarg(rn(node->u.w), rn(node->v.w));
1486 break;
1487 case jit_code_va_arg_d:
1488 vaarg_d(rn(node->u.w), rn(node->v.w));
1489 break;
1490 case jit_code_live: case jit_code_ellipsis:
1491 case jit_code_va_push:
1492 case jit_code_allocai: case jit_code_allocar:
1493 case jit_code_arg:
1494 case jit_code_arg_f: case jit_code_arg_d:
1495 case jit_code_va_end:
1496 case jit_code_ret:
1497 case jit_code_retr: case jit_code_reti:
1498 case jit_code_retr_f: case jit_code_reti_f:
1499 case jit_code_retr_d: case jit_code_reti_d:
1500 case jit_code_getarg_c: case jit_code_getarg_uc:
1501 case jit_code_getarg_s: case jit_code_getarg_us:
1502 case jit_code_getarg_i:
1503 #if __WORDSIZE == 64
1504 case jit_code_getarg_ui: case jit_code_getarg_l:
1505 #endif
1506 case jit_code_getarg_f: case jit_code_getarg_d:
1507 case jit_code_putargr: case jit_code_putargi:
1508 case jit_code_putargr_f: case jit_code_putargi_f:
1509 case jit_code_putargr_d: case jit_code_putargi_d:
1510 case jit_code_pushargr: case jit_code_pushargi:
1511 case jit_code_pushargr_f: case jit_code_pushargi_f:
1512 case jit_code_pushargr_d: case jit_code_pushargi_d:
1513 case jit_code_retval_c: case jit_code_retval_uc:
1514 case jit_code_retval_s: case jit_code_retval_us:
1515 case jit_code_retval_i:
1516 #if __WORDSIZE == 64
1517 case jit_code_retval_ui: case jit_code_retval_l:
1518 #endif
1519 case jit_code_retval_f: case jit_code_retval_d:
1520 case jit_code_prepare:
1521 case jit_code_finishr: case jit_code_finishi:
1522 break;
1523 default:
1524 abort();
1525 }
1526 jit_regarg_clr(node, value);
1527 assert(_jitc->regarg == 0 && _jitc->synth == 0);
1528 /* update register live state */
1529 jit_reglive(node);
1530 }
1531 #undef case_brw
1532 #undef case_brr
1533 #undef case_wrr
1534 #undef case_rrw
1535 #undef case_rrr
1536 #undef case_wr
1537 #undef case_rw
1538 #undef case_rr
1539
1540 for (offset = 0; offset < _jitc->patches.offset; offset++) {
1541 node = _jitc->patches.ptr[offset].node;
1542 word = node->code == jit_code_movi ? node->v.n->u.w : node->u.n->u.w;
1543 patch_at(_jitc->patches.ptr[offset].inst, word);
1544 }
1545
1546 jit_flush(_jit->code.ptr, _jit->pc.uc);
1547
1548 return (_jit->code.ptr);
1549 }
1550
1551 #define CODE 1
1552 # include "jit_s390-cpu.c"
1553 # include "jit_s390-fpu.c"
1554 #undef CODE
1555
1556 void
jit_flush(void * fptr,void * tptr)1557 jit_flush(void *fptr, void *tptr)
1558 {
1559 #if defined(__GNUC__)
1560 jit_word_t f, t, s;
1561
1562 s = sysconf(_SC_PAGE_SIZE);
1563 f = (jit_word_t)fptr & -s;
1564 t = (((jit_word_t)tptr) + s - 1) & -s;
1565 __clear_cache((void *)f, (void *)t);
1566 #endif
1567 }
1568
1569 void
_emit_ldxi(jit_state_t * _jit,jit_gpr_t r0,jit_gpr_t r1,jit_word_t i0)1570 _emit_ldxi(jit_state_t *_jit, jit_gpr_t r0, jit_gpr_t r1, jit_word_t i0)
1571 {
1572 ldxi(rn(r0), rn(r1), i0);
1573 }
1574
1575 void
_emit_stxi(jit_state_t * _jit,jit_word_t i0,jit_gpr_t r0,jit_gpr_t r1)1576 _emit_stxi(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_gpr_t r1)
1577 {
1578 stxi(i0, rn(r0), rn(r1));
1579 }
1580
1581 void
_emit_ldxi_d(jit_state_t * _jit,jit_fpr_t r0,jit_gpr_t r1,jit_word_t i0)1582 _emit_ldxi_d(jit_state_t *_jit, jit_fpr_t r0, jit_gpr_t r1, jit_word_t i0)
1583 {
1584 ldxi_d(rn(r0), rn(r1), i0);
1585 }
1586
1587 void
_emit_stxi_d(jit_state_t * _jit,jit_word_t i0,jit_gpr_t r0,jit_fpr_t r1)1588 _emit_stxi_d(jit_state_t *_jit, jit_word_t i0, jit_gpr_t r0, jit_fpr_t r1)
1589 {
1590 stxi_d(i0, rn(r0), rn(r1));
1591 }
1592
1593 static jit_int32_t
_jit_get_reg_pair(jit_state_t * _jit)1594 _jit_get_reg_pair(jit_state_t *_jit)
1595 {
1596 jit_int32_t r1, r2;
1597 /* Try to find a register pair for use with operations that
1598 * require a odd based register pair. Search for the best
1599 * match to avoid spills or at least a valid operation.
1600 */
1601
1602 /* Try non callee save first */
1603 if (jit_reg_free_p(_R0) && jit_reg_free_p(_R1))
1604 r1 = _R0, r2 = _R1;
1605 else if (jit_reg_free_p(_R2) && jit_reg_free_p(_R3))
1606 r1 = _R2, r2 = _R3;
1607 else if (jit_reg_free_p(_R4) && jit_reg_free_p(_R5))
1608 r1 = _R4, r2 = _R5;
1609 /* Try callee save registers */
1610 else if (jit_reg_free_p(_R10) && jit_reg_free_p(_R11))
1611 r1 = _R10, r2 = _R11;
1612 else if (jit_reg_free_p(_R8) && jit_reg_free_p(_R9))
1613 r1 = _R8, r2 = _R9;
1614 else if (jit_reg_free_p(_R6) && jit_reg_free_p(_R7))
1615 r1 = _R6, r2 = _R7;
1616
1617 /* We *must* find a register pair */
1618 else if (jit_reg_free_if_spill_p(_R0) && jit_reg_free_if_spill_p(_R1))
1619 r1 = _R0, r2 = _R1;
1620 else if (jit_reg_free_if_spill_p(_R2) && jit_reg_free_if_spill_p(_R3))
1621 r1 = _R2, r2 = _R3;
1622 else if (jit_reg_free_if_spill_p(_R4) && jit_reg_free_if_spill_p(_R5))
1623 r1 = _R4, r2 = _R5;
1624 else if (jit_reg_free_if_spill_p(_R10) && jit_reg_free_if_spill_p(_R11))
1625 r1 = _R10, r2 = _R11;
1626 else if (jit_reg_free_if_spill_p(_R8) && jit_reg_free_if_spill_p(_R9))
1627 r1 = _R8, r2 = _R9;
1628 else if (jit_reg_free_if_spill_p(_R6) && jit_reg_free_if_spill_p(_R7))
1629 r1 = _R6, r2 = _R7;
1630 else
1631 /* Do not jit_get_reg() all registers to avoid it */
1632 abort();
1633
1634 (void)jit_get_reg(jit_class_gpr|jit_class_named|r1);
1635 (void)jit_get_reg(jit_class_gpr|jit_class_named|r2);
1636
1637 return (r1);
1638 }
1639
1640 static void
_jit_unget_reg_pair(jit_state_t * _jit,jit_int32_t reg)1641 _jit_unget_reg_pair(jit_state_t *_jit, jit_int32_t reg)
1642 {
1643 jit_int32_t r1, r2;
1644 r1 = reg;
1645 switch (r1) {
1646 case _R0: r2 = _R1; break;
1647 case _R2: r2 = _R3; break;
1648 case _R4: r2 = _R5; break;
1649 case _R6: r2 = _R7; break;
1650 case _R8: r2 = _R9; break;
1651 case _R10: r2 = _R11; break;
1652 default: abort();
1653 }
1654 jit_unget_reg(r1);
1655 jit_unget_reg(r2);
1656 }
1657
1658 static jit_int32_t
_jit_get_reg_but_zero(jit_state_t * _jit,jit_int32_t flags)1659 _jit_get_reg_but_zero(jit_state_t *_jit, jit_int32_t flags)
1660 {
1661 jit_int32_t reg;
1662 reg = jit_get_reg(jit_class_gpr);
1663 if (reg == _R0) {
1664 reg = jit_get_reg(jit_class_gpr|flags);
1665 jit_unget_reg(_R0);
1666 }
1667 return (reg);
1668 }
1669
1670 static void
_patch(jit_state_t * _jit,jit_word_t instr,jit_node_t * node)1671 _patch(jit_state_t *_jit, jit_word_t instr, jit_node_t *node)
1672 {
1673 jit_int32_t flag;
1674
1675 assert(node->flag & jit_flag_node);
1676 if (node->code == jit_code_movi)
1677 flag = node->v.n->flag;
1678 else
1679 flag = node->u.n->flag;
1680 assert(!(flag & jit_flag_patch));
1681 if (_jitc->patches.offset >= _jitc->patches.length) {
1682 jit_realloc((jit_pointer_t *)&_jitc->patches.ptr,
1683 _jitc->patches.length * sizeof(jit_patch_t),
1684 (_jitc->patches.length + 1024) * sizeof(jit_patch_t));
1685 _jitc->patches.length += 1024;
1686 }
1687 _jitc->patches.ptr[_jitc->patches.offset].inst = instr;
1688 _jitc->patches.ptr[_jitc->patches.offset].node = node;
1689 ++_jitc->patches.offset;
1690 }
1691