1 //! S390x ISA: binary code emission.
2
3 use crate::binemit::{Reloc, StackMap};
4 use crate::ir::condcodes::IntCC;
5 use crate::ir::MemFlags;
6 use crate::ir::{SourceLoc, TrapCode};
7 use crate::isa::s390x::inst::*;
8 use crate::isa::s390x::settings as s390x_settings;
9 use core::convert::TryFrom;
10 use regalloc::{Reg, RegClass};
11
12 /// Memory addressing mode finalization: convert "special" modes (e.g.,
13 /// generic arbitrary stack offset) into real addressing modes, possibly by
14 /// emitting some helper instructions that come immediately before the use
15 /// of this amode.
mem_finalize( mem: &MemArg, state: &EmitState, have_d12: bool, have_d20: bool, have_pcrel: bool, have_index: bool, ) -> (SmallVec<[Inst; 4]>, MemArg)16 pub fn mem_finalize(
17 mem: &MemArg,
18 state: &EmitState,
19 have_d12: bool,
20 have_d20: bool,
21 have_pcrel: bool,
22 have_index: bool,
23 ) -> (SmallVec<[Inst; 4]>, MemArg) {
24 let mut insts = SmallVec::new();
25
26 // Resolve virtual addressing modes.
27 let mem = match mem {
28 &MemArg::RegOffset { off, .. }
29 | &MemArg::InitialSPOffset { off }
30 | &MemArg::NominalSPOffset { off } => {
31 let base = match mem {
32 &MemArg::RegOffset { reg, .. } => reg,
33 &MemArg::InitialSPOffset { .. } | &MemArg::NominalSPOffset { .. } => stack_reg(),
34 _ => unreachable!(),
35 };
36 let adj = match mem {
37 &MemArg::InitialSPOffset { .. } => {
38 state.initial_sp_offset + state.virtual_sp_offset
39 }
40 &MemArg::NominalSPOffset { .. } => state.virtual_sp_offset,
41 _ => 0,
42 };
43 let off = off + adj;
44
45 if let Some(disp) = UImm12::maybe_from_u64(off as u64) {
46 MemArg::BXD12 {
47 base,
48 index: zero_reg(),
49 disp,
50 flags: mem.get_flags(),
51 }
52 } else if let Some(disp) = SImm20::maybe_from_i64(off) {
53 MemArg::BXD20 {
54 base,
55 index: zero_reg(),
56 disp,
57 flags: mem.get_flags(),
58 }
59 } else {
60 let tmp = writable_spilltmp_reg();
61 assert!(base != tmp.to_reg());
62 insts.extend(Inst::load_constant64(tmp, off as u64));
63 MemArg::reg_plus_reg(base, tmp.to_reg(), mem.get_flags())
64 }
65 }
66 _ => mem.clone(),
67 };
68
69 // If this addressing mode cannot be handled by the instruction, use load-address.
70 let need_load_address = match &mem {
71 &MemArg::Label { .. } | &MemArg::Symbol { .. } if !have_pcrel => true,
72 &MemArg::BXD20 { .. } if !have_d20 => true,
73 &MemArg::BXD12 { index, .. } | &MemArg::BXD20 { index, .. } if !have_index => {
74 index != zero_reg()
75 }
76 _ => false,
77 };
78 let mem = if need_load_address {
79 let flags = mem.get_flags();
80 let tmp = writable_spilltmp_reg();
81 insts.push(Inst::LoadAddr { rd: tmp, mem });
82 MemArg::reg(tmp.to_reg(), flags)
83 } else {
84 mem
85 };
86
87 // Convert 12-bit displacement to 20-bit if required.
88 let mem = match &mem {
89 &MemArg::BXD12 {
90 base,
91 index,
92 disp,
93 flags,
94 } if !have_d12 => {
95 assert!(have_d20);
96 MemArg::BXD20 {
97 base,
98 index,
99 disp: SImm20::from_uimm12(disp),
100 flags,
101 }
102 }
103 _ => mem,
104 };
105
106 (insts, mem)
107 }
108
mem_emit( rd: Reg, mem: &MemArg, opcode_rx: Option<u16>, opcode_rxy: Option<u16>, opcode_ril: Option<u16>, add_trap: bool, sink: &mut MachBuffer<Inst>, emit_info: &EmitInfo, state: &mut EmitState, )109 pub fn mem_emit(
110 rd: Reg,
111 mem: &MemArg,
112 opcode_rx: Option<u16>,
113 opcode_rxy: Option<u16>,
114 opcode_ril: Option<u16>,
115 add_trap: bool,
116 sink: &mut MachBuffer<Inst>,
117 emit_info: &EmitInfo,
118 state: &mut EmitState,
119 ) {
120 let (mem_insts, mem) = mem_finalize(
121 mem,
122 state,
123 opcode_rx.is_some(),
124 opcode_rxy.is_some(),
125 opcode_ril.is_some(),
126 true,
127 );
128 for inst in mem_insts.into_iter() {
129 inst.emit(sink, emit_info, state);
130 }
131
132 if add_trap && mem.can_trap() {
133 let srcloc = state.cur_srcloc();
134 if srcloc != SourceLoc::default() {
135 sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
136 }
137 }
138
139 match &mem {
140 &MemArg::BXD12 {
141 base, index, disp, ..
142 } => {
143 put(
144 sink,
145 &enc_rx(opcode_rx.unwrap(), rd, base, index, disp.bits()),
146 );
147 }
148 &MemArg::BXD20 {
149 base, index, disp, ..
150 } => {
151 put(
152 sink,
153 &enc_rxy(opcode_rxy.unwrap(), rd, base, index, disp.bits()),
154 );
155 }
156 &MemArg::Label { ref target } => {
157 if let Some(l) = target.as_label() {
158 sink.use_label_at_offset(sink.cur_offset(), l, LabelUse::BranchRIL);
159 }
160 put(
161 sink,
162 &enc_ril_b(opcode_ril.unwrap(), rd, target.as_ril_offset_or_zero()),
163 );
164 }
165 &MemArg::Symbol {
166 ref name, offset, ..
167 } => {
168 let reloc = Reloc::S390xPCRel32Dbl;
169 let srcloc = state.cur_srcloc();
170 put_with_reloc(
171 sink,
172 &enc_ril_b(opcode_ril.unwrap(), rd, 0),
173 2,
174 srcloc,
175 reloc,
176 name,
177 offset.into(),
178 );
179 }
180 _ => unreachable!(),
181 }
182 }
183
mem_rs_emit( rd: Reg, rn: Reg, mem: &MemArg, opcode_rs: Option<u16>, opcode_rsy: Option<u16>, add_trap: bool, sink: &mut MachBuffer<Inst>, emit_info: &EmitInfo, state: &mut EmitState, )184 pub fn mem_rs_emit(
185 rd: Reg,
186 rn: Reg,
187 mem: &MemArg,
188 opcode_rs: Option<u16>,
189 opcode_rsy: Option<u16>,
190 add_trap: bool,
191 sink: &mut MachBuffer<Inst>,
192 emit_info: &EmitInfo,
193 state: &mut EmitState,
194 ) {
195 let (mem_insts, mem) = mem_finalize(
196 mem,
197 state,
198 opcode_rs.is_some(),
199 opcode_rsy.is_some(),
200 false,
201 false,
202 );
203 for inst in mem_insts.into_iter() {
204 inst.emit(sink, emit_info, state);
205 }
206
207 if add_trap && mem.can_trap() {
208 let srcloc = state.cur_srcloc();
209 if srcloc != SourceLoc::default() {
210 sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
211 }
212 }
213
214 match &mem {
215 &MemArg::BXD12 {
216 base, index, disp, ..
217 } => {
218 assert!(index == zero_reg());
219 put(sink, &enc_rs(opcode_rs.unwrap(), rd, rn, base, disp.bits()));
220 }
221 &MemArg::BXD20 {
222 base, index, disp, ..
223 } => {
224 assert!(index == zero_reg());
225 put(
226 sink,
227 &enc_rsy(opcode_rsy.unwrap(), rd, rn, base, disp.bits()),
228 );
229 }
230 _ => unreachable!(),
231 }
232 }
233
mem_imm8_emit( imm: u8, mem: &MemArg, opcode_si: u16, opcode_siy: u16, add_trap: bool, sink: &mut MachBuffer<Inst>, emit_info: &EmitInfo, state: &mut EmitState, )234 pub fn mem_imm8_emit(
235 imm: u8,
236 mem: &MemArg,
237 opcode_si: u16,
238 opcode_siy: u16,
239 add_trap: bool,
240 sink: &mut MachBuffer<Inst>,
241 emit_info: &EmitInfo,
242 state: &mut EmitState,
243 ) {
244 let (mem_insts, mem) = mem_finalize(mem, state, true, true, false, false);
245 for inst in mem_insts.into_iter() {
246 inst.emit(sink, emit_info, state);
247 }
248
249 if add_trap && mem.can_trap() {
250 let srcloc = state.cur_srcloc();
251 if srcloc != SourceLoc::default() {
252 sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
253 }
254 }
255
256 match &mem {
257 &MemArg::BXD12 {
258 base, index, disp, ..
259 } => {
260 assert!(index == zero_reg());
261 put(sink, &enc_si(opcode_si, base, disp.bits(), imm));
262 }
263 &MemArg::BXD20 {
264 base, index, disp, ..
265 } => {
266 assert!(index == zero_reg());
267 put(sink, &enc_siy(opcode_siy, base, disp.bits(), imm));
268 }
269 _ => unreachable!(),
270 }
271 }
272
mem_imm16_emit( imm: i16, mem: &MemArg, opcode_sil: u16, add_trap: bool, sink: &mut MachBuffer<Inst>, emit_info: &EmitInfo, state: &mut EmitState, )273 pub fn mem_imm16_emit(
274 imm: i16,
275 mem: &MemArg,
276 opcode_sil: u16,
277 add_trap: bool,
278 sink: &mut MachBuffer<Inst>,
279 emit_info: &EmitInfo,
280 state: &mut EmitState,
281 ) {
282 let (mem_insts, mem) = mem_finalize(mem, state, true, false, false, false);
283 for inst in mem_insts.into_iter() {
284 inst.emit(sink, emit_info, state);
285 }
286
287 if add_trap && mem.can_trap() {
288 let srcloc = state.cur_srcloc();
289 if srcloc != SourceLoc::default() {
290 sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
291 }
292 }
293
294 match &mem {
295 &MemArg::BXD12 {
296 base, index, disp, ..
297 } => {
298 assert!(index == zero_reg());
299 put(sink, &enc_sil(opcode_sil, base, disp.bits(), imm));
300 }
301 _ => unreachable!(),
302 }
303 }
304
305 //=============================================================================
306 // Instructions and subcomponents: emission
307
machreg_to_gpr(m: Reg) -> u8308 fn machreg_to_gpr(m: Reg) -> u8 {
309 assert_eq!(m.get_class(), RegClass::I64);
310 u8::try_from(m.to_real_reg().get_hw_encoding()).unwrap()
311 }
312
machreg_to_fpr(m: Reg) -> u8313 fn machreg_to_fpr(m: Reg) -> u8 {
314 assert_eq!(m.get_class(), RegClass::F64);
315 u8::try_from(m.to_real_reg().get_hw_encoding()).unwrap()
316 }
317
machreg_to_gpr_or_fpr(m: Reg) -> u8318 fn machreg_to_gpr_or_fpr(m: Reg) -> u8 {
319 u8::try_from(m.to_real_reg().get_hw_encoding()).unwrap()
320 }
321
322 /// E-type instructions.
323 ///
324 /// 15
325 /// opcode
326 /// 0
327 ///
enc_e(opcode: u16) -> [u8; 2]328 fn enc_e(opcode: u16) -> [u8; 2] {
329 let mut enc: [u8; 2] = [0; 2];
330 let opcode1 = ((opcode >> 8) & 0xff) as u8;
331 let opcode2 = (opcode & 0xff) as u8;
332
333 enc[0] = opcode1;
334 enc[1] = opcode2;
335 enc
336 }
337
338 /// RIa-type instructions.
339 ///
340 /// 31 23 19 15
341 /// opcode1 r1 opcode2 i2
342 /// 24 20 16 0
343 ///
enc_ri_a(opcode: u16, r1: Reg, i2: u16) -> [u8; 4]344 fn enc_ri_a(opcode: u16, r1: Reg, i2: u16) -> [u8; 4] {
345 let mut enc: [u8; 4] = [0; 4];
346 let opcode1 = ((opcode >> 4) & 0xff) as u8;
347 let opcode2 = (opcode & 0xf) as u8;
348 let r1 = machreg_to_gpr(r1) & 0x0f;
349
350 enc[0] = opcode1;
351 enc[1] = r1 << 4 | opcode2;
352 enc[2..].copy_from_slice(&i2.to_be_bytes());
353 enc
354 }
355
356 /// RIb-type instructions.
357 ///
358 /// 31 23 19 15
359 /// opcode1 r1 opcode2 ri2
360 /// 24 20 16 0
361 ///
enc_ri_b(opcode: u16, r1: Reg, ri2: i32) -> [u8; 4]362 fn enc_ri_b(opcode: u16, r1: Reg, ri2: i32) -> [u8; 4] {
363 let mut enc: [u8; 4] = [0; 4];
364 let opcode1 = ((opcode >> 4) & 0xff) as u8;
365 let opcode2 = (opcode & 0xf) as u8;
366 let r1 = machreg_to_gpr(r1) & 0x0f;
367 let ri2 = ((ri2 >> 1) & 0xffff) as u16;
368
369 enc[0] = opcode1;
370 enc[1] = r1 << 4 | opcode2;
371 enc[2..].copy_from_slice(&ri2.to_be_bytes());
372 enc
373 }
374
375 /// RIc-type instructions.
376 ///
377 /// 31 23 19 15
378 /// opcode1 m1 opcode2 ri2
379 /// 24 20 16 0
380 ///
enc_ri_c(opcode: u16, m1: u8, ri2: i32) -> [u8; 4]381 fn enc_ri_c(opcode: u16, m1: u8, ri2: i32) -> [u8; 4] {
382 let mut enc: [u8; 4] = [0; 4];
383 let opcode1 = ((opcode >> 4) & 0xff) as u8;
384 let opcode2 = (opcode & 0xf) as u8;
385 let m1 = m1 & 0x0f;
386 let ri2 = ((ri2 >> 1) & 0xffff) as u16;
387
388 enc[0] = opcode1;
389 enc[1] = m1 << 4 | opcode2;
390 enc[2..].copy_from_slice(&ri2.to_be_bytes());
391 enc
392 }
393
394 /// RIEa-type instructions.
395 ///
396 /// 47 39 35 31 15 11 7
397 /// opcode1 r1 -- i2 m3 -- opcode2
398 /// 40 36 32 16 12 8 0
399 ///
enc_rie_a(opcode: u16, r1: Reg, i2: u16, m3: u8) -> [u8; 6]400 fn enc_rie_a(opcode: u16, r1: Reg, i2: u16, m3: u8) -> [u8; 6] {
401 let mut enc: [u8; 6] = [0; 6];
402 let opcode1 = ((opcode >> 8) & 0xff) as u8;
403 let opcode2 = (opcode & 0xff) as u8;
404 let r1 = machreg_to_gpr(r1) & 0x0f;
405 let m3 = m3 & 0x0f;
406
407 enc[0] = opcode1;
408 enc[1] = r1 << 4;
409 enc[2..4].copy_from_slice(&i2.to_be_bytes());
410 enc[4] = m3 << 4;
411 enc[5] = opcode2;
412 enc
413 }
414
415 /// RIEd-type instructions.
416 ///
417 /// 47 39 35 31 15 7
418 /// opcode1 r1 r3 i2 -- opcode2
419 /// 40 36 32 16 8 0
420 ///
enc_rie_d(opcode: u16, r1: Reg, r3: Reg, i2: u16) -> [u8; 6]421 fn enc_rie_d(opcode: u16, r1: Reg, r3: Reg, i2: u16) -> [u8; 6] {
422 let mut enc: [u8; 6] = [0; 6];
423 let opcode1 = ((opcode >> 8) & 0xff) as u8;
424 let opcode2 = (opcode & 0xff) as u8;
425 let r1 = machreg_to_gpr(r1) & 0x0f;
426 let r3 = machreg_to_gpr(r3) & 0x0f;
427
428 enc[0] = opcode1;
429 enc[1] = r1 << 4 | r3;
430 enc[2..4].copy_from_slice(&i2.to_be_bytes());
431 enc[5] = opcode2;
432 enc
433 }
434
435 /// RIEg-type instructions.
436 ///
437 /// 47 39 35 31 15 7
438 /// opcode1 r1 m3 i2 -- opcode2
439 /// 40 36 32 16 8 0
440 ///
enc_rie_g(opcode: u16, r1: Reg, i2: u16, m3: u8) -> [u8; 6]441 fn enc_rie_g(opcode: u16, r1: Reg, i2: u16, m3: u8) -> [u8; 6] {
442 let mut enc: [u8; 6] = [0; 6];
443 let opcode1 = ((opcode >> 8) & 0xff) as u8;
444 let opcode2 = (opcode & 0xff) as u8;
445 let r1 = machreg_to_gpr(r1) & 0x0f;
446 let m3 = m3 & 0x0f;
447
448 enc[0] = opcode1;
449 enc[1] = r1 << 4 | m3;
450 enc[2..4].copy_from_slice(&i2.to_be_bytes());
451 enc[5] = opcode2;
452 enc
453 }
454
455 /// RILa-type instructions.
456 ///
457 /// 47 39 35 31
458 /// opcode1 r1 opcode2 i2
459 /// 40 36 32 0
460 ///
enc_ril_a(opcode: u16, r1: Reg, i2: u32) -> [u8; 6]461 fn enc_ril_a(opcode: u16, r1: Reg, i2: u32) -> [u8; 6] {
462 let mut enc: [u8; 6] = [0; 6];
463 let opcode1 = ((opcode >> 4) & 0xff) as u8;
464 let opcode2 = (opcode & 0xf) as u8;
465 let r1 = machreg_to_gpr(r1) & 0x0f;
466
467 enc[0] = opcode1;
468 enc[1] = r1 << 4 | opcode2;
469 enc[2..].copy_from_slice(&i2.to_be_bytes());
470 enc
471 }
472
473 /// RILb-type instructions.
474 ///
475 /// 47 39 35 31
476 /// opcode1 r1 opcode2 ri2
477 /// 40 36 32 0
478 ///
enc_ril_b(opcode: u16, r1: Reg, ri2: u32) -> [u8; 6]479 fn enc_ril_b(opcode: u16, r1: Reg, ri2: u32) -> [u8; 6] {
480 let mut enc: [u8; 6] = [0; 6];
481 let opcode1 = ((opcode >> 4) & 0xff) as u8;
482 let opcode2 = (opcode & 0xf) as u8;
483 let r1 = machreg_to_gpr(r1) & 0x0f;
484
485 enc[0] = opcode1;
486 enc[1] = r1 << 4 | opcode2;
487 enc[2..].copy_from_slice(&ri2.to_be_bytes());
488 enc
489 }
490
491 /// RILc-type instructions.
492 ///
493 /// 47 39 35 31
494 /// opcode1 m1 opcode2 i2
495 /// 40 36 32 0
496 ///
enc_ril_c(opcode: u16, m1: u8, ri2: u32) -> [u8; 6]497 fn enc_ril_c(opcode: u16, m1: u8, ri2: u32) -> [u8; 6] {
498 let mut enc: [u8; 6] = [0; 6];
499 let opcode1 = ((opcode >> 4) & 0xff) as u8;
500 let opcode2 = (opcode & 0xf) as u8;
501 let m1 = m1 & 0x0f;
502
503 enc[0] = opcode1;
504 enc[1] = m1 << 4 | opcode2;
505 enc[2..].copy_from_slice(&ri2.to_be_bytes());
506 enc
507 }
508
509 /// RR-type instructions.
510 ///
511 /// 15 7 3
512 /// opcode r1 r2
513 /// 8 4 0
514 ///
enc_rr(opcode: u16, r1: Reg, r2: Reg) -> [u8; 2]515 fn enc_rr(opcode: u16, r1: Reg, r2: Reg) -> [u8; 2] {
516 let mut enc: [u8; 2] = [0; 2];
517 let opcode = (opcode & 0xff) as u8;
518 let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
519 let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
520
521 enc[0] = opcode;
522 enc[1] = r1 << 4 | r2;
523 enc
524 }
525
526 /// RRD-type instructions.
527 ///
528 /// 31 15 11 7 3
529 /// opcode r1 -- r3 r2
530 /// 16 12 8 4 0
531 ///
enc_rrd(opcode: u16, r1: Reg, r2: Reg, r3: Reg) -> [u8; 4]532 fn enc_rrd(opcode: u16, r1: Reg, r2: Reg, r3: Reg) -> [u8; 4] {
533 let mut enc: [u8; 4] = [0; 4];
534 let opcode1 = ((opcode >> 8) & 0xff) as u8;
535 let opcode2 = (opcode & 0xff) as u8;
536 let r1 = machreg_to_fpr(r1) & 0x0f;
537 let r2 = machreg_to_fpr(r2) & 0x0f;
538 let r3 = machreg_to_fpr(r3) & 0x0f;
539
540 enc[0] = opcode1;
541 enc[1] = opcode2;
542 enc[2] = r1 << 4;
543 enc[3] = r3 << 4 | r2;
544 enc
545 }
546
547 /// RRE-type instructions.
548 ///
549 /// 31 15 7 3
550 /// opcode -- r1 r2
551 /// 16 8 4 0
552 ///
enc_rre(opcode: u16, r1: Reg, r2: Reg) -> [u8; 4]553 fn enc_rre(opcode: u16, r1: Reg, r2: Reg) -> [u8; 4] {
554 let mut enc: [u8; 4] = [0; 4];
555 let opcode1 = ((opcode >> 8) & 0xff) as u8;
556 let opcode2 = (opcode & 0xff) as u8;
557 let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
558 let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
559
560 enc[0] = opcode1;
561 enc[1] = opcode2;
562 enc[3] = r1 << 4 | r2;
563 enc
564 }
565
566 /// RRFa/b-type instructions.
567 ///
568 /// 31 15 11 7 3
569 /// opcode r3 m4 r1 r2
570 /// 16 12 8 4 0
571 ///
enc_rrf_ab(opcode: u16, r1: Reg, r2: Reg, r3: Reg, m4: u8) -> [u8; 4]572 fn enc_rrf_ab(opcode: u16, r1: Reg, r2: Reg, r3: Reg, m4: u8) -> [u8; 4] {
573 let mut enc: [u8; 4] = [0; 4];
574 let opcode1 = ((opcode >> 8) & 0xff) as u8;
575 let opcode2 = (opcode & 0xff) as u8;
576 let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
577 let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
578 let r3 = machreg_to_gpr_or_fpr(r3) & 0x0f;
579 let m4 = m4 & 0x0f;
580
581 enc[0] = opcode1;
582 enc[1] = opcode2;
583 enc[2] = r3 << 4 | m4;
584 enc[3] = r1 << 4 | r2;
585 enc
586 }
587
588 /// RRFc/d/e-type instructions.
589 ///
590 /// 31 15 11 7 3
591 /// opcode m3 m4 r1 r2
592 /// 16 12 8 4 0
593 ///
enc_rrf_cde(opcode: u16, r1: Reg, r2: Reg, m3: u8, m4: u8) -> [u8; 4]594 fn enc_rrf_cde(opcode: u16, r1: Reg, r2: Reg, m3: u8, m4: u8) -> [u8; 4] {
595 let mut enc: [u8; 4] = [0; 4];
596 let opcode1 = ((opcode >> 8) & 0xff) as u8;
597 let opcode2 = (opcode & 0xff) as u8;
598 let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
599 let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
600 let m3 = m3 & 0x0f;
601 let m4 = m4 & 0x0f;
602
603 enc[0] = opcode1;
604 enc[1] = opcode2;
605 enc[2] = m3 << 4 | m4;
606 enc[3] = r1 << 4 | r2;
607 enc
608 }
609
610 /// RS-type instructions.
611 ///
612 /// 31 23 19 15 11
613 /// opcode r1 r3 b2 d2
614 /// 24 20 16 12 0
615 ///
enc_rs(opcode: u16, r1: Reg, r3: Reg, b2: Reg, d2: u32) -> [u8; 4]616 fn enc_rs(opcode: u16, r1: Reg, r3: Reg, b2: Reg, d2: u32) -> [u8; 4] {
617 let opcode = (opcode & 0xff) as u8;
618 let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
619 let r3 = machreg_to_gpr_or_fpr(r3) & 0x0f;
620 let b2 = machreg_to_gpr(b2) & 0x0f;
621 let d2_lo = (d2 & 0xff) as u8;
622 let d2_hi = ((d2 >> 8) & 0x0f) as u8;
623
624 let mut enc: [u8; 4] = [0; 4];
625 enc[0] = opcode;
626 enc[1] = r1 << 4 | r3;
627 enc[2] = b2 << 4 | d2_hi;
628 enc[3] = d2_lo;
629 enc
630 }
631
632 /// RSY-type instructions.
633 ///
634 /// 47 39 35 31 27 15 7
635 /// opcode1 r1 r3 b2 dl2 dh2 opcode2
636 /// 40 36 32 28 16 8 0
637 ///
enc_rsy(opcode: u16, r1: Reg, r3: Reg, b2: Reg, d2: u32) -> [u8; 6]638 fn enc_rsy(opcode: u16, r1: Reg, r3: Reg, b2: Reg, d2: u32) -> [u8; 6] {
639 let opcode1 = ((opcode >> 8) & 0xff) as u8;
640 let opcode2 = (opcode & 0xff) as u8;
641 let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
642 let r3 = machreg_to_gpr_or_fpr(r3) & 0x0f;
643 let b2 = machreg_to_gpr(b2) & 0x0f;
644 let dl2_lo = (d2 & 0xff) as u8;
645 let dl2_hi = ((d2 >> 8) & 0x0f) as u8;
646 let dh2 = ((d2 >> 12) & 0xff) as u8;
647
648 let mut enc: [u8; 6] = [0; 6];
649 enc[0] = opcode1;
650 enc[1] = r1 << 4 | r3;
651 enc[2] = b2 << 4 | dl2_hi;
652 enc[3] = dl2_lo;
653 enc[4] = dh2;
654 enc[5] = opcode2;
655 enc
656 }
657
658 /// RX-type instructions.
659 ///
660 /// 31 23 19 15 11
661 /// opcode r1 x2 b2 d2
662 /// 24 20 16 12 0
663 ///
enc_rx(opcode: u16, r1: Reg, b2: Reg, x2: Reg, d2: u32) -> [u8; 4]664 fn enc_rx(opcode: u16, r1: Reg, b2: Reg, x2: Reg, d2: u32) -> [u8; 4] {
665 let opcode = (opcode & 0xff) as u8;
666 let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
667 let b2 = machreg_to_gpr(b2) & 0x0f;
668 let x2 = machreg_to_gpr(x2) & 0x0f;
669 let d2_lo = (d2 & 0xff) as u8;
670 let d2_hi = ((d2 >> 8) & 0x0f) as u8;
671
672 let mut enc: [u8; 4] = [0; 4];
673 enc[0] = opcode;
674 enc[1] = r1 << 4 | x2;
675 enc[2] = b2 << 4 | d2_hi;
676 enc[3] = d2_lo;
677 enc
678 }
679
680 /// RXY-type instructions.
681 ///
682 /// 47 39 35 31 27 15 7
683 /// opcode1 r1 x2 b2 dl2 dh2 opcode2
684 /// 40 36 32 28 16 8 0
685 ///
enc_rxy(opcode: u16, r1: Reg, b2: Reg, x2: Reg, d2: u32) -> [u8; 6]686 fn enc_rxy(opcode: u16, r1: Reg, b2: Reg, x2: Reg, d2: u32) -> [u8; 6] {
687 let opcode1 = ((opcode >> 8) & 0xff) as u8;
688 let opcode2 = (opcode & 0xff) as u8;
689 let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
690 let b2 = machreg_to_gpr(b2) & 0x0f;
691 let x2 = machreg_to_gpr(x2) & 0x0f;
692 let dl2_lo = (d2 & 0xff) as u8;
693 let dl2_hi = ((d2 >> 8) & 0x0f) as u8;
694 let dh2 = ((d2 >> 12) & 0xff) as u8;
695
696 let mut enc: [u8; 6] = [0; 6];
697 enc[0] = opcode1;
698 enc[1] = r1 << 4 | x2;
699 enc[2] = b2 << 4 | dl2_hi;
700 enc[3] = dl2_lo;
701 enc[4] = dh2;
702 enc[5] = opcode2;
703 enc
704 }
705
706 /// SI-type instructions.
707 ///
708 /// 31 23 15 11
709 /// opcode i2 b1 d1
710 /// 24 16 12 0
711 ///
enc_si(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 4]712 fn enc_si(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 4] {
713 let opcode = (opcode & 0xff) as u8;
714 let b1 = machreg_to_gpr(b1) & 0x0f;
715 let d1_lo = (d1 & 0xff) as u8;
716 let d1_hi = ((d1 >> 8) & 0x0f) as u8;
717
718 let mut enc: [u8; 4] = [0; 4];
719 enc[0] = opcode;
720 enc[1] = i2;
721 enc[2] = b1 << 4 | d1_hi;
722 enc[3] = d1_lo;
723 enc
724 }
725
726 /// SIL-type instructions.
727 ///
728 /// 47 31 27 15
729 /// opcode b1 d1 i2
730 /// 32 28 16 0
731 ///
enc_sil(opcode: u16, b1: Reg, d1: u32, i2: i16) -> [u8; 6]732 fn enc_sil(opcode: u16, b1: Reg, d1: u32, i2: i16) -> [u8; 6] {
733 let opcode1 = ((opcode >> 8) & 0xff) as u8;
734 let opcode2 = (opcode & 0xff) as u8;
735 let b1 = machreg_to_gpr(b1) & 0x0f;
736 let d1_lo = (d1 & 0xff) as u8;
737 let d1_hi = ((d1 >> 8) & 0x0f) as u8;
738
739 let mut enc: [u8; 6] = [0; 6];
740 enc[0] = opcode1;
741 enc[1] = opcode2;
742 enc[2] = b1 << 4 | d1_hi;
743 enc[3] = d1_lo;
744 enc[4..].copy_from_slice(&i2.to_be_bytes());
745 enc
746 }
747
748 /// SIY-type instructions.
749 ///
750 /// 47 39 31 27 15 7
751 /// opcode1 i2 b1 dl1 dh1 opcode2
752 /// 40 32 28 16 8 0
753 ///
enc_siy(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 6]754 fn enc_siy(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 6] {
755 let opcode1 = ((opcode >> 8) & 0xff) as u8;
756 let opcode2 = (opcode & 0xff) as u8;
757 let b1 = machreg_to_gpr(b1) & 0x0f;
758 let dl1_lo = (d1 & 0xff) as u8;
759 let dl1_hi = ((d1 >> 8) & 0x0f) as u8;
760 let dh1 = ((d1 >> 12) & 0xff) as u8;
761
762 let mut enc: [u8; 6] = [0; 6];
763 enc[0] = opcode1;
764 enc[1] = i2;
765 enc[2] = b1 << 4 | dl1_hi;
766 enc[3] = dl1_lo;
767 enc[4] = dh1;
768 enc[5] = opcode2;
769 enc
770 }
771
772 /// VRR-type instructions.
773 ///
774 /// 47 39 35 31 27 23 19 15 11 7
775 /// opcode1 v1 v2 v3 - m6 m5 m4 rxb opcode2
776 /// 40 36 32 28 24 20 16 12 8 0
777 ///
enc_vrr(opcode: u16, v1: Reg, v2: Reg, v3: Reg, m4: u8, m5: u8, m6: u8) -> [u8; 6]778 fn enc_vrr(opcode: u16, v1: Reg, v2: Reg, v3: Reg, m4: u8, m5: u8, m6: u8) -> [u8; 6] {
779 let opcode1 = ((opcode >> 8) & 0xff) as u8;
780 let opcode2 = (opcode & 0xff) as u8;
781 let rxb = 0; // FIXME
782 let v1 = machreg_to_fpr(v1) & 0x0f; // FIXME
783 let v2 = machreg_to_fpr(v2) & 0x0f; // FIXME
784 let v3 = machreg_to_fpr(v3) & 0x0f; // FIXME
785 let m4 = m4 & 0x0f;
786 let m5 = m5 & 0x0f;
787 let m6 = m6 & 0x0f;
788
789 let mut enc: [u8; 6] = [0; 6];
790 enc[0] = opcode1;
791 enc[1] = v1 << 4 | v2;
792 enc[2] = v3 << 4;
793 enc[3] = m6 << 4 | m5;
794 enc[4] = m4 << 4 | rxb;
795 enc[5] = opcode2;
796 enc
797 }
798
799 /// VRX-type instructions.
800 ///
801 /// 47 39 35 31 27 15 11 7
802 /// opcode1 v1 x2 b2 d2 m3 rxb opcode2
803 /// 40 36 32 28 16 12 8 0
804 ///
enc_vrx(opcode: u16, v1: Reg, b2: Reg, x2: Reg, d2: u32, m3: u8) -> [u8; 6]805 fn enc_vrx(opcode: u16, v1: Reg, b2: Reg, x2: Reg, d2: u32, m3: u8) -> [u8; 6] {
806 let opcode1 = ((opcode >> 8) & 0xff) as u8;
807 let opcode2 = (opcode & 0xff) as u8;
808 let rxb = 0; // FIXME
809 let v1 = machreg_to_fpr(v1) & 0x0f; // FIXME
810 let b2 = machreg_to_gpr(b2) & 0x0f;
811 let x2 = machreg_to_gpr(x2) & 0x0f;
812 let d2_lo = (d2 & 0xff) as u8;
813 let d2_hi = ((d2 >> 8) & 0x0f) as u8;
814 let m3 = m3 & 0x0f;
815
816 let mut enc: [u8; 6] = [0; 6];
817 enc[0] = opcode1;
818 enc[1] = v1 << 4 | x2;
819 enc[2] = b2 << 4 | d2_hi;
820 enc[3] = d2_lo;
821 enc[4] = m3 << 4 | rxb;
822 enc[5] = opcode2;
823 enc
824 }
825
826 /// Emit encoding to sink.
put(sink: &mut MachBuffer<Inst>, enc: &[u8])827 fn put(sink: &mut MachBuffer<Inst>, enc: &[u8]) {
828 for byte in enc {
829 sink.put1(*byte);
830 }
831 }
832
833 /// Emit encoding to sink, adding a trap on the last byte.
put_with_trap(sink: &mut MachBuffer<Inst>, enc: &[u8], srcloc: SourceLoc, trap_code: TrapCode)834 fn put_with_trap(sink: &mut MachBuffer<Inst>, enc: &[u8], srcloc: SourceLoc, trap_code: TrapCode) {
835 let len = enc.len();
836 for i in 0..len - 1 {
837 sink.put1(enc[i]);
838 }
839 sink.add_trap(srcloc, trap_code);
840 sink.put1(enc[len - 1]);
841 }
842
843 /// Emit encoding to sink, adding a relocation at byte offset.
put_with_reloc( sink: &mut MachBuffer<Inst>, enc: &[u8], offset: usize, ri2_srcloc: SourceLoc, ri2_reloc: Reloc, ri2_name: &ExternalName, ri2_offset: i64, )844 fn put_with_reloc(
845 sink: &mut MachBuffer<Inst>,
846 enc: &[u8],
847 offset: usize,
848 ri2_srcloc: SourceLoc,
849 ri2_reloc: Reloc,
850 ri2_name: &ExternalName,
851 ri2_offset: i64,
852 ) {
853 let len = enc.len();
854 for i in 0..offset {
855 sink.put1(enc[i]);
856 }
857 sink.add_reloc(ri2_srcloc, ri2_reloc, ri2_name, ri2_offset + offset as i64);
858 for i in offset..len {
859 sink.put1(enc[i]);
860 }
861 }
862
863 /// State carried between emissions of a sequence of instructions.
864 #[derive(Default, Clone, Debug)]
865 pub struct EmitState {
866 pub(crate) initial_sp_offset: i64,
867 pub(crate) virtual_sp_offset: i64,
868 /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
869 stack_map: Option<StackMap>,
870 /// Current source-code location corresponding to instruction to be emitted.
871 cur_srcloc: SourceLoc,
872 }
873
874 impl MachInstEmitState<Inst> for EmitState {
new(abi: &dyn ABICallee<I = Inst>) -> Self875 fn new(abi: &dyn ABICallee<I = Inst>) -> Self {
876 EmitState {
877 virtual_sp_offset: 0,
878 initial_sp_offset: abi.frame_size() as i64,
879 stack_map: None,
880 cur_srcloc: SourceLoc::default(),
881 }
882 }
883
pre_safepoint(&mut self, stack_map: StackMap)884 fn pre_safepoint(&mut self, stack_map: StackMap) {
885 self.stack_map = Some(stack_map);
886 }
887
pre_sourceloc(&mut self, srcloc: SourceLoc)888 fn pre_sourceloc(&mut self, srcloc: SourceLoc) {
889 self.cur_srcloc = srcloc;
890 }
891 }
892
893 impl EmitState {
take_stack_map(&mut self) -> Option<StackMap>894 fn take_stack_map(&mut self) -> Option<StackMap> {
895 self.stack_map.take()
896 }
897
clear_post_insn(&mut self)898 fn clear_post_insn(&mut self) {
899 self.stack_map = None;
900 }
901
cur_srcloc(&self) -> SourceLoc902 fn cur_srcloc(&self) -> SourceLoc {
903 self.cur_srcloc
904 }
905 }
906
907 /// Constant state used during function compilation.
908 pub struct EmitInfo {
909 flags: settings::Flags,
910 isa_flags: s390x_settings::Flags,
911 }
912
913 impl EmitInfo {
new(flags: settings::Flags, isa_flags: s390x_settings::Flags) -> Self914 pub(crate) fn new(flags: settings::Flags, isa_flags: s390x_settings::Flags) -> Self {
915 Self { flags, isa_flags }
916 }
917 }
918
919 impl MachInstEmitInfo for EmitInfo {
flags(&self) -> &settings::Flags920 fn flags(&self) -> &settings::Flags {
921 &self.flags
922 }
923 }
924
925 impl MachInstEmit for Inst {
926 type State = EmitState;
927 type Info = EmitInfo;
928
emit(&self, sink: &mut MachBuffer<Inst>, emit_info: &Self::Info, state: &mut EmitState)929 fn emit(&self, sink: &mut MachBuffer<Inst>, emit_info: &Self::Info, state: &mut EmitState) {
930 // Verify that we can emit this Inst in the current ISA
931 let matches_isa_flags = |iset_requirement: &InstructionSet| -> bool {
932 match iset_requirement {
933 // Baseline ISA is z14
934 InstructionSet::Base => true,
935 // Miscellaneous-Instruction-Extensions Facility 2 (z15)
936 InstructionSet::MIE2 => emit_info.isa_flags.has_mie2(),
937 // Vector-Enhancements Facility 2 (z15)
938 InstructionSet::VXRS_EXT2 => emit_info.isa_flags.has_vxrs_ext2(),
939 }
940 };
941 let isa_requirements = self.available_in_isa();
942 if !matches_isa_flags(&isa_requirements) {
943 panic!(
944 "Cannot emit inst '{:?}' for target; failed to match ISA requirements: {:?}",
945 self, isa_requirements
946 )
947 }
948
949 // N.B.: we *must* not exceed the "worst-case size" used to compute
950 // where to insert islands, except when islands are explicitly triggered
951 // (with an `EmitIsland`). We check this in debug builds. This is `mut`
952 // to allow disabling the check for `JTSequence`, which is always
953 // emitted following an `EmitIsland`.
954 let mut start_off = sink.cur_offset();
955
956 match self {
957 &Inst::AluRRR { alu_op, rd, rn, rm } => {
958 let (opcode, have_rr) = match alu_op {
959 ALUOp::Add32 => (0xb9f8, true), // ARK
960 ALUOp::Add64 => (0xb9e8, true), // AGRK
961 ALUOp::Sub32 => (0xb9f9, true), // SRK
962 ALUOp::Sub64 => (0xb9e9, true), // SGRK
963 ALUOp::Mul32 => (0xb9fd, true), // MSRKC
964 ALUOp::Mul64 => (0xb9ed, true), // MSGRKC
965 ALUOp::And32 => (0xb9f4, true), // NRK
966 ALUOp::And64 => (0xb9e4, true), // NGRK
967 ALUOp::Orr32 => (0xb9f6, true), // ORK
968 ALUOp::Orr64 => (0xb9e6, true), // OGRK
969 ALUOp::Xor32 => (0xb9f7, true), // XRK
970 ALUOp::Xor64 => (0xb9e7, true), // XGRK
971 ALUOp::AndNot32 => (0xb974, false), // NNRK
972 ALUOp::AndNot64 => (0xb964, false), // NNGRK
973 ALUOp::OrrNot32 => (0xb976, false), // NORK
974 ALUOp::OrrNot64 => (0xb966, false), // NOGRK
975 ALUOp::XorNot32 => (0xb977, false), // NXRK
976 ALUOp::XorNot64 => (0xb967, false), // NXGRK
977 _ => unreachable!(),
978 };
979 if have_rr && rd.to_reg() == rn {
980 let inst = Inst::AluRR { alu_op, rd, rm };
981 inst.emit(sink, emit_info, state);
982 } else {
983 put(sink, &enc_rrf_ab(opcode, rd.to_reg(), rn, rm, 0));
984 }
985 }
986 &Inst::AluRRSImm16 {
987 alu_op,
988 rd,
989 rn,
990 imm,
991 } => {
992 if rd.to_reg() == rn {
993 let inst = Inst::AluRSImm16 { alu_op, rd, imm };
994 inst.emit(sink, emit_info, state);
995 } else {
996 let opcode = match alu_op {
997 ALUOp::Add32 => 0xecd8, // AHIK
998 ALUOp::Add64 => 0xecd9, // AGHIK
999 _ => unreachable!(),
1000 };
1001 put(sink, &enc_rie_d(opcode, rd.to_reg(), rn, imm as u16));
1002 }
1003 }
1004 &Inst::AluRR { alu_op, rd, rm } => {
1005 let (opcode, is_rre) = match alu_op {
1006 ALUOp::Add32 => (0x1a, false), // AR
1007 ALUOp::Add64 => (0xb908, true), // AGR
1008 ALUOp::Add64Ext32 => (0xb918, true), // AGFR
1009 ALUOp::Sub32 => (0x1b, false), // SR
1010 ALUOp::Sub64 => (0xb909, true), // SGR
1011 ALUOp::Sub64Ext32 => (0xb919, true), // SGFR
1012 ALUOp::Mul32 => (0xb252, true), // MSR
1013 ALUOp::Mul64 => (0xb90c, true), // MSGR
1014 ALUOp::Mul64Ext32 => (0xb91c, true), // MSGFR
1015 ALUOp::And32 => (0x14, false), // NR
1016 ALUOp::And64 => (0xb980, true), // NGR
1017 ALUOp::Orr32 => (0x16, false), // OR
1018 ALUOp::Orr64 => (0xb981, true), // OGR
1019 ALUOp::Xor32 => (0x17, false), // XR
1020 ALUOp::Xor64 => (0xb982, true), // XGR
1021 _ => unreachable!(),
1022 };
1023 if is_rre {
1024 put(sink, &enc_rre(opcode, rd.to_reg(), rm));
1025 } else {
1026 put(sink, &enc_rr(opcode, rd.to_reg(), rm));
1027 }
1028 }
1029 &Inst::AluRX {
1030 alu_op,
1031 rd,
1032 ref mem,
1033 } => {
1034 let (opcode_rx, opcode_rxy) = match alu_op {
1035 ALUOp::Add32 => (Some(0x5a), Some(0xe35a)), // A(Y)
1036 ALUOp::Add32Ext16 => (Some(0x4a), Some(0xe34a)), // AH(Y)
1037 ALUOp::Add64 => (None, Some(0xe308)), // AG
1038 ALUOp::Add64Ext16 => (None, Some(0xe338)), // AGH
1039 ALUOp::Add64Ext32 => (None, Some(0xe318)), // AGF
1040 ALUOp::Sub32 => (Some(0x5b), Some(0xe35b)), // S(Y)
1041 ALUOp::Sub32Ext16 => (Some(0x4b), Some(0xe37b)), // SH(Y)
1042 ALUOp::Sub64 => (None, Some(0xe309)), // SG
1043 ALUOp::Sub64Ext16 => (None, Some(0xe339)), // SGH
1044 ALUOp::Sub64Ext32 => (None, Some(0xe319)), // SGF
1045 ALUOp::Mul32 => (Some(0x71), Some(0xe351)), // MS(Y)
1046 ALUOp::Mul32Ext16 => (Some(0x4c), Some(0xe37c)), // MH(Y)
1047 ALUOp::Mul64 => (None, Some(0xe30c)), // MSG
1048 ALUOp::Mul64Ext16 => (None, Some(0xe33c)), // MSH
1049 ALUOp::Mul64Ext32 => (None, Some(0xe31c)), // MSGF
1050 ALUOp::And32 => (Some(0x54), Some(0xe354)), // N(Y)
1051 ALUOp::And64 => (None, Some(0xe380)), // NG
1052 ALUOp::Orr32 => (Some(0x56), Some(0xe356)), // O(Y)
1053 ALUOp::Orr64 => (None, Some(0xe381)), // OG
1054 ALUOp::Xor32 => (Some(0x57), Some(0xe357)), // X(Y)
1055 ALUOp::Xor64 => (None, Some(0xe382)), // XG
1056 _ => unreachable!(),
1057 };
1058 let rd = rd.to_reg();
1059 mem_emit(
1060 rd, mem, opcode_rx, opcode_rxy, None, true, sink, emit_info, state,
1061 );
1062 }
1063 &Inst::AluRSImm16 { alu_op, rd, imm } => {
1064 let opcode = match alu_op {
1065 ALUOp::Add32 => 0xa7a, // AHI
1066 ALUOp::Add64 => 0xa7b, // AGHI
1067 ALUOp::Mul32 => 0xa7c, // MHI
1068 ALUOp::Mul64 => 0xa7d, // MGHI
1069 _ => unreachable!(),
1070 };
1071 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm as u16));
1072 }
1073 &Inst::AluRSImm32 { alu_op, rd, imm } => {
1074 let opcode = match alu_op {
1075 ALUOp::Add32 => 0xc29, // AFI
1076 ALUOp::Add64 => 0xc28, // AGFI
1077 ALUOp::Mul32 => 0xc21, // MSFI
1078 ALUOp::Mul64 => 0xc20, // MSGFI
1079 _ => unreachable!(),
1080 };
1081 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm as u32));
1082 }
1083 &Inst::AluRUImm32 { alu_op, rd, imm } => {
1084 let opcode = match alu_op {
1085 ALUOp::Add32 => 0xc2b, // ALFI
1086 ALUOp::Add64 => 0xc2a, // ALGFI
1087 ALUOp::Sub32 => 0xc25, // SLFI
1088 ALUOp::Sub64 => 0xc24, // SLGFI
1089 _ => unreachable!(),
1090 };
1091 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm));
1092 }
1093 &Inst::AluRUImm16Shifted { alu_op, rd, imm } => {
1094 let opcode = match (alu_op, imm.shift) {
1095 (ALUOp::And32, 0) => 0xa57, // NILL
1096 (ALUOp::And32, 1) => 0xa56, // NILH
1097 (ALUOp::And64, 0) => 0xa57, // NILL
1098 (ALUOp::And64, 1) => 0xa56, // NILH
1099 (ALUOp::And64, 2) => 0xa55, // NIHL
1100 (ALUOp::And64, 3) => 0xa54, // NIHL
1101 (ALUOp::Orr32, 0) => 0xa5b, // OILL
1102 (ALUOp::Orr32, 1) => 0xa5a, // OILH
1103 (ALUOp::Orr64, 0) => 0xa5b, // OILL
1104 (ALUOp::Orr64, 1) => 0xa5a, // OILH
1105 (ALUOp::Orr64, 2) => 0xa59, // OIHL
1106 (ALUOp::Orr64, 3) => 0xa58, // OIHH
1107 _ => unreachable!(),
1108 };
1109 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm.bits));
1110 }
1111 &Inst::AluRUImm32Shifted { alu_op, rd, imm } => {
1112 let opcode = match (alu_op, imm.shift) {
1113 (ALUOp::And32, 0) => 0xc0b, // NILF
1114 (ALUOp::And64, 0) => 0xc0b, // NILF
1115 (ALUOp::And64, 1) => 0xc0a, // NIHF
1116 (ALUOp::Orr32, 0) => 0xc0d, // OILF
1117 (ALUOp::Orr64, 0) => 0xc0d, // OILF
1118 (ALUOp::Orr64, 1) => 0xc0c, // OILF
1119 (ALUOp::Xor32, 0) => 0xc07, // XILF
1120 (ALUOp::Xor64, 0) => 0xc07, // XILF
1121 (ALUOp::Xor64, 1) => 0xc06, // XILH
1122 _ => unreachable!(),
1123 };
1124 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits));
1125 }
1126
1127 &Inst::SMulWide { rn, rm } => {
1128 let opcode = 0xb9ec; // MGRK
1129 put(sink, &enc_rrf_ab(opcode, gpr(0), rn, rm, 0));
1130 }
1131 &Inst::UMulWide { rn } => {
1132 let opcode = 0xb986; // MLGR
1133 put(sink, &enc_rre(opcode, gpr(0), rn));
1134 }
1135 &Inst::SDivMod32 { rn } => {
1136 let opcode = 0xb91d; // DSGFR
1137 let srcloc = state.cur_srcloc();
1138 let trap_code = TrapCode::IntegerDivisionByZero;
1139 put_with_trap(sink, &enc_rre(opcode, gpr(0), rn), srcloc, trap_code);
1140 }
1141 &Inst::SDivMod64 { rn } => {
1142 let opcode = 0xb90d; // DSGR
1143 let srcloc = state.cur_srcloc();
1144 let trap_code = TrapCode::IntegerDivisionByZero;
1145 put_with_trap(sink, &enc_rre(opcode, gpr(0), rn), srcloc, trap_code);
1146 }
1147 &Inst::UDivMod32 { rn } => {
1148 let opcode = 0xb997; // DLR
1149 let srcloc = state.cur_srcloc();
1150 let trap_code = TrapCode::IntegerDivisionByZero;
1151 put_with_trap(sink, &enc_rre(opcode, gpr(0), rn), srcloc, trap_code);
1152 }
1153 &Inst::UDivMod64 { rn } => {
1154 let opcode = 0xb987; // DLGR
1155 let srcloc = state.cur_srcloc();
1156 let trap_code = TrapCode::IntegerDivisionByZero;
1157 put_with_trap(sink, &enc_rre(opcode, gpr(0), rn), srcloc, trap_code);
1158 }
1159 &Inst::Flogr { rn } => {
1160 let opcode = 0xb983; // FLOGR
1161 put(sink, &enc_rre(opcode, gpr(0), rn));
1162 }
1163
1164 &Inst::ShiftRR {
1165 shift_op,
1166 rd,
1167 rn,
1168 shift_imm,
1169 shift_reg,
1170 } => {
1171 let opcode = match shift_op {
1172 ShiftOp::RotL32 => 0xeb1d, // RLL
1173 ShiftOp::RotL64 => 0xeb1c, // RLLG
1174 ShiftOp::LShL32 => 0xebdf, // SLLK (SLL ?)
1175 ShiftOp::LShL64 => 0xeb0d, // SLLG
1176 ShiftOp::LShR32 => 0xebde, // SRLK (SRL ?)
1177 ShiftOp::LShR64 => 0xeb0c, // SRLG
1178 ShiftOp::AShR32 => 0xebdc, // SRAK (SRA ?)
1179 ShiftOp::AShR64 => 0xeb0a, // SRAG
1180 };
1181 let shift_reg = match shift_reg {
1182 Some(reg) => reg,
1183 None => zero_reg(),
1184 };
1185 put(
1186 sink,
1187 &enc_rsy(opcode, rd.to_reg(), rn, shift_reg, shift_imm.bits()),
1188 );
1189 }
1190
1191 &Inst::UnaryRR { op, rd, rn } => {
1192 match op {
1193 UnaryOp::Abs32 => {
1194 let opcode = 0x10; // LPR
1195 put(sink, &enc_rr(opcode, rd.to_reg(), rn));
1196 }
1197 UnaryOp::Abs64 => {
1198 let opcode = 0xb900; // LPGR
1199 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1200 }
1201 UnaryOp::Abs64Ext32 => {
1202 let opcode = 0xb910; // LPGFR
1203 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1204 }
1205 UnaryOp::Neg32 => {
1206 let opcode = 0x13; // LCR
1207 put(sink, &enc_rr(opcode, rd.to_reg(), rn));
1208 }
1209 UnaryOp::Neg64 => {
1210 let opcode = 0xb903; // LCGR
1211 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1212 }
1213 UnaryOp::Neg64Ext32 => {
1214 let opcode = 0xb913; // LCGFR
1215 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1216 }
1217 UnaryOp::PopcntByte => {
1218 let opcode = 0xb9e1; // POPCNT
1219 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, 0, 0));
1220 }
1221 UnaryOp::PopcntReg => {
1222 let opcode = 0xb9e1; // POPCNT
1223 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, 8, 0));
1224 }
1225 }
1226 }
1227
1228 &Inst::Extend {
1229 rd,
1230 rn,
1231 signed,
1232 from_bits,
1233 to_bits,
1234 } => {
1235 let opcode = match (signed, from_bits, to_bits) {
1236 (_, 1, 32) => 0xb926, // LBR
1237 (_, 1, 64) => 0xb906, // LGBR
1238 (false, 8, 32) => 0xb994, // LLCR
1239 (false, 8, 64) => 0xb984, // LLGCR
1240 (true, 8, 32) => 0xb926, // LBR
1241 (true, 8, 64) => 0xb906, // LGBR
1242 (false, 16, 32) => 0xb995, // LLHR
1243 (false, 16, 64) => 0xb985, // LLGHR
1244 (true, 16, 32) => 0xb927, // LHR
1245 (true, 16, 64) => 0xb907, // LGHR
1246 (false, 32, 64) => 0xb916, // LLGFR
1247 (true, 32, 64) => 0xb914, // LGFR
1248 _ => panic!(
1249 "Unsupported extend combination: signed = {}, from_bits = {}, to_bits = {}",
1250 signed, from_bits, to_bits
1251 ),
1252 };
1253 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1254 }
1255
1256 &Inst::CmpRR { op, rn, rm } => {
1257 let (opcode, is_rre) = match op {
1258 CmpOp::CmpS32 => (0x19, false), // CR
1259 CmpOp::CmpS64 => (0xb920, true), // CGR
1260 CmpOp::CmpS64Ext32 => (0xb930, true), // CGFR
1261 CmpOp::CmpL32 => (0x15, false), // CLR
1262 CmpOp::CmpL64 => (0xb921, true), // CLGR
1263 CmpOp::CmpL64Ext32 => (0xb931, true), // CLGFR
1264 _ => unreachable!(),
1265 };
1266 if is_rre {
1267 put(sink, &enc_rre(opcode, rn, rm));
1268 } else {
1269 put(sink, &enc_rr(opcode, rn, rm));
1270 }
1271 }
1272 &Inst::CmpRX { op, rn, ref mem } => {
1273 let (opcode_rx, opcode_rxy, opcode_ril) = match op {
1274 CmpOp::CmpS32 => (Some(0x59), Some(0xe359), Some(0xc6d)), // C(Y), CRL
1275 CmpOp::CmpS32Ext16 => (Some(0x49), Some(0xe379), Some(0xc65)), // CH(Y), CHRL
1276 CmpOp::CmpS64 => (None, Some(0xe320), Some(0xc68)), // CG, CGRL
1277 CmpOp::CmpS64Ext16 => (None, Some(0xe334), Some(0xc64)), // CGH, CGHRL
1278 CmpOp::CmpS64Ext32 => (None, Some(0xe330), Some(0xc6c)), // CGF, CGFRL
1279 CmpOp::CmpL32 => (Some(0x55), Some(0xe355), Some(0xc6f)), // CL(Y), CLRL
1280 CmpOp::CmpL32Ext16 => (None, None, Some(0xc67)), // CLHRL
1281 CmpOp::CmpL64 => (None, Some(0xe321), Some(0xc6a)), // CLG, CLGRL
1282 CmpOp::CmpL64Ext16 => (None, None, Some(0xc66)), // CLGHRL
1283 CmpOp::CmpL64Ext32 => (None, Some(0xe331), Some(0xc6e)), // CLGF, CLGFRL
1284 };
1285 mem_emit(
1286 rn, mem, opcode_rx, opcode_rxy, opcode_ril, true, sink, emit_info, state,
1287 );
1288 }
1289 &Inst::CmpRSImm16 { op, rn, imm } => {
1290 let opcode = match op {
1291 CmpOp::CmpS32 => 0xa7e, // CHI
1292 CmpOp::CmpS64 => 0xa7f, // CGHI
1293 _ => unreachable!(),
1294 };
1295 put(sink, &enc_ri_a(opcode, rn, imm as u16));
1296 }
1297 &Inst::CmpRSImm32 { op, rn, imm } => {
1298 let opcode = match op {
1299 CmpOp::CmpS32 => 0xc2d, // CFI
1300 CmpOp::CmpS64 => 0xc2c, // CGFI
1301 _ => unreachable!(),
1302 };
1303 put(sink, &enc_ril_a(opcode, rn, imm as u32));
1304 }
1305 &Inst::CmpRUImm32 { op, rn, imm } => {
1306 let opcode = match op {
1307 CmpOp::CmpL32 => 0xc2f, // CLFI
1308 CmpOp::CmpL64 => 0xc2e, // CLGFI
1309 _ => unreachable!(),
1310 };
1311 put(sink, &enc_ril_a(opcode, rn, imm));
1312 }
1313 &Inst::CmpTrapRR {
1314 op,
1315 rn,
1316 rm,
1317 cond,
1318 trap_code,
1319 } => {
1320 let opcode = match op {
1321 CmpOp::CmpS32 => 0xb972, // CRT
1322 CmpOp::CmpS64 => 0xb960, // CGRT
1323 CmpOp::CmpL32 => 0xb973, // CLRT
1324 CmpOp::CmpL64 => 0xb961, // CLGRT
1325 _ => unreachable!(),
1326 };
1327 let srcloc = state.cur_srcloc();
1328 put_with_trap(
1329 sink,
1330 &enc_rrf_cde(opcode, rn, rm, cond.bits(), 0),
1331 srcloc,
1332 trap_code,
1333 );
1334 }
1335 &Inst::CmpTrapRSImm16 {
1336 op,
1337 rn,
1338 imm,
1339 cond,
1340 trap_code,
1341 } => {
1342 let opcode = match op {
1343 CmpOp::CmpS32 => 0xec72, // CIT
1344 CmpOp::CmpS64 => 0xec70, // CGIT
1345 _ => unreachable!(),
1346 };
1347 let srcloc = state.cur_srcloc();
1348 put_with_trap(
1349 sink,
1350 &enc_rie_a(opcode, rn, imm as u16, cond.bits()),
1351 srcloc,
1352 trap_code,
1353 );
1354 }
1355 &Inst::CmpTrapRUImm16 {
1356 op,
1357 rn,
1358 imm,
1359 cond,
1360 trap_code,
1361 } => {
1362 let opcode = match op {
1363 CmpOp::CmpL32 => 0xec73, // CLFIT
1364 CmpOp::CmpL64 => 0xec71, // CLGIT
1365 _ => unreachable!(),
1366 };
1367 let srcloc = state.cur_srcloc();
1368 put_with_trap(
1369 sink,
1370 &enc_rie_a(opcode, rn, imm, cond.bits()),
1371 srcloc,
1372 trap_code,
1373 );
1374 }
1375
1376 &Inst::AtomicRmw {
1377 alu_op,
1378 rd,
1379 rn,
1380 ref mem,
1381 } => {
1382 let opcode = match alu_op {
1383 ALUOp::Add32 => 0xebf8, // LAA
1384 ALUOp::Add64 => 0xebe8, // LAAG
1385 ALUOp::And32 => 0xebf4, // LAN
1386 ALUOp::And64 => 0xebe4, // LANG
1387 ALUOp::Orr32 => 0xebf6, // LAO
1388 ALUOp::Orr64 => 0xebe6, // LAOG
1389 ALUOp::Xor32 => 0xebf7, // LAX
1390 ALUOp::Xor64 => 0xebe7, // LAXG
1391 _ => unreachable!(),
1392 };
1393
1394 let rd = rd.to_reg();
1395 mem_rs_emit(
1396 rd,
1397 rn,
1398 mem,
1399 None,
1400 Some(opcode),
1401 true,
1402 sink,
1403 emit_info,
1404 state,
1405 );
1406 }
1407 &Inst::AtomicCas32 { rd, rn, ref mem } | &Inst::AtomicCas64 { rd, rn, ref mem } => {
1408 let (opcode_rs, opcode_rsy) = match self {
1409 &Inst::AtomicCas32 { .. } => (Some(0xba), Some(0xeb14)), // CS(Y)
1410 &Inst::AtomicCas64 { .. } => (None, Some(0xeb30)), // CSG
1411 _ => unreachable!(),
1412 };
1413
1414 let rd = rd.to_reg();
1415 mem_rs_emit(
1416 rd, rn, mem, opcode_rs, opcode_rsy, true, sink, emit_info, state,
1417 );
1418 }
1419 &Inst::Fence => {
1420 put(sink, &enc_e(0x07e0));
1421 }
1422
1423 &Inst::Load32 { rd, ref mem }
1424 | &Inst::Load32ZExt8 { rd, ref mem }
1425 | &Inst::Load32SExt8 { rd, ref mem }
1426 | &Inst::Load32ZExt16 { rd, ref mem }
1427 | &Inst::Load32SExt16 { rd, ref mem }
1428 | &Inst::Load64 { rd, ref mem }
1429 | &Inst::Load64ZExt8 { rd, ref mem }
1430 | &Inst::Load64SExt8 { rd, ref mem }
1431 | &Inst::Load64ZExt16 { rd, ref mem }
1432 | &Inst::Load64SExt16 { rd, ref mem }
1433 | &Inst::Load64ZExt32 { rd, ref mem }
1434 | &Inst::Load64SExt32 { rd, ref mem }
1435 | &Inst::LoadRev16 { rd, ref mem }
1436 | &Inst::LoadRev32 { rd, ref mem }
1437 | &Inst::LoadRev64 { rd, ref mem }
1438 | &Inst::FpuLoad32 { rd, ref mem }
1439 | &Inst::FpuLoad64 { rd, ref mem } => {
1440 let (opcode_rx, opcode_rxy, opcode_ril) = match self {
1441 &Inst::Load32 { .. } => (Some(0x58), Some(0xe358), Some(0xc4d)), // L(Y), LRL
1442 &Inst::Load32ZExt8 { .. } => (None, Some(0xe394), None), // LLC
1443 &Inst::Load32SExt8 { .. } => (None, Some(0xe376), None), // LB
1444 &Inst::Load32ZExt16 { .. } => (None, Some(0xe395), Some(0xc42)), // LLH, LLHRL
1445 &Inst::Load32SExt16 { .. } => (Some(0x48), Some(0xe378), Some(0xc45)), // LH(Y), LHRL
1446 &Inst::Load64 { .. } => (None, Some(0xe304), Some(0xc48)), // LG, LGRL
1447 &Inst::Load64ZExt8 { .. } => (None, Some(0xe390), None), // LLGC
1448 &Inst::Load64SExt8 { .. } => (None, Some(0xe377), None), // LGB
1449 &Inst::Load64ZExt16 { .. } => (None, Some(0xe391), Some(0xc46)), // LLGH, LLGHRL
1450 &Inst::Load64SExt16 { .. } => (None, Some(0xe315), Some(0xc44)), // LGH, LGHRL
1451 &Inst::Load64ZExt32 { .. } => (None, Some(0xe316), Some(0xc4e)), // LLGF, LLGFRL
1452 &Inst::Load64SExt32 { .. } => (None, Some(0xe314), Some(0xc4c)), // LGF, LGFRL
1453 &Inst::LoadRev16 { .. } => (None, Some(0xe31f), None), // LRVH
1454 &Inst::LoadRev32 { .. } => (None, Some(0xe31e), None), // LRV
1455 &Inst::LoadRev64 { .. } => (None, Some(0xe30f), None), // LRVG
1456 &Inst::FpuLoad32 { .. } => (Some(0x78), Some(0xed64), None), // LE(Y)
1457 &Inst::FpuLoad64 { .. } => (Some(0x68), Some(0xed65), None), // LD(Y)
1458 _ => unreachable!(),
1459 };
1460 let rd = rd.to_reg();
1461 mem_emit(
1462 rd, mem, opcode_rx, opcode_rxy, opcode_ril, true, sink, emit_info, state,
1463 );
1464 }
1465 &Inst::FpuLoadRev32 { rd, ref mem } | &Inst::FpuLoadRev64 { rd, ref mem } => {
1466 let opcode = match self {
1467 &Inst::FpuLoadRev32 { .. } => 0xe603, // VLEBRF
1468 &Inst::FpuLoadRev64 { .. } => 0xe602, // VLEBRG
1469 _ => unreachable!(),
1470 };
1471
1472 let (mem_insts, mem) = mem_finalize(mem, state, true, false, false, true);
1473 for inst in mem_insts.into_iter() {
1474 inst.emit(sink, emit_info, state);
1475 }
1476
1477 let srcloc = state.cur_srcloc();
1478 if srcloc != SourceLoc::default() && mem.can_trap() {
1479 sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
1480 }
1481
1482 match &mem {
1483 &MemArg::BXD12 {
1484 base, index, disp, ..
1485 } => {
1486 put(
1487 sink,
1488 &enc_vrx(opcode, rd.to_reg(), base, index, disp.bits(), 0),
1489 );
1490 }
1491 _ => unreachable!(),
1492 }
1493 }
1494
1495 &Inst::Store8 { rd, ref mem }
1496 | &Inst::Store16 { rd, ref mem }
1497 | &Inst::Store32 { rd, ref mem }
1498 | &Inst::Store64 { rd, ref mem }
1499 | &Inst::StoreRev16 { rd, ref mem }
1500 | &Inst::StoreRev32 { rd, ref mem }
1501 | &Inst::StoreRev64 { rd, ref mem }
1502 | &Inst::FpuStore32 { rd, ref mem }
1503 | &Inst::FpuStore64 { rd, ref mem } => {
1504 let (opcode_rx, opcode_rxy, opcode_ril) = match self {
1505 &Inst::Store8 { .. } => (Some(0x42), Some(0xe372), None), // STC(Y)
1506 &Inst::Store16 { .. } => (Some(0x40), Some(0xe370), Some(0xc47)), // STH(Y), STHRL
1507 &Inst::Store32 { .. } => (Some(0x50), Some(0xe350), Some(0xc4f)), // ST(Y), STRL
1508 &Inst::Store64 { .. } => (None, Some(0xe324), Some(0xc4b)), // STG, STGRL
1509 &Inst::StoreRev16 { .. } => (None, Some(0xe33f), None), // STRVH
1510 &Inst::StoreRev32 { .. } => (None, Some(0xe33e), None), // STRV
1511 &Inst::StoreRev64 { .. } => (None, Some(0xe32f), None), // STRVG
1512 &Inst::FpuStore32 { .. } => (Some(0x70), Some(0xed66), None), // STE(Y)
1513 &Inst::FpuStore64 { .. } => (Some(0x60), Some(0xed67), None), // STD(Y)
1514 _ => unreachable!(),
1515 };
1516 mem_emit(
1517 rd, mem, opcode_rx, opcode_rxy, opcode_ril, true, sink, emit_info, state,
1518 );
1519 }
1520 &Inst::StoreImm8 { imm, ref mem } => {
1521 let opcode_si = 0x92; // MVI
1522 let opcode_siy = 0xeb52; // MVIY
1523 mem_imm8_emit(
1524 imm, mem, opcode_si, opcode_siy, true, sink, emit_info, state,
1525 );
1526 }
1527 &Inst::StoreImm16 { imm, ref mem }
1528 | &Inst::StoreImm32SExt16 { imm, ref mem }
1529 | &Inst::StoreImm64SExt16 { imm, ref mem } => {
1530 let opcode = match self {
1531 &Inst::StoreImm16 { .. } => 0xe544, // MVHHI
1532 &Inst::StoreImm32SExt16 { .. } => 0xe54c, // MVHI
1533 &Inst::StoreImm64SExt16 { .. } => 0xe548, // MVGHI
1534 _ => unreachable!(),
1535 };
1536 mem_imm16_emit(imm, mem, opcode, true, sink, emit_info, state);
1537 }
1538 &Inst::FpuStoreRev32 { rd, ref mem } | &Inst::FpuStoreRev64 { rd, ref mem } => {
1539 let opcode = match self {
1540 &Inst::FpuStoreRev32 { .. } => 0xe60b, // VSTEBRF
1541 &Inst::FpuStoreRev64 { .. } => 0xe60a, // VSTEBRG
1542 _ => unreachable!(),
1543 };
1544
1545 let (mem_insts, mem) = mem_finalize(mem, state, true, false, false, true);
1546 for inst in mem_insts.into_iter() {
1547 inst.emit(sink, emit_info, state);
1548 }
1549
1550 let srcloc = state.cur_srcloc();
1551 if srcloc != SourceLoc::default() && mem.can_trap() {
1552 sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
1553 }
1554
1555 match &mem {
1556 &MemArg::BXD12 {
1557 base, index, disp, ..
1558 } => {
1559 put(sink, &enc_vrx(opcode, rd, base, index, disp.bits(), 0));
1560 }
1561 _ => unreachable!(),
1562 }
1563 }
1564
1565 &Inst::LoadMultiple64 {
1566 rt,
1567 rt2,
1568 addr_reg,
1569 addr_off,
1570 } => {
1571 let opcode = 0xeb04; // LMG
1572 let rt = rt.to_reg();
1573 let rt2 = rt2.to_reg();
1574 put(sink, &enc_rsy(opcode, rt, rt2, addr_reg, addr_off.bits()));
1575 }
1576 &Inst::StoreMultiple64 {
1577 rt,
1578 rt2,
1579 addr_reg,
1580 addr_off,
1581 } => {
1582 let opcode = 0xeb24; // STMG
1583 put(sink, &enc_rsy(opcode, rt, rt2, addr_reg, addr_off.bits()));
1584 }
1585
1586 &Inst::LoadAddr { rd, ref mem } => {
1587 let opcode_rx = Some(0x41); // LA
1588 let opcode_rxy = Some(0xe371); // LAY
1589 let opcode_ril = Some(0xc00); // LARL
1590 let rd = rd.to_reg();
1591 mem_emit(
1592 rd, mem, opcode_rx, opcode_rxy, opcode_ril, false, sink, emit_info, state,
1593 );
1594 }
1595
1596 &Inst::Mov64 { rd, rm } => {
1597 let opcode = 0xb904; // LGR
1598 put(sink, &enc_rre(opcode, rd.to_reg(), rm));
1599 }
1600 &Inst::Mov32 { rd, rm } => {
1601 let opcode = 0x18; // LR
1602 put(sink, &enc_rr(opcode, rd.to_reg(), rm));
1603 }
1604 &Inst::Mov32Imm { rd, imm } => {
1605 let opcode = 0xc09; // IILF
1606 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm));
1607 }
1608 &Inst::Mov32SImm16 { rd, imm } => {
1609 let opcode = 0xa78; // LHI
1610 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm as u16));
1611 }
1612 &Inst::Mov64SImm16 { rd, imm } => {
1613 let opcode = 0xa79; // LGHI
1614 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm as u16));
1615 }
1616 &Inst::Mov64SImm32 { rd, imm } => {
1617 let opcode = 0xc01; // LGFI
1618 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm as u32));
1619 }
1620 &Inst::CMov32 { rd, cond, rm } => {
1621 let opcode = 0xb9f2; // LOCR
1622 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rm, cond.bits(), 0));
1623 }
1624 &Inst::CMov64 { rd, cond, rm } => {
1625 let opcode = 0xb9e2; // LOCGR
1626 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rm, cond.bits(), 0));
1627 }
1628 &Inst::CMov32SImm16 { rd, cond, imm } => {
1629 let opcode = 0xec42; // LOCHI
1630 put(
1631 sink,
1632 &enc_rie_g(opcode, rd.to_reg(), imm as u16, cond.bits()),
1633 );
1634 }
1635 &Inst::CMov64SImm16 { rd, cond, imm } => {
1636 let opcode = 0xec46; // LOCGHI
1637 put(
1638 sink,
1639 &enc_rie_g(opcode, rd.to_reg(), imm as u16, cond.bits()),
1640 );
1641 }
1642 &Inst::Mov64UImm16Shifted { rd, imm } => {
1643 let opcode = match imm.shift {
1644 0 => 0xa5f, // LLILL
1645 1 => 0xa5e, // LLILH
1646 2 => 0xa5d, // LLIHL
1647 3 => 0xa5c, // LLIHH
1648 _ => unreachable!(),
1649 };
1650 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm.bits));
1651 }
1652 &Inst::Mov64UImm32Shifted { rd, imm } => {
1653 let opcode = match imm.shift {
1654 0 => 0xc0f, // LLILF
1655 1 => 0xc0e, // LLIHF
1656 _ => unreachable!(),
1657 };
1658 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits));
1659 }
1660 &Inst::Insert64UImm16Shifted { rd, imm } => {
1661 let opcode = match imm.shift {
1662 0 => 0xa53, // IILL
1663 1 => 0xa52, // IILH
1664 2 => 0xa51, // IIHL
1665 3 => 0xa50, // IIHH
1666 _ => unreachable!(),
1667 };
1668 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm.bits));
1669 }
1670 &Inst::Insert64UImm32Shifted { rd, imm } => {
1671 let opcode = match imm.shift {
1672 0 => 0xc09, // IILF
1673 1 => 0xc08, // IIHF
1674 _ => unreachable!(),
1675 };
1676 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits));
1677 }
1678 &Inst::LoadExtNameFar {
1679 rd,
1680 ref name,
1681 offset,
1682 } => {
1683 let opcode = 0xa75; // BRAS
1684 let srcloc = state.cur_srcloc();
1685 let reg = writable_spilltmp_reg().to_reg();
1686 put(sink, &enc_ri_b(opcode, reg, 12));
1687 sink.add_reloc(srcloc, Reloc::Abs8, name, offset);
1688 if emit_info.flags().emit_all_ones_funcaddrs() {
1689 sink.put8(u64::max_value());
1690 } else {
1691 sink.put8(0);
1692 }
1693 let inst = Inst::Load64 {
1694 rd,
1695 mem: MemArg::reg(reg, MemFlags::trusted()),
1696 };
1697 inst.emit(sink, emit_info, state);
1698 }
1699
1700 &Inst::FpuMove32 { rd, rn } => {
1701 let opcode = 0x38; // LER
1702 put(sink, &enc_rr(opcode, rd.to_reg(), rn));
1703 }
1704 &Inst::FpuMove64 { rd, rn } => {
1705 let opcode = 0x28; // LDR
1706 put(sink, &enc_rr(opcode, rd.to_reg(), rn));
1707 }
1708 &Inst::FpuCMov32 { rd, cond, rm } => {
1709 let opcode = 0xa74; // BCR
1710 put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 2));
1711 let opcode = 0x38; // LER
1712 put(sink, &enc_rr(opcode, rd.to_reg(), rm));
1713 }
1714 &Inst::FpuCMov64 { rd, cond, rm } => {
1715 let opcode = 0xa74; // BCR
1716 put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 2));
1717 let opcode = 0x28; // LDR
1718 put(sink, &enc_rr(opcode, rd.to_reg(), rm));
1719 }
1720 &Inst::MovToFpr { rd, rn } => {
1721 let opcode = 0xb3c1; // LDGR
1722 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1723 }
1724 &Inst::MovFromFpr { rd, rn } => {
1725 let opcode = 0xb3cd; // LGDR
1726 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1727 }
1728 &Inst::LoadFpuConst32 { rd, const_data } => {
1729 let opcode = 0xa75; // BRAS
1730 let reg = writable_spilltmp_reg().to_reg();
1731 put(sink, &enc_ri_b(opcode, reg, 8));
1732 sink.put4(const_data.to_bits().swap_bytes());
1733 let inst = Inst::FpuLoad32 {
1734 rd,
1735 mem: MemArg::reg(reg, MemFlags::trusted()),
1736 };
1737 inst.emit(sink, emit_info, state);
1738 }
1739 &Inst::LoadFpuConst64 { rd, const_data } => {
1740 let opcode = 0xa75; // BRAS
1741 let reg = writable_spilltmp_reg().to_reg();
1742 put(sink, &enc_ri_b(opcode, reg, 12));
1743 sink.put8(const_data.to_bits().swap_bytes());
1744 let inst = Inst::FpuLoad64 {
1745 rd,
1746 mem: MemArg::reg(reg, MemFlags::trusted()),
1747 };
1748 inst.emit(sink, emit_info, state);
1749 }
1750
1751 &Inst::FpuCopysign { rd, rn, rm } => {
1752 let opcode = 0xb372; // CPSDR
1753 put(sink, &enc_rrf_ab(opcode, rd.to_reg(), rn, rm, 0));
1754 }
1755 &Inst::FpuRR { fpu_op, rd, rn } => {
1756 let opcode = match fpu_op {
1757 FPUOp1::Abs32 => 0xb300, // LPEBR
1758 FPUOp1::Abs64 => 0xb310, // LPDBR
1759 FPUOp1::Neg32 => 0xb303, // LCEBR
1760 FPUOp1::Neg64 => 0xb313, // LCDBR
1761 FPUOp1::NegAbs32 => 0xb301, // LNEBR
1762 FPUOp1::NegAbs64 => 0xb311, // LNDBR
1763 FPUOp1::Sqrt32 => 0xb314, // SQEBR
1764 FPUOp1::Sqrt64 => 0xb315, // SQDBR
1765 FPUOp1::Cvt32To64 => 0xb304, // LDEBR
1766 FPUOp1::Cvt64To32 => 0xb344, // LEDBR
1767 };
1768 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1769 }
1770 &Inst::FpuRRR { fpu_op, rd, rm } => {
1771 let opcode = match fpu_op {
1772 FPUOp2::Add32 => 0xb30a, // AEBR
1773 FPUOp2::Add64 => 0xb31a, // ADBR
1774 FPUOp2::Sub32 => 0xb30b, // SEBR
1775 FPUOp2::Sub64 => 0xb31b, // SDBR
1776 FPUOp2::Mul32 => 0xb317, // MEEBR
1777 FPUOp2::Mul64 => 0xb31c, // MDBR
1778 FPUOp2::Div32 => 0xb30d, // DEBR
1779 FPUOp2::Div64 => 0xb31d, // DDBR
1780 _ => unimplemented!(),
1781 };
1782 put(sink, &enc_rre(opcode, rd.to_reg(), rm));
1783 }
1784 &Inst::FpuRRRR { fpu_op, rd, rn, rm } => {
1785 let opcode = match fpu_op {
1786 FPUOp3::MAdd32 => 0xb30e, // MAEBR
1787 FPUOp3::MAdd64 => 0xb31e, // MADBR
1788 FPUOp3::MSub32 => 0xb30f, // MSEBR
1789 FPUOp3::MSub64 => 0xb31f, // MSDBR
1790 };
1791 put(sink, &enc_rrd(opcode, rd.to_reg(), rm, rn));
1792 }
1793 &Inst::FpuToInt { op, rd, rn } => {
1794 let opcode = match op {
1795 FpuToIntOp::F32ToI32 => 0xb398, // CFEBRA
1796 FpuToIntOp::F32ToU32 => 0xb39c, // CLFEBR
1797 FpuToIntOp::F32ToI64 => 0xb3a8, // CGEBRA
1798 FpuToIntOp::F32ToU64 => 0xb3ac, // CLGEBR
1799 FpuToIntOp::F64ToI32 => 0xb399, // CFDBRA
1800 FpuToIntOp::F64ToU32 => 0xb39d, // CLFDBR
1801 FpuToIntOp::F64ToI64 => 0xb3a9, // CGDBRA
1802 FpuToIntOp::F64ToU64 => 0xb3ad, // CLGDBR
1803 };
1804 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, 5, 0));
1805 }
1806 &Inst::IntToFpu { op, rd, rn } => {
1807 let opcode = match op {
1808 IntToFpuOp::I32ToF32 => 0xb394, // CEFBRA
1809 IntToFpuOp::U32ToF32 => 0xb390, // CELFBR
1810 IntToFpuOp::I64ToF32 => 0xb3a4, // CEGBRA
1811 IntToFpuOp::U64ToF32 => 0xb3a0, // CELGBR
1812 IntToFpuOp::I32ToF64 => 0xb395, // CDFBRA
1813 IntToFpuOp::U32ToF64 => 0xb391, // CDLFBR
1814 IntToFpuOp::I64ToF64 => 0xb3a5, // CDGBRA
1815 IntToFpuOp::U64ToF64 => 0xb3a1, // CDLGBR
1816 };
1817 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, 0, 0));
1818 }
1819 &Inst::FpuRound { op, rd, rn } => {
1820 let (opcode, m3) = match op {
1821 FpuRoundMode::Minus32 => (0xb357, 7), // FIEBR
1822 FpuRoundMode::Minus64 => (0xb35f, 7), // FIDBR
1823 FpuRoundMode::Plus32 => (0xb357, 6), // FIEBR
1824 FpuRoundMode::Plus64 => (0xb35f, 6), // FIDBR
1825 FpuRoundMode::Zero32 => (0xb357, 5), // FIEBR
1826 FpuRoundMode::Zero64 => (0xb35f, 5), // FIDBR
1827 FpuRoundMode::Nearest32 => (0xb357, 4), // FIEBR
1828 FpuRoundMode::Nearest64 => (0xb35f, 4), // FIDBR
1829 };
1830 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, m3, 0));
1831 }
1832 &Inst::FpuVecRRR { fpu_op, rd, rn, rm } => {
1833 let (opcode, m4) = match fpu_op {
1834 FPUOp2::Max32 => (0xe7ef, 2), // VFMAX
1835 FPUOp2::Max64 => (0xe7ef, 3), // VFMAX
1836 FPUOp2::Min32 => (0xe7ee, 2), // VFMIN
1837 FPUOp2::Min64 => (0xe7ee, 3), // VFMIN
1838 _ => unimplemented!(),
1839 };
1840 put(sink, &enc_vrr(opcode, rd.to_reg(), rn, rm, m4, 8, 1));
1841 }
1842 &Inst::FpuCmp32 { rn, rm } => {
1843 let opcode = 0xb309; // CEBR
1844 put(sink, &enc_rre(opcode, rn, rm));
1845 }
1846 &Inst::FpuCmp64 { rn, rm } => {
1847 let opcode = 0xb319; // CDBR
1848 put(sink, &enc_rre(opcode, rn, rm));
1849 }
1850
1851 &Inst::Call { link, ref info } => {
1852 let opcode = 0xc05; // BRASL
1853 let reloc = Reloc::S390xPCRel32Dbl;
1854 let srcloc = state.cur_srcloc();
1855 if let Some(s) = state.take_stack_map() {
1856 sink.add_stack_map(StackMapExtent::UpcomingBytes(6), s);
1857 }
1858 put_with_reloc(
1859 sink,
1860 &enc_ril_b(opcode, link.to_reg(), 0),
1861 2,
1862 srcloc,
1863 reloc,
1864 &info.dest,
1865 0,
1866 );
1867 if info.opcode.is_call() {
1868 sink.add_call_site(srcloc, info.opcode);
1869 }
1870 }
1871 &Inst::CallInd { link, ref info } => {
1872 let opcode = 0x0d; // BASR
1873 let srcloc = state.cur_srcloc();
1874 if let Some(s) = state.take_stack_map() {
1875 sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s);
1876 }
1877 put(sink, &enc_rr(opcode, link.to_reg(), info.rn));
1878 if info.opcode.is_call() {
1879 sink.add_call_site(srcloc, info.opcode);
1880 }
1881 }
1882 &Inst::Ret { link } => {
1883 let opcode = 0x07; // BCR
1884 put(sink, &enc_rr(opcode, gpr(15), link));
1885 }
1886 &Inst::EpiloguePlaceholder => {
1887 // Noop; this is just a placeholder for epilogues.
1888 }
1889 &Inst::Jump { ref dest } => {
1890 let off = sink.cur_offset();
1891 // Indicate that the jump uses a label, if so, so that a fixup can occur later.
1892 if let Some(l) = dest.as_label() {
1893 sink.use_label_at_offset(off, l, LabelUse::BranchRIL);
1894 sink.add_uncond_branch(off, off + 6, l);
1895 }
1896 // Emit the jump itself.
1897 let opcode = 0xc04; // BCRL
1898 put(sink, &enc_ril_c(opcode, 15, dest.as_ril_offset_or_zero()));
1899 }
1900 &Inst::IndirectBr { rn, .. } => {
1901 let opcode = 0x07; // BCR
1902 put(sink, &enc_rr(opcode, gpr(15), rn));
1903 }
1904 &Inst::CondBr {
1905 ref taken,
1906 ref not_taken,
1907 cond,
1908 } => {
1909 let opcode = 0xc04; // BCRL
1910
1911 // Conditional part first.
1912 let cond_off = sink.cur_offset();
1913 if let Some(l) = taken.as_label() {
1914 sink.use_label_at_offset(cond_off, l, LabelUse::BranchRIL);
1915 let inverted = &enc_ril_c(opcode, cond.invert().bits(), 0);
1916 sink.add_cond_branch(cond_off, cond_off + 6, l, inverted);
1917 }
1918 put(
1919 sink,
1920 &enc_ril_c(opcode, cond.bits(), taken.as_ril_offset_or_zero()),
1921 );
1922
1923 // Unconditional part next.
1924 let uncond_off = sink.cur_offset();
1925 if let Some(l) = not_taken.as_label() {
1926 sink.use_label_at_offset(uncond_off, l, LabelUse::BranchRIL);
1927 sink.add_uncond_branch(uncond_off, uncond_off + 6, l);
1928 }
1929 put(
1930 sink,
1931 &enc_ril_c(opcode, 15, not_taken.as_ril_offset_or_zero()),
1932 );
1933 }
1934 &Inst::OneWayCondBr { ref target, cond } => {
1935 let opcode = 0xc04; // BCRL
1936 if let Some(l) = target.as_label() {
1937 sink.use_label_at_offset(sink.cur_offset(), l, LabelUse::BranchRIL);
1938 }
1939 put(
1940 sink,
1941 &enc_ril_c(opcode, cond.bits(), target.as_ril_offset_or_zero()),
1942 );
1943 }
1944 &Inst::Nop0 => {}
1945 &Inst::Nop2 => {
1946 put(sink, &enc_e(0x0707));
1947 }
1948 &Inst::Debugtrap => {
1949 put(sink, &enc_e(0x0001));
1950 }
1951 &Inst::Trap { trap_code } => {
1952 if let Some(s) = state.take_stack_map() {
1953 sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s);
1954 }
1955 let srcloc = state.cur_srcloc();
1956 put_with_trap(sink, &enc_e(0x0000), srcloc, trap_code);
1957 }
1958 &Inst::TrapIf { cond, trap_code } => {
1959 // Branch over trap if condition is false.
1960 let opcode = 0xa74; // BCR
1961 put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 2));
1962 // Now emit the actual trap.
1963 if let Some(s) = state.take_stack_map() {
1964 sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s);
1965 }
1966 let srcloc = state.cur_srcloc();
1967 put_with_trap(sink, &enc_e(0x0000), srcloc, trap_code);
1968 }
1969 &Inst::JTSequence {
1970 ridx,
1971 rtmp1,
1972 rtmp2,
1973 ref info,
1974 ..
1975 } => {
1976 let table_label = sink.get_label();
1977
1978 // This sequence is *one* instruction in the vcode, and is expanded only here at
1979 // emission time, because we cannot allow the regalloc to insert spills/reloads in
1980 // the middle; we depend on hardcoded PC-rel addressing below.
1981
1982 // Bounds-check index and branch to default.
1983 let inst = Inst::CmpRUImm32 {
1984 op: CmpOp::CmpL64,
1985 rn: ridx,
1986 imm: info.targets.len() as u32,
1987 };
1988 inst.emit(sink, emit_info, state);
1989 let inst = Inst::OneWayCondBr {
1990 target: info.default_target,
1991 cond: Cond::from_intcc(IntCC::UnsignedGreaterThanOrEqual),
1992 };
1993 inst.emit(sink, emit_info, state);
1994
1995 // Set rtmp2 to index scaled by entry size.
1996 let inst = Inst::ShiftRR {
1997 shift_op: ShiftOp::LShL64,
1998 rd: rtmp2,
1999 rn: ridx,
2000 shift_imm: SImm20::maybe_from_i64(2).unwrap(),
2001 shift_reg: None,
2002 };
2003 inst.emit(sink, emit_info, state);
2004
2005 // Set rtmp1 to address of jump table.
2006 let inst = Inst::LoadAddr {
2007 rd: rtmp1,
2008 mem: MemArg::Label {
2009 target: BranchTarget::Label(table_label),
2010 },
2011 };
2012 inst.emit(sink, emit_info, state);
2013
2014 // Set rtmp2 to value loaded out of jump table.
2015 let inst = Inst::Load64SExt32 {
2016 rd: rtmp2,
2017 mem: MemArg::reg_plus_reg(rtmp1.to_reg(), rtmp2.to_reg(), MemFlags::trusted()),
2018 };
2019 inst.emit(sink, emit_info, state);
2020
2021 // Set rtmp1 to target address (rtmp1 + rtmp2).
2022 let inst = Inst::AluRRR {
2023 alu_op: ALUOp::Add64,
2024 rd: rtmp1,
2025 rn: rtmp1.to_reg(),
2026 rm: rtmp2.to_reg(),
2027 };
2028 inst.emit(sink, emit_info, state);
2029
2030 // Branch to computed address. (`targets` here is only used for successor queries
2031 // and is not needed for emission.)
2032 let inst = Inst::IndirectBr {
2033 rn: rtmp1.to_reg(),
2034 targets: vec![],
2035 };
2036 inst.emit(sink, emit_info, state);
2037
2038 // Emit jump table (table of 32-bit offsets).
2039 sink.bind_label(table_label);
2040 let jt_off = sink.cur_offset();
2041 for &target in info.targets.iter() {
2042 let word_off = sink.cur_offset();
2043 let off_into_table = word_off - jt_off;
2044 sink.use_label_at_offset(
2045 word_off,
2046 target.as_label().unwrap(),
2047 LabelUse::PCRel32,
2048 );
2049 sink.put4(off_into_table.swap_bytes());
2050 }
2051
2052 // Lowering produces an EmitIsland before using a JTSequence, so we can safely
2053 // disable the worst-case-size check in this case.
2054 start_off = sink.cur_offset();
2055 }
2056
2057 &Inst::VirtualSPOffsetAdj { offset } => {
2058 log::trace!(
2059 "virtual sp offset adjusted by {} -> {}",
2060 offset,
2061 state.virtual_sp_offset + offset
2062 );
2063 state.virtual_sp_offset += offset;
2064 }
2065
2066 &Inst::ValueLabelMarker { .. } => {
2067 // Nothing; this is only used to compute debug info.
2068 }
2069
2070 &Inst::Unwind { ref inst } => {
2071 sink.add_unwind(inst.clone());
2072 }
2073 }
2074
2075 let end_off = sink.cur_offset();
2076 debug_assert!((end_off - start_off) <= Inst::worst_case_size());
2077
2078 state.clear_post_insn();
2079 }
2080
pretty_print(&self, mb_rru: Option<&RealRegUniverse>, state: &mut EmitState) -> String2081 fn pretty_print(&self, mb_rru: Option<&RealRegUniverse>, state: &mut EmitState) -> String {
2082 self.print_with_state(mb_rru, state)
2083 }
2084 }
2085