1 //! S390x ISA: binary code emission.
2 
3 use crate::binemit::{Reloc, StackMap};
4 use crate::ir::condcodes::IntCC;
5 use crate::ir::MemFlags;
6 use crate::ir::{SourceLoc, TrapCode};
7 use crate::isa::s390x::inst::*;
8 use crate::isa::s390x::settings as s390x_settings;
9 use core::convert::TryFrom;
10 use regalloc::{Reg, RegClass};
11 
12 /// Memory addressing mode finalization: convert "special" modes (e.g.,
13 /// generic arbitrary stack offset) into real addressing modes, possibly by
14 /// emitting some helper instructions that come immediately before the use
15 /// of this amode.
mem_finalize( mem: &MemArg, state: &EmitState, have_d12: bool, have_d20: bool, have_pcrel: bool, have_index: bool, ) -> (SmallVec<[Inst; 4]>, MemArg)16 pub fn mem_finalize(
17     mem: &MemArg,
18     state: &EmitState,
19     have_d12: bool,
20     have_d20: bool,
21     have_pcrel: bool,
22     have_index: bool,
23 ) -> (SmallVec<[Inst; 4]>, MemArg) {
24     let mut insts = SmallVec::new();
25 
26     // Resolve virtual addressing modes.
27     let mem = match mem {
28         &MemArg::RegOffset { off, .. }
29         | &MemArg::InitialSPOffset { off }
30         | &MemArg::NominalSPOffset { off } => {
31             let base = match mem {
32                 &MemArg::RegOffset { reg, .. } => reg,
33                 &MemArg::InitialSPOffset { .. } | &MemArg::NominalSPOffset { .. } => stack_reg(),
34                 _ => unreachable!(),
35             };
36             let adj = match mem {
37                 &MemArg::InitialSPOffset { .. } => {
38                     state.initial_sp_offset + state.virtual_sp_offset
39                 }
40                 &MemArg::NominalSPOffset { .. } => state.virtual_sp_offset,
41                 _ => 0,
42             };
43             let off = off + adj;
44 
45             if let Some(disp) = UImm12::maybe_from_u64(off as u64) {
46                 MemArg::BXD12 {
47                     base,
48                     index: zero_reg(),
49                     disp,
50                     flags: mem.get_flags(),
51                 }
52             } else if let Some(disp) = SImm20::maybe_from_i64(off) {
53                 MemArg::BXD20 {
54                     base,
55                     index: zero_reg(),
56                     disp,
57                     flags: mem.get_flags(),
58                 }
59             } else {
60                 let tmp = writable_spilltmp_reg();
61                 assert!(base != tmp.to_reg());
62                 insts.extend(Inst::load_constant64(tmp, off as u64));
63                 MemArg::reg_plus_reg(base, tmp.to_reg(), mem.get_flags())
64             }
65         }
66         _ => mem.clone(),
67     };
68 
69     // If this addressing mode cannot be handled by the instruction, use load-address.
70     let need_load_address = match &mem {
71         &MemArg::Label { .. } | &MemArg::Symbol { .. } if !have_pcrel => true,
72         &MemArg::BXD20 { .. } if !have_d20 => true,
73         &MemArg::BXD12 { index, .. } | &MemArg::BXD20 { index, .. } if !have_index => {
74             index != zero_reg()
75         }
76         _ => false,
77     };
78     let mem = if need_load_address {
79         let flags = mem.get_flags();
80         let tmp = writable_spilltmp_reg();
81         insts.push(Inst::LoadAddr { rd: tmp, mem });
82         MemArg::reg(tmp.to_reg(), flags)
83     } else {
84         mem
85     };
86 
87     // Convert 12-bit displacement to 20-bit if required.
88     let mem = match &mem {
89         &MemArg::BXD12 {
90             base,
91             index,
92             disp,
93             flags,
94         } if !have_d12 => {
95             assert!(have_d20);
96             MemArg::BXD20 {
97                 base,
98                 index,
99                 disp: SImm20::from_uimm12(disp),
100                 flags,
101             }
102         }
103         _ => mem,
104     };
105 
106     (insts, mem)
107 }
108 
mem_emit( rd: Reg, mem: &MemArg, opcode_rx: Option<u16>, opcode_rxy: Option<u16>, opcode_ril: Option<u16>, add_trap: bool, sink: &mut MachBuffer<Inst>, emit_info: &EmitInfo, state: &mut EmitState, )109 pub fn mem_emit(
110     rd: Reg,
111     mem: &MemArg,
112     opcode_rx: Option<u16>,
113     opcode_rxy: Option<u16>,
114     opcode_ril: Option<u16>,
115     add_trap: bool,
116     sink: &mut MachBuffer<Inst>,
117     emit_info: &EmitInfo,
118     state: &mut EmitState,
119 ) {
120     let (mem_insts, mem) = mem_finalize(
121         mem,
122         state,
123         opcode_rx.is_some(),
124         opcode_rxy.is_some(),
125         opcode_ril.is_some(),
126         true,
127     );
128     for inst in mem_insts.into_iter() {
129         inst.emit(sink, emit_info, state);
130     }
131 
132     if add_trap && mem.can_trap() {
133         let srcloc = state.cur_srcloc();
134         if srcloc != SourceLoc::default() {
135             sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
136         }
137     }
138 
139     match &mem {
140         &MemArg::BXD12 {
141             base, index, disp, ..
142         } => {
143             put(
144                 sink,
145                 &enc_rx(opcode_rx.unwrap(), rd, base, index, disp.bits()),
146             );
147         }
148         &MemArg::BXD20 {
149             base, index, disp, ..
150         } => {
151             put(
152                 sink,
153                 &enc_rxy(opcode_rxy.unwrap(), rd, base, index, disp.bits()),
154             );
155         }
156         &MemArg::Label { ref target } => {
157             if let Some(l) = target.as_label() {
158                 sink.use_label_at_offset(sink.cur_offset(), l, LabelUse::BranchRIL);
159             }
160             put(
161                 sink,
162                 &enc_ril_b(opcode_ril.unwrap(), rd, target.as_ril_offset_or_zero()),
163             );
164         }
165         &MemArg::Symbol {
166             ref name, offset, ..
167         } => {
168             let reloc = Reloc::S390xPCRel32Dbl;
169             let srcloc = state.cur_srcloc();
170             put_with_reloc(
171                 sink,
172                 &enc_ril_b(opcode_ril.unwrap(), rd, 0),
173                 2,
174                 srcloc,
175                 reloc,
176                 name,
177                 offset.into(),
178             );
179         }
180         _ => unreachable!(),
181     }
182 }
183 
mem_rs_emit( rd: Reg, rn: Reg, mem: &MemArg, opcode_rs: Option<u16>, opcode_rsy: Option<u16>, add_trap: bool, sink: &mut MachBuffer<Inst>, emit_info: &EmitInfo, state: &mut EmitState, )184 pub fn mem_rs_emit(
185     rd: Reg,
186     rn: Reg,
187     mem: &MemArg,
188     opcode_rs: Option<u16>,
189     opcode_rsy: Option<u16>,
190     add_trap: bool,
191     sink: &mut MachBuffer<Inst>,
192     emit_info: &EmitInfo,
193     state: &mut EmitState,
194 ) {
195     let (mem_insts, mem) = mem_finalize(
196         mem,
197         state,
198         opcode_rs.is_some(),
199         opcode_rsy.is_some(),
200         false,
201         false,
202     );
203     for inst in mem_insts.into_iter() {
204         inst.emit(sink, emit_info, state);
205     }
206 
207     if add_trap && mem.can_trap() {
208         let srcloc = state.cur_srcloc();
209         if srcloc != SourceLoc::default() {
210             sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
211         }
212     }
213 
214     match &mem {
215         &MemArg::BXD12 {
216             base, index, disp, ..
217         } => {
218             assert!(index == zero_reg());
219             put(sink, &enc_rs(opcode_rs.unwrap(), rd, rn, base, disp.bits()));
220         }
221         &MemArg::BXD20 {
222             base, index, disp, ..
223         } => {
224             assert!(index == zero_reg());
225             put(
226                 sink,
227                 &enc_rsy(opcode_rsy.unwrap(), rd, rn, base, disp.bits()),
228             );
229         }
230         _ => unreachable!(),
231     }
232 }
233 
mem_imm8_emit( imm: u8, mem: &MemArg, opcode_si: u16, opcode_siy: u16, add_trap: bool, sink: &mut MachBuffer<Inst>, emit_info: &EmitInfo, state: &mut EmitState, )234 pub fn mem_imm8_emit(
235     imm: u8,
236     mem: &MemArg,
237     opcode_si: u16,
238     opcode_siy: u16,
239     add_trap: bool,
240     sink: &mut MachBuffer<Inst>,
241     emit_info: &EmitInfo,
242     state: &mut EmitState,
243 ) {
244     let (mem_insts, mem) = mem_finalize(mem, state, true, true, false, false);
245     for inst in mem_insts.into_iter() {
246         inst.emit(sink, emit_info, state);
247     }
248 
249     if add_trap && mem.can_trap() {
250         let srcloc = state.cur_srcloc();
251         if srcloc != SourceLoc::default() {
252             sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
253         }
254     }
255 
256     match &mem {
257         &MemArg::BXD12 {
258             base, index, disp, ..
259         } => {
260             assert!(index == zero_reg());
261             put(sink, &enc_si(opcode_si, base, disp.bits(), imm));
262         }
263         &MemArg::BXD20 {
264             base, index, disp, ..
265         } => {
266             assert!(index == zero_reg());
267             put(sink, &enc_siy(opcode_siy, base, disp.bits(), imm));
268         }
269         _ => unreachable!(),
270     }
271 }
272 
mem_imm16_emit( imm: i16, mem: &MemArg, opcode_sil: u16, add_trap: bool, sink: &mut MachBuffer<Inst>, emit_info: &EmitInfo, state: &mut EmitState, )273 pub fn mem_imm16_emit(
274     imm: i16,
275     mem: &MemArg,
276     opcode_sil: u16,
277     add_trap: bool,
278     sink: &mut MachBuffer<Inst>,
279     emit_info: &EmitInfo,
280     state: &mut EmitState,
281 ) {
282     let (mem_insts, mem) = mem_finalize(mem, state, true, false, false, false);
283     for inst in mem_insts.into_iter() {
284         inst.emit(sink, emit_info, state);
285     }
286 
287     if add_trap && mem.can_trap() {
288         let srcloc = state.cur_srcloc();
289         if srcloc != SourceLoc::default() {
290             sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
291         }
292     }
293 
294     match &mem {
295         &MemArg::BXD12 {
296             base, index, disp, ..
297         } => {
298             assert!(index == zero_reg());
299             put(sink, &enc_sil(opcode_sil, base, disp.bits(), imm));
300         }
301         _ => unreachable!(),
302     }
303 }
304 
305 //=============================================================================
306 // Instructions and subcomponents: emission
307 
machreg_to_gpr(m: Reg) -> u8308 fn machreg_to_gpr(m: Reg) -> u8 {
309     assert_eq!(m.get_class(), RegClass::I64);
310     u8::try_from(m.to_real_reg().get_hw_encoding()).unwrap()
311 }
312 
machreg_to_fpr(m: Reg) -> u8313 fn machreg_to_fpr(m: Reg) -> u8 {
314     assert_eq!(m.get_class(), RegClass::F64);
315     u8::try_from(m.to_real_reg().get_hw_encoding()).unwrap()
316 }
317 
machreg_to_gpr_or_fpr(m: Reg) -> u8318 fn machreg_to_gpr_or_fpr(m: Reg) -> u8 {
319     u8::try_from(m.to_real_reg().get_hw_encoding()).unwrap()
320 }
321 
322 /// E-type instructions.
323 ///
324 ///   15
325 ///   opcode
326 ///        0
327 ///
enc_e(opcode: u16) -> [u8; 2]328 fn enc_e(opcode: u16) -> [u8; 2] {
329     let mut enc: [u8; 2] = [0; 2];
330     let opcode1 = ((opcode >> 8) & 0xff) as u8;
331     let opcode2 = (opcode & 0xff) as u8;
332 
333     enc[0] = opcode1;
334     enc[1] = opcode2;
335     enc
336 }
337 
338 /// RIa-type instructions.
339 ///
340 ///   31      23 19      15
341 ///   opcode1 r1 opcode2 i2
342 ///        24 20      16  0
343 ///
enc_ri_a(opcode: u16, r1: Reg, i2: u16) -> [u8; 4]344 fn enc_ri_a(opcode: u16, r1: Reg, i2: u16) -> [u8; 4] {
345     let mut enc: [u8; 4] = [0; 4];
346     let opcode1 = ((opcode >> 4) & 0xff) as u8;
347     let opcode2 = (opcode & 0xf) as u8;
348     let r1 = machreg_to_gpr(r1) & 0x0f;
349 
350     enc[0] = opcode1;
351     enc[1] = r1 << 4 | opcode2;
352     enc[2..].copy_from_slice(&i2.to_be_bytes());
353     enc
354 }
355 
356 /// RIb-type instructions.
357 ///
358 ///   31      23 19      15
359 ///   opcode1 r1 opcode2 ri2
360 ///        24 20      16   0
361 ///
enc_ri_b(opcode: u16, r1: Reg, ri2: i32) -> [u8; 4]362 fn enc_ri_b(opcode: u16, r1: Reg, ri2: i32) -> [u8; 4] {
363     let mut enc: [u8; 4] = [0; 4];
364     let opcode1 = ((opcode >> 4) & 0xff) as u8;
365     let opcode2 = (opcode & 0xf) as u8;
366     let r1 = machreg_to_gpr(r1) & 0x0f;
367     let ri2 = ((ri2 >> 1) & 0xffff) as u16;
368 
369     enc[0] = opcode1;
370     enc[1] = r1 << 4 | opcode2;
371     enc[2..].copy_from_slice(&ri2.to_be_bytes());
372     enc
373 }
374 
375 /// RIc-type instructions.
376 ///
377 ///   31      23 19      15
378 ///   opcode1 m1 opcode2 ri2
379 ///        24 20      16   0
380 ///
enc_ri_c(opcode: u16, m1: u8, ri2: i32) -> [u8; 4]381 fn enc_ri_c(opcode: u16, m1: u8, ri2: i32) -> [u8; 4] {
382     let mut enc: [u8; 4] = [0; 4];
383     let opcode1 = ((opcode >> 4) & 0xff) as u8;
384     let opcode2 = (opcode & 0xf) as u8;
385     let m1 = m1 & 0x0f;
386     let ri2 = ((ri2 >> 1) & 0xffff) as u16;
387 
388     enc[0] = opcode1;
389     enc[1] = m1 << 4 | opcode2;
390     enc[2..].copy_from_slice(&ri2.to_be_bytes());
391     enc
392 }
393 
394 /// RIEa-type instructions.
395 ///
396 ///   47      39 35 31 15 11 7
397 ///   opcode1 r1 -- i2 m3 -- opcode2
398 ///        40 36 32 16 12 8       0
399 ///
enc_rie_a(opcode: u16, r1: Reg, i2: u16, m3: u8) -> [u8; 6]400 fn enc_rie_a(opcode: u16, r1: Reg, i2: u16, m3: u8) -> [u8; 6] {
401     let mut enc: [u8; 6] = [0; 6];
402     let opcode1 = ((opcode >> 8) & 0xff) as u8;
403     let opcode2 = (opcode & 0xff) as u8;
404     let r1 = machreg_to_gpr(r1) & 0x0f;
405     let m3 = m3 & 0x0f;
406 
407     enc[0] = opcode1;
408     enc[1] = r1 << 4;
409     enc[2..4].copy_from_slice(&i2.to_be_bytes());
410     enc[4] = m3 << 4;
411     enc[5] = opcode2;
412     enc
413 }
414 
415 /// RIEd-type instructions.
416 ///
417 ///   47      39 35 31 15 7
418 ///   opcode1 r1 r3 i2 -- opcode2
419 ///        40 36 32 16  8       0
420 ///
enc_rie_d(opcode: u16, r1: Reg, r3: Reg, i2: u16) -> [u8; 6]421 fn enc_rie_d(opcode: u16, r1: Reg, r3: Reg, i2: u16) -> [u8; 6] {
422     let mut enc: [u8; 6] = [0; 6];
423     let opcode1 = ((opcode >> 8) & 0xff) as u8;
424     let opcode2 = (opcode & 0xff) as u8;
425     let r1 = machreg_to_gpr(r1) & 0x0f;
426     let r3 = machreg_to_gpr(r3) & 0x0f;
427 
428     enc[0] = opcode1;
429     enc[1] = r1 << 4 | r3;
430     enc[2..4].copy_from_slice(&i2.to_be_bytes());
431     enc[5] = opcode2;
432     enc
433 }
434 
435 /// RIEg-type instructions.
436 ///
437 ///   47      39 35 31 15 7
438 ///   opcode1 r1 m3 i2 -- opcode2
439 ///        40 36 32 16  8       0
440 ///
enc_rie_g(opcode: u16, r1: Reg, i2: u16, m3: u8) -> [u8; 6]441 fn enc_rie_g(opcode: u16, r1: Reg, i2: u16, m3: u8) -> [u8; 6] {
442     let mut enc: [u8; 6] = [0; 6];
443     let opcode1 = ((opcode >> 8) & 0xff) as u8;
444     let opcode2 = (opcode & 0xff) as u8;
445     let r1 = machreg_to_gpr(r1) & 0x0f;
446     let m3 = m3 & 0x0f;
447 
448     enc[0] = opcode1;
449     enc[1] = r1 << 4 | m3;
450     enc[2..4].copy_from_slice(&i2.to_be_bytes());
451     enc[5] = opcode2;
452     enc
453 }
454 
455 /// RILa-type instructions.
456 ///
457 ///   47      39 35      31
458 ///   opcode1 r1 opcode2 i2
459 ///        40 36      32  0
460 ///
enc_ril_a(opcode: u16, r1: Reg, i2: u32) -> [u8; 6]461 fn enc_ril_a(opcode: u16, r1: Reg, i2: u32) -> [u8; 6] {
462     let mut enc: [u8; 6] = [0; 6];
463     let opcode1 = ((opcode >> 4) & 0xff) as u8;
464     let opcode2 = (opcode & 0xf) as u8;
465     let r1 = machreg_to_gpr(r1) & 0x0f;
466 
467     enc[0] = opcode1;
468     enc[1] = r1 << 4 | opcode2;
469     enc[2..].copy_from_slice(&i2.to_be_bytes());
470     enc
471 }
472 
473 /// RILb-type instructions.
474 ///
475 ///   47      39 35      31
476 ///   opcode1 r1 opcode2 ri2
477 ///        40 36      32   0
478 ///
enc_ril_b(opcode: u16, r1: Reg, ri2: u32) -> [u8; 6]479 fn enc_ril_b(opcode: u16, r1: Reg, ri2: u32) -> [u8; 6] {
480     let mut enc: [u8; 6] = [0; 6];
481     let opcode1 = ((opcode >> 4) & 0xff) as u8;
482     let opcode2 = (opcode & 0xf) as u8;
483     let r1 = machreg_to_gpr(r1) & 0x0f;
484 
485     enc[0] = opcode1;
486     enc[1] = r1 << 4 | opcode2;
487     enc[2..].copy_from_slice(&ri2.to_be_bytes());
488     enc
489 }
490 
491 /// RILc-type instructions.
492 ///
493 ///   47      39 35      31
494 ///   opcode1 m1 opcode2 i2
495 ///        40 36      32  0
496 ///
enc_ril_c(opcode: u16, m1: u8, ri2: u32) -> [u8; 6]497 fn enc_ril_c(opcode: u16, m1: u8, ri2: u32) -> [u8; 6] {
498     let mut enc: [u8; 6] = [0; 6];
499     let opcode1 = ((opcode >> 4) & 0xff) as u8;
500     let opcode2 = (opcode & 0xf) as u8;
501     let m1 = m1 & 0x0f;
502 
503     enc[0] = opcode1;
504     enc[1] = m1 << 4 | opcode2;
505     enc[2..].copy_from_slice(&ri2.to_be_bytes());
506     enc
507 }
508 
509 /// RR-type instructions.
510 ///
511 ///   15     7  3
512 ///   opcode r1 r2
513 ///        8  4  0
514 ///
enc_rr(opcode: u16, r1: Reg, r2: Reg) -> [u8; 2]515 fn enc_rr(opcode: u16, r1: Reg, r2: Reg) -> [u8; 2] {
516     let mut enc: [u8; 2] = [0; 2];
517     let opcode = (opcode & 0xff) as u8;
518     let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
519     let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
520 
521     enc[0] = opcode;
522     enc[1] = r1 << 4 | r2;
523     enc
524 }
525 
526 /// RRD-type instructions.
527 ///
528 ///   31     15 11 7  3
529 ///   opcode r1 -- r3 r2
530 ///       16 12  8 4  0
531 ///
enc_rrd(opcode: u16, r1: Reg, r2: Reg, r3: Reg) -> [u8; 4]532 fn enc_rrd(opcode: u16, r1: Reg, r2: Reg, r3: Reg) -> [u8; 4] {
533     let mut enc: [u8; 4] = [0; 4];
534     let opcode1 = ((opcode >> 8) & 0xff) as u8;
535     let opcode2 = (opcode & 0xff) as u8;
536     let r1 = machreg_to_fpr(r1) & 0x0f;
537     let r2 = machreg_to_fpr(r2) & 0x0f;
538     let r3 = machreg_to_fpr(r3) & 0x0f;
539 
540     enc[0] = opcode1;
541     enc[1] = opcode2;
542     enc[2] = r1 << 4;
543     enc[3] = r3 << 4 | r2;
544     enc
545 }
546 
547 /// RRE-type instructions.
548 ///
549 ///   31     15 7  3
550 ///   opcode -- r1 r2
551 ///       16  8  4  0
552 ///
enc_rre(opcode: u16, r1: Reg, r2: Reg) -> [u8; 4]553 fn enc_rre(opcode: u16, r1: Reg, r2: Reg) -> [u8; 4] {
554     let mut enc: [u8; 4] = [0; 4];
555     let opcode1 = ((opcode >> 8) & 0xff) as u8;
556     let opcode2 = (opcode & 0xff) as u8;
557     let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
558     let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
559 
560     enc[0] = opcode1;
561     enc[1] = opcode2;
562     enc[3] = r1 << 4 | r2;
563     enc
564 }
565 
566 /// RRFa/b-type instructions.
567 ///
568 ///   31     15 11 7  3
569 ///   opcode r3 m4 r1 r2
570 ///       16 12  8  4  0
571 ///
enc_rrf_ab(opcode: u16, r1: Reg, r2: Reg, r3: Reg, m4: u8) -> [u8; 4]572 fn enc_rrf_ab(opcode: u16, r1: Reg, r2: Reg, r3: Reg, m4: u8) -> [u8; 4] {
573     let mut enc: [u8; 4] = [0; 4];
574     let opcode1 = ((opcode >> 8) & 0xff) as u8;
575     let opcode2 = (opcode & 0xff) as u8;
576     let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
577     let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
578     let r3 = machreg_to_gpr_or_fpr(r3) & 0x0f;
579     let m4 = m4 & 0x0f;
580 
581     enc[0] = opcode1;
582     enc[1] = opcode2;
583     enc[2] = r3 << 4 | m4;
584     enc[3] = r1 << 4 | r2;
585     enc
586 }
587 
588 /// RRFc/d/e-type instructions.
589 ///
590 ///   31     15 11 7  3
591 ///   opcode m3 m4 r1 r2
592 ///       16 12  8  4  0
593 ///
enc_rrf_cde(opcode: u16, r1: Reg, r2: Reg, m3: u8, m4: u8) -> [u8; 4]594 fn enc_rrf_cde(opcode: u16, r1: Reg, r2: Reg, m3: u8, m4: u8) -> [u8; 4] {
595     let mut enc: [u8; 4] = [0; 4];
596     let opcode1 = ((opcode >> 8) & 0xff) as u8;
597     let opcode2 = (opcode & 0xff) as u8;
598     let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
599     let r2 = machreg_to_gpr_or_fpr(r2) & 0x0f;
600     let m3 = m3 & 0x0f;
601     let m4 = m4 & 0x0f;
602 
603     enc[0] = opcode1;
604     enc[1] = opcode2;
605     enc[2] = m3 << 4 | m4;
606     enc[3] = r1 << 4 | r2;
607     enc
608 }
609 
610 /// RS-type instructions.
611 ///
612 ///   31     23 19 15 11
613 ///   opcode r1 r3 b2 d2
614 ///       24 20 16 12  0
615 ///
enc_rs(opcode: u16, r1: Reg, r3: Reg, b2: Reg, d2: u32) -> [u8; 4]616 fn enc_rs(opcode: u16, r1: Reg, r3: Reg, b2: Reg, d2: u32) -> [u8; 4] {
617     let opcode = (opcode & 0xff) as u8;
618     let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
619     let r3 = machreg_to_gpr_or_fpr(r3) & 0x0f;
620     let b2 = machreg_to_gpr(b2) & 0x0f;
621     let d2_lo = (d2 & 0xff) as u8;
622     let d2_hi = ((d2 >> 8) & 0x0f) as u8;
623 
624     let mut enc: [u8; 4] = [0; 4];
625     enc[0] = opcode;
626     enc[1] = r1 << 4 | r3;
627     enc[2] = b2 << 4 | d2_hi;
628     enc[3] = d2_lo;
629     enc
630 }
631 
632 /// RSY-type instructions.
633 ///
634 ///   47      39 35 31 27  15  7
635 ///   opcode1 r1 r3 b2 dl2 dh2 opcode2
636 ///        40 36 32 28  16   8       0
637 ///
enc_rsy(opcode: u16, r1: Reg, r3: Reg, b2: Reg, d2: u32) -> [u8; 6]638 fn enc_rsy(opcode: u16, r1: Reg, r3: Reg, b2: Reg, d2: u32) -> [u8; 6] {
639     let opcode1 = ((opcode >> 8) & 0xff) as u8;
640     let opcode2 = (opcode & 0xff) as u8;
641     let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
642     let r3 = machreg_to_gpr_or_fpr(r3) & 0x0f;
643     let b2 = machreg_to_gpr(b2) & 0x0f;
644     let dl2_lo = (d2 & 0xff) as u8;
645     let dl2_hi = ((d2 >> 8) & 0x0f) as u8;
646     let dh2 = ((d2 >> 12) & 0xff) as u8;
647 
648     let mut enc: [u8; 6] = [0; 6];
649     enc[0] = opcode1;
650     enc[1] = r1 << 4 | r3;
651     enc[2] = b2 << 4 | dl2_hi;
652     enc[3] = dl2_lo;
653     enc[4] = dh2;
654     enc[5] = opcode2;
655     enc
656 }
657 
658 /// RX-type instructions.
659 ///
660 ///   31     23 19 15 11
661 ///   opcode r1 x2 b2 d2
662 ///       24 20 16 12  0
663 ///
enc_rx(opcode: u16, r1: Reg, b2: Reg, x2: Reg, d2: u32) -> [u8; 4]664 fn enc_rx(opcode: u16, r1: Reg, b2: Reg, x2: Reg, d2: u32) -> [u8; 4] {
665     let opcode = (opcode & 0xff) as u8;
666     let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
667     let b2 = machreg_to_gpr(b2) & 0x0f;
668     let x2 = machreg_to_gpr(x2) & 0x0f;
669     let d2_lo = (d2 & 0xff) as u8;
670     let d2_hi = ((d2 >> 8) & 0x0f) as u8;
671 
672     let mut enc: [u8; 4] = [0; 4];
673     enc[0] = opcode;
674     enc[1] = r1 << 4 | x2;
675     enc[2] = b2 << 4 | d2_hi;
676     enc[3] = d2_lo;
677     enc
678 }
679 
680 /// RXY-type instructions.
681 ///
682 ///   47      39 35 31 27  15  7
683 ///   opcode1 r1 x2 b2 dl2 dh2 opcode2
684 ///        40 36 32 28  16   8       0
685 ///
enc_rxy(opcode: u16, r1: Reg, b2: Reg, x2: Reg, d2: u32) -> [u8; 6]686 fn enc_rxy(opcode: u16, r1: Reg, b2: Reg, x2: Reg, d2: u32) -> [u8; 6] {
687     let opcode1 = ((opcode >> 8) & 0xff) as u8;
688     let opcode2 = (opcode & 0xff) as u8;
689     let r1 = machreg_to_gpr_or_fpr(r1) & 0x0f;
690     let b2 = machreg_to_gpr(b2) & 0x0f;
691     let x2 = machreg_to_gpr(x2) & 0x0f;
692     let dl2_lo = (d2 & 0xff) as u8;
693     let dl2_hi = ((d2 >> 8) & 0x0f) as u8;
694     let dh2 = ((d2 >> 12) & 0xff) as u8;
695 
696     let mut enc: [u8; 6] = [0; 6];
697     enc[0] = opcode1;
698     enc[1] = r1 << 4 | x2;
699     enc[2] = b2 << 4 | dl2_hi;
700     enc[3] = dl2_lo;
701     enc[4] = dh2;
702     enc[5] = opcode2;
703     enc
704 }
705 
706 /// SI-type instructions.
707 ///
708 ///   31     23 15 11
709 ///   opcode i2 b1 d1
710 ///       24 16 12  0
711 ///
enc_si(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 4]712 fn enc_si(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 4] {
713     let opcode = (opcode & 0xff) as u8;
714     let b1 = machreg_to_gpr(b1) & 0x0f;
715     let d1_lo = (d1 & 0xff) as u8;
716     let d1_hi = ((d1 >> 8) & 0x0f) as u8;
717 
718     let mut enc: [u8; 4] = [0; 4];
719     enc[0] = opcode;
720     enc[1] = i2;
721     enc[2] = b1 << 4 | d1_hi;
722     enc[3] = d1_lo;
723     enc
724 }
725 
726 /// SIL-type instructions.
727 ///
728 ///   47     31 27 15
729 ///   opcode b1 d1 i2
730 ///       32 28 16  0
731 ///
enc_sil(opcode: u16, b1: Reg, d1: u32, i2: i16) -> [u8; 6]732 fn enc_sil(opcode: u16, b1: Reg, d1: u32, i2: i16) -> [u8; 6] {
733     let opcode1 = ((opcode >> 8) & 0xff) as u8;
734     let opcode2 = (opcode & 0xff) as u8;
735     let b1 = machreg_to_gpr(b1) & 0x0f;
736     let d1_lo = (d1 & 0xff) as u8;
737     let d1_hi = ((d1 >> 8) & 0x0f) as u8;
738 
739     let mut enc: [u8; 6] = [0; 6];
740     enc[0] = opcode1;
741     enc[1] = opcode2;
742     enc[2] = b1 << 4 | d1_hi;
743     enc[3] = d1_lo;
744     enc[4..].copy_from_slice(&i2.to_be_bytes());
745     enc
746 }
747 
748 /// SIY-type instructions.
749 ///
750 ///   47      39 31 27  15  7
751 ///   opcode1 i2 b1 dl1 dh1 opcode2
752 ///        40 32 28  16   8       0
753 ///
enc_siy(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 6]754 fn enc_siy(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 6] {
755     let opcode1 = ((opcode >> 8) & 0xff) as u8;
756     let opcode2 = (opcode & 0xff) as u8;
757     let b1 = machreg_to_gpr(b1) & 0x0f;
758     let dl1_lo = (d1 & 0xff) as u8;
759     let dl1_hi = ((d1 >> 8) & 0x0f) as u8;
760     let dh1 = ((d1 >> 12) & 0xff) as u8;
761 
762     let mut enc: [u8; 6] = [0; 6];
763     enc[0] = opcode1;
764     enc[1] = i2;
765     enc[2] = b1 << 4 | dl1_hi;
766     enc[3] = dl1_lo;
767     enc[4] = dh1;
768     enc[5] = opcode2;
769     enc
770 }
771 
772 /// VRR-type instructions.
773 ///
774 ///   47      39 35 31 27 23 19 15 11  7
775 ///   opcode1 v1 v2 v3 -  m6 m5 m4 rxb opcode2
776 ///        40 36 32 28 24 20 16 12   8       0
777 ///
enc_vrr(opcode: u16, v1: Reg, v2: Reg, v3: Reg, m4: u8, m5: u8, m6: u8) -> [u8; 6]778 fn enc_vrr(opcode: u16, v1: Reg, v2: Reg, v3: Reg, m4: u8, m5: u8, m6: u8) -> [u8; 6] {
779     let opcode1 = ((opcode >> 8) & 0xff) as u8;
780     let opcode2 = (opcode & 0xff) as u8;
781     let rxb = 0; // FIXME
782     let v1 = machreg_to_fpr(v1) & 0x0f; // FIXME
783     let v2 = machreg_to_fpr(v2) & 0x0f; // FIXME
784     let v3 = machreg_to_fpr(v3) & 0x0f; // FIXME
785     let m4 = m4 & 0x0f;
786     let m5 = m5 & 0x0f;
787     let m6 = m6 & 0x0f;
788 
789     let mut enc: [u8; 6] = [0; 6];
790     enc[0] = opcode1;
791     enc[1] = v1 << 4 | v2;
792     enc[2] = v3 << 4;
793     enc[3] = m6 << 4 | m5;
794     enc[4] = m4 << 4 | rxb;
795     enc[5] = opcode2;
796     enc
797 }
798 
799 /// VRX-type instructions.
800 ///
801 ///   47      39 35 31 27 15 11  7
802 ///   opcode1 v1 x2 b2 d2 m3 rxb opcode2
803 ///        40 36 32 28 16 12   8       0
804 ///
enc_vrx(opcode: u16, v1: Reg, b2: Reg, x2: Reg, d2: u32, m3: u8) -> [u8; 6]805 fn enc_vrx(opcode: u16, v1: Reg, b2: Reg, x2: Reg, d2: u32, m3: u8) -> [u8; 6] {
806     let opcode1 = ((opcode >> 8) & 0xff) as u8;
807     let opcode2 = (opcode & 0xff) as u8;
808     let rxb = 0; // FIXME
809     let v1 = machreg_to_fpr(v1) & 0x0f; // FIXME
810     let b2 = machreg_to_gpr(b2) & 0x0f;
811     let x2 = machreg_to_gpr(x2) & 0x0f;
812     let d2_lo = (d2 & 0xff) as u8;
813     let d2_hi = ((d2 >> 8) & 0x0f) as u8;
814     let m3 = m3 & 0x0f;
815 
816     let mut enc: [u8; 6] = [0; 6];
817     enc[0] = opcode1;
818     enc[1] = v1 << 4 | x2;
819     enc[2] = b2 << 4 | d2_hi;
820     enc[3] = d2_lo;
821     enc[4] = m3 << 4 | rxb;
822     enc[5] = opcode2;
823     enc
824 }
825 
826 /// Emit encoding to sink.
put(sink: &mut MachBuffer<Inst>, enc: &[u8])827 fn put(sink: &mut MachBuffer<Inst>, enc: &[u8]) {
828     for byte in enc {
829         sink.put1(*byte);
830     }
831 }
832 
833 /// Emit encoding to sink, adding a trap on the last byte.
put_with_trap(sink: &mut MachBuffer<Inst>, enc: &[u8], srcloc: SourceLoc, trap_code: TrapCode)834 fn put_with_trap(sink: &mut MachBuffer<Inst>, enc: &[u8], srcloc: SourceLoc, trap_code: TrapCode) {
835     let len = enc.len();
836     for i in 0..len - 1 {
837         sink.put1(enc[i]);
838     }
839     sink.add_trap(srcloc, trap_code);
840     sink.put1(enc[len - 1]);
841 }
842 
843 /// Emit encoding to sink, adding a relocation at byte offset.
put_with_reloc( sink: &mut MachBuffer<Inst>, enc: &[u8], offset: usize, ri2_srcloc: SourceLoc, ri2_reloc: Reloc, ri2_name: &ExternalName, ri2_offset: i64, )844 fn put_with_reloc(
845     sink: &mut MachBuffer<Inst>,
846     enc: &[u8],
847     offset: usize,
848     ri2_srcloc: SourceLoc,
849     ri2_reloc: Reloc,
850     ri2_name: &ExternalName,
851     ri2_offset: i64,
852 ) {
853     let len = enc.len();
854     for i in 0..offset {
855         sink.put1(enc[i]);
856     }
857     sink.add_reloc(ri2_srcloc, ri2_reloc, ri2_name, ri2_offset + offset as i64);
858     for i in offset..len {
859         sink.put1(enc[i]);
860     }
861 }
862 
863 /// State carried between emissions of a sequence of instructions.
864 #[derive(Default, Clone, Debug)]
865 pub struct EmitState {
866     pub(crate) initial_sp_offset: i64,
867     pub(crate) virtual_sp_offset: i64,
868     /// Safepoint stack map for upcoming instruction, as provided to `pre_safepoint()`.
869     stack_map: Option<StackMap>,
870     /// Current source-code location corresponding to instruction to be emitted.
871     cur_srcloc: SourceLoc,
872 }
873 
874 impl MachInstEmitState<Inst> for EmitState {
new(abi: &dyn ABICallee<I = Inst>) -> Self875     fn new(abi: &dyn ABICallee<I = Inst>) -> Self {
876         EmitState {
877             virtual_sp_offset: 0,
878             initial_sp_offset: abi.frame_size() as i64,
879             stack_map: None,
880             cur_srcloc: SourceLoc::default(),
881         }
882     }
883 
pre_safepoint(&mut self, stack_map: StackMap)884     fn pre_safepoint(&mut self, stack_map: StackMap) {
885         self.stack_map = Some(stack_map);
886     }
887 
pre_sourceloc(&mut self, srcloc: SourceLoc)888     fn pre_sourceloc(&mut self, srcloc: SourceLoc) {
889         self.cur_srcloc = srcloc;
890     }
891 }
892 
893 impl EmitState {
take_stack_map(&mut self) -> Option<StackMap>894     fn take_stack_map(&mut self) -> Option<StackMap> {
895         self.stack_map.take()
896     }
897 
clear_post_insn(&mut self)898     fn clear_post_insn(&mut self) {
899         self.stack_map = None;
900     }
901 
cur_srcloc(&self) -> SourceLoc902     fn cur_srcloc(&self) -> SourceLoc {
903         self.cur_srcloc
904     }
905 }
906 
907 /// Constant state used during function compilation.
908 pub struct EmitInfo {
909     flags: settings::Flags,
910     isa_flags: s390x_settings::Flags,
911 }
912 
913 impl EmitInfo {
new(flags: settings::Flags, isa_flags: s390x_settings::Flags) -> Self914     pub(crate) fn new(flags: settings::Flags, isa_flags: s390x_settings::Flags) -> Self {
915         Self { flags, isa_flags }
916     }
917 }
918 
919 impl MachInstEmitInfo for EmitInfo {
flags(&self) -> &settings::Flags920     fn flags(&self) -> &settings::Flags {
921         &self.flags
922     }
923 }
924 
925 impl MachInstEmit for Inst {
926     type State = EmitState;
927     type Info = EmitInfo;
928 
emit(&self, sink: &mut MachBuffer<Inst>, emit_info: &Self::Info, state: &mut EmitState)929     fn emit(&self, sink: &mut MachBuffer<Inst>, emit_info: &Self::Info, state: &mut EmitState) {
930         // Verify that we can emit this Inst in the current ISA
931         let matches_isa_flags = |iset_requirement: &InstructionSet| -> bool {
932             match iset_requirement {
933                 // Baseline ISA is z14
934                 InstructionSet::Base => true,
935                 // Miscellaneous-Instruction-Extensions Facility 2 (z15)
936                 InstructionSet::MIE2 => emit_info.isa_flags.has_mie2(),
937                 // Vector-Enhancements Facility 2 (z15)
938                 InstructionSet::VXRS_EXT2 => emit_info.isa_flags.has_vxrs_ext2(),
939             }
940         };
941         let isa_requirements = self.available_in_isa();
942         if !matches_isa_flags(&isa_requirements) {
943             panic!(
944                 "Cannot emit inst '{:?}' for target; failed to match ISA requirements: {:?}",
945                 self, isa_requirements
946             )
947         }
948 
949         // N.B.: we *must* not exceed the "worst-case size" used to compute
950         // where to insert islands, except when islands are explicitly triggered
951         // (with an `EmitIsland`). We check this in debug builds. This is `mut`
952         // to allow disabling the check for `JTSequence`, which is always
953         // emitted following an `EmitIsland`.
954         let mut start_off = sink.cur_offset();
955 
956         match self {
957             &Inst::AluRRR { alu_op, rd, rn, rm } => {
958                 let (opcode, have_rr) = match alu_op {
959                     ALUOp::Add32 => (0xb9f8, true),     // ARK
960                     ALUOp::Add64 => (0xb9e8, true),     // AGRK
961                     ALUOp::Sub32 => (0xb9f9, true),     // SRK
962                     ALUOp::Sub64 => (0xb9e9, true),     // SGRK
963                     ALUOp::Mul32 => (0xb9fd, true),     // MSRKC
964                     ALUOp::Mul64 => (0xb9ed, true),     // MSGRKC
965                     ALUOp::And32 => (0xb9f4, true),     // NRK
966                     ALUOp::And64 => (0xb9e4, true),     // NGRK
967                     ALUOp::Orr32 => (0xb9f6, true),     // ORK
968                     ALUOp::Orr64 => (0xb9e6, true),     // OGRK
969                     ALUOp::Xor32 => (0xb9f7, true),     // XRK
970                     ALUOp::Xor64 => (0xb9e7, true),     // XGRK
971                     ALUOp::AndNot32 => (0xb974, false), // NNRK
972                     ALUOp::AndNot64 => (0xb964, false), // NNGRK
973                     ALUOp::OrrNot32 => (0xb976, false), // NORK
974                     ALUOp::OrrNot64 => (0xb966, false), // NOGRK
975                     ALUOp::XorNot32 => (0xb977, false), // NXRK
976                     ALUOp::XorNot64 => (0xb967, false), // NXGRK
977                     _ => unreachable!(),
978                 };
979                 if have_rr && rd.to_reg() == rn {
980                     let inst = Inst::AluRR { alu_op, rd, rm };
981                     inst.emit(sink, emit_info, state);
982                 } else {
983                     put(sink, &enc_rrf_ab(opcode, rd.to_reg(), rn, rm, 0));
984                 }
985             }
986             &Inst::AluRRSImm16 {
987                 alu_op,
988                 rd,
989                 rn,
990                 imm,
991             } => {
992                 if rd.to_reg() == rn {
993                     let inst = Inst::AluRSImm16 { alu_op, rd, imm };
994                     inst.emit(sink, emit_info, state);
995                 } else {
996                     let opcode = match alu_op {
997                         ALUOp::Add32 => 0xecd8, // AHIK
998                         ALUOp::Add64 => 0xecd9, // AGHIK
999                         _ => unreachable!(),
1000                     };
1001                     put(sink, &enc_rie_d(opcode, rd.to_reg(), rn, imm as u16));
1002                 }
1003             }
1004             &Inst::AluRR { alu_op, rd, rm } => {
1005                 let (opcode, is_rre) = match alu_op {
1006                     ALUOp::Add32 => (0x1a, false),       // AR
1007                     ALUOp::Add64 => (0xb908, true),      // AGR
1008                     ALUOp::Add64Ext32 => (0xb918, true), // AGFR
1009                     ALUOp::Sub32 => (0x1b, false),       // SR
1010                     ALUOp::Sub64 => (0xb909, true),      // SGR
1011                     ALUOp::Sub64Ext32 => (0xb919, true), // SGFR
1012                     ALUOp::Mul32 => (0xb252, true),      // MSR
1013                     ALUOp::Mul64 => (0xb90c, true),      // MSGR
1014                     ALUOp::Mul64Ext32 => (0xb91c, true), // MSGFR
1015                     ALUOp::And32 => (0x14, false),       // NR
1016                     ALUOp::And64 => (0xb980, true),      // NGR
1017                     ALUOp::Orr32 => (0x16, false),       // OR
1018                     ALUOp::Orr64 => (0xb981, true),      // OGR
1019                     ALUOp::Xor32 => (0x17, false),       // XR
1020                     ALUOp::Xor64 => (0xb982, true),      // XGR
1021                     _ => unreachable!(),
1022                 };
1023                 if is_rre {
1024                     put(sink, &enc_rre(opcode, rd.to_reg(), rm));
1025                 } else {
1026                     put(sink, &enc_rr(opcode, rd.to_reg(), rm));
1027                 }
1028             }
1029             &Inst::AluRX {
1030                 alu_op,
1031                 rd,
1032                 ref mem,
1033             } => {
1034                 let (opcode_rx, opcode_rxy) = match alu_op {
1035                     ALUOp::Add32 => (Some(0x5a), Some(0xe35a)),      // A(Y)
1036                     ALUOp::Add32Ext16 => (Some(0x4a), Some(0xe34a)), // AH(Y)
1037                     ALUOp::Add64 => (None, Some(0xe308)),            // AG
1038                     ALUOp::Add64Ext16 => (None, Some(0xe338)),       // AGH
1039                     ALUOp::Add64Ext32 => (None, Some(0xe318)),       // AGF
1040                     ALUOp::Sub32 => (Some(0x5b), Some(0xe35b)),      // S(Y)
1041                     ALUOp::Sub32Ext16 => (Some(0x4b), Some(0xe37b)), // SH(Y)
1042                     ALUOp::Sub64 => (None, Some(0xe309)),            // SG
1043                     ALUOp::Sub64Ext16 => (None, Some(0xe339)),       // SGH
1044                     ALUOp::Sub64Ext32 => (None, Some(0xe319)),       // SGF
1045                     ALUOp::Mul32 => (Some(0x71), Some(0xe351)),      // MS(Y)
1046                     ALUOp::Mul32Ext16 => (Some(0x4c), Some(0xe37c)), // MH(Y)
1047                     ALUOp::Mul64 => (None, Some(0xe30c)),            // MSG
1048                     ALUOp::Mul64Ext16 => (None, Some(0xe33c)),       // MSH
1049                     ALUOp::Mul64Ext32 => (None, Some(0xe31c)),       // MSGF
1050                     ALUOp::And32 => (Some(0x54), Some(0xe354)),      // N(Y)
1051                     ALUOp::And64 => (None, Some(0xe380)),            // NG
1052                     ALUOp::Orr32 => (Some(0x56), Some(0xe356)),      // O(Y)
1053                     ALUOp::Orr64 => (None, Some(0xe381)),            // OG
1054                     ALUOp::Xor32 => (Some(0x57), Some(0xe357)),      // X(Y)
1055                     ALUOp::Xor64 => (None, Some(0xe382)),            // XG
1056                     _ => unreachable!(),
1057                 };
1058                 let rd = rd.to_reg();
1059                 mem_emit(
1060                     rd, mem, opcode_rx, opcode_rxy, None, true, sink, emit_info, state,
1061                 );
1062             }
1063             &Inst::AluRSImm16 { alu_op, rd, imm } => {
1064                 let opcode = match alu_op {
1065                     ALUOp::Add32 => 0xa7a, // AHI
1066                     ALUOp::Add64 => 0xa7b, // AGHI
1067                     ALUOp::Mul32 => 0xa7c, // MHI
1068                     ALUOp::Mul64 => 0xa7d, // MGHI
1069                     _ => unreachable!(),
1070                 };
1071                 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm as u16));
1072             }
1073             &Inst::AluRSImm32 { alu_op, rd, imm } => {
1074                 let opcode = match alu_op {
1075                     ALUOp::Add32 => 0xc29, // AFI
1076                     ALUOp::Add64 => 0xc28, // AGFI
1077                     ALUOp::Mul32 => 0xc21, // MSFI
1078                     ALUOp::Mul64 => 0xc20, // MSGFI
1079                     _ => unreachable!(),
1080                 };
1081                 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm as u32));
1082             }
1083             &Inst::AluRUImm32 { alu_op, rd, imm } => {
1084                 let opcode = match alu_op {
1085                     ALUOp::Add32 => 0xc2b, // ALFI
1086                     ALUOp::Add64 => 0xc2a, // ALGFI
1087                     ALUOp::Sub32 => 0xc25, // SLFI
1088                     ALUOp::Sub64 => 0xc24, // SLGFI
1089                     _ => unreachable!(),
1090                 };
1091                 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm));
1092             }
1093             &Inst::AluRUImm16Shifted { alu_op, rd, imm } => {
1094                 let opcode = match (alu_op, imm.shift) {
1095                     (ALUOp::And32, 0) => 0xa57, // NILL
1096                     (ALUOp::And32, 1) => 0xa56, // NILH
1097                     (ALUOp::And64, 0) => 0xa57, // NILL
1098                     (ALUOp::And64, 1) => 0xa56, // NILH
1099                     (ALUOp::And64, 2) => 0xa55, // NIHL
1100                     (ALUOp::And64, 3) => 0xa54, // NIHL
1101                     (ALUOp::Orr32, 0) => 0xa5b, // OILL
1102                     (ALUOp::Orr32, 1) => 0xa5a, // OILH
1103                     (ALUOp::Orr64, 0) => 0xa5b, // OILL
1104                     (ALUOp::Orr64, 1) => 0xa5a, // OILH
1105                     (ALUOp::Orr64, 2) => 0xa59, // OIHL
1106                     (ALUOp::Orr64, 3) => 0xa58, // OIHH
1107                     _ => unreachable!(),
1108                 };
1109                 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm.bits));
1110             }
1111             &Inst::AluRUImm32Shifted { alu_op, rd, imm } => {
1112                 let opcode = match (alu_op, imm.shift) {
1113                     (ALUOp::And32, 0) => 0xc0b, // NILF
1114                     (ALUOp::And64, 0) => 0xc0b, // NILF
1115                     (ALUOp::And64, 1) => 0xc0a, // NIHF
1116                     (ALUOp::Orr32, 0) => 0xc0d, // OILF
1117                     (ALUOp::Orr64, 0) => 0xc0d, // OILF
1118                     (ALUOp::Orr64, 1) => 0xc0c, // OILF
1119                     (ALUOp::Xor32, 0) => 0xc07, // XILF
1120                     (ALUOp::Xor64, 0) => 0xc07, // XILF
1121                     (ALUOp::Xor64, 1) => 0xc06, // XILH
1122                     _ => unreachable!(),
1123                 };
1124                 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits));
1125             }
1126 
1127             &Inst::SMulWide { rn, rm } => {
1128                 let opcode = 0xb9ec; // MGRK
1129                 put(sink, &enc_rrf_ab(opcode, gpr(0), rn, rm, 0));
1130             }
1131             &Inst::UMulWide { rn } => {
1132                 let opcode = 0xb986; // MLGR
1133                 put(sink, &enc_rre(opcode, gpr(0), rn));
1134             }
1135             &Inst::SDivMod32 { rn } => {
1136                 let opcode = 0xb91d; // DSGFR
1137                 let srcloc = state.cur_srcloc();
1138                 let trap_code = TrapCode::IntegerDivisionByZero;
1139                 put_with_trap(sink, &enc_rre(opcode, gpr(0), rn), srcloc, trap_code);
1140             }
1141             &Inst::SDivMod64 { rn } => {
1142                 let opcode = 0xb90d; // DSGR
1143                 let srcloc = state.cur_srcloc();
1144                 let trap_code = TrapCode::IntegerDivisionByZero;
1145                 put_with_trap(sink, &enc_rre(opcode, gpr(0), rn), srcloc, trap_code);
1146             }
1147             &Inst::UDivMod32 { rn } => {
1148                 let opcode = 0xb997; // DLR
1149                 let srcloc = state.cur_srcloc();
1150                 let trap_code = TrapCode::IntegerDivisionByZero;
1151                 put_with_trap(sink, &enc_rre(opcode, gpr(0), rn), srcloc, trap_code);
1152             }
1153             &Inst::UDivMod64 { rn } => {
1154                 let opcode = 0xb987; // DLGR
1155                 let srcloc = state.cur_srcloc();
1156                 let trap_code = TrapCode::IntegerDivisionByZero;
1157                 put_with_trap(sink, &enc_rre(opcode, gpr(0), rn), srcloc, trap_code);
1158             }
1159             &Inst::Flogr { rn } => {
1160                 let opcode = 0xb983; // FLOGR
1161                 put(sink, &enc_rre(opcode, gpr(0), rn));
1162             }
1163 
1164             &Inst::ShiftRR {
1165                 shift_op,
1166                 rd,
1167                 rn,
1168                 shift_imm,
1169                 shift_reg,
1170             } => {
1171                 let opcode = match shift_op {
1172                     ShiftOp::RotL32 => 0xeb1d, // RLL
1173                     ShiftOp::RotL64 => 0xeb1c, // RLLG
1174                     ShiftOp::LShL32 => 0xebdf, // SLLK  (SLL ?)
1175                     ShiftOp::LShL64 => 0xeb0d, // SLLG
1176                     ShiftOp::LShR32 => 0xebde, // SRLK  (SRL ?)
1177                     ShiftOp::LShR64 => 0xeb0c, // SRLG
1178                     ShiftOp::AShR32 => 0xebdc, // SRAK  (SRA ?)
1179                     ShiftOp::AShR64 => 0xeb0a, // SRAG
1180                 };
1181                 let shift_reg = match shift_reg {
1182                     Some(reg) => reg,
1183                     None => zero_reg(),
1184                 };
1185                 put(
1186                     sink,
1187                     &enc_rsy(opcode, rd.to_reg(), rn, shift_reg, shift_imm.bits()),
1188                 );
1189             }
1190 
1191             &Inst::UnaryRR { op, rd, rn } => {
1192                 match op {
1193                     UnaryOp::Abs32 => {
1194                         let opcode = 0x10; // LPR
1195                         put(sink, &enc_rr(opcode, rd.to_reg(), rn));
1196                     }
1197                     UnaryOp::Abs64 => {
1198                         let opcode = 0xb900; // LPGR
1199                         put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1200                     }
1201                     UnaryOp::Abs64Ext32 => {
1202                         let opcode = 0xb910; // LPGFR
1203                         put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1204                     }
1205                     UnaryOp::Neg32 => {
1206                         let opcode = 0x13; // LCR
1207                         put(sink, &enc_rr(opcode, rd.to_reg(), rn));
1208                     }
1209                     UnaryOp::Neg64 => {
1210                         let opcode = 0xb903; // LCGR
1211                         put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1212                     }
1213                     UnaryOp::Neg64Ext32 => {
1214                         let opcode = 0xb913; // LCGFR
1215                         put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1216                     }
1217                     UnaryOp::PopcntByte => {
1218                         let opcode = 0xb9e1; // POPCNT
1219                         put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, 0, 0));
1220                     }
1221                     UnaryOp::PopcntReg => {
1222                         let opcode = 0xb9e1; // POPCNT
1223                         put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, 8, 0));
1224                     }
1225                 }
1226             }
1227 
1228             &Inst::Extend {
1229                 rd,
1230                 rn,
1231                 signed,
1232                 from_bits,
1233                 to_bits,
1234             } => {
1235                 let opcode = match (signed, from_bits, to_bits) {
1236                     (_, 1, 32) => 0xb926,      // LBR
1237                     (_, 1, 64) => 0xb906,      // LGBR
1238                     (false, 8, 32) => 0xb994,  // LLCR
1239                     (false, 8, 64) => 0xb984,  // LLGCR
1240                     (true, 8, 32) => 0xb926,   // LBR
1241                     (true, 8, 64) => 0xb906,   // LGBR
1242                     (false, 16, 32) => 0xb995, // LLHR
1243                     (false, 16, 64) => 0xb985, // LLGHR
1244                     (true, 16, 32) => 0xb927,  // LHR
1245                     (true, 16, 64) => 0xb907,  // LGHR
1246                     (false, 32, 64) => 0xb916, // LLGFR
1247                     (true, 32, 64) => 0xb914,  // LGFR
1248                     _ => panic!(
1249                         "Unsupported extend combination: signed = {}, from_bits = {}, to_bits = {}",
1250                         signed, from_bits, to_bits
1251                     ),
1252                 };
1253                 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1254             }
1255 
1256             &Inst::CmpRR { op, rn, rm } => {
1257                 let (opcode, is_rre) = match op {
1258                     CmpOp::CmpS32 => (0x19, false),       // CR
1259                     CmpOp::CmpS64 => (0xb920, true),      // CGR
1260                     CmpOp::CmpS64Ext32 => (0xb930, true), // CGFR
1261                     CmpOp::CmpL32 => (0x15, false),       // CLR
1262                     CmpOp::CmpL64 => (0xb921, true),      // CLGR
1263                     CmpOp::CmpL64Ext32 => (0xb931, true), // CLGFR
1264                     _ => unreachable!(),
1265                 };
1266                 if is_rre {
1267                     put(sink, &enc_rre(opcode, rn, rm));
1268                 } else {
1269                     put(sink, &enc_rr(opcode, rn, rm));
1270                 }
1271             }
1272             &Inst::CmpRX { op, rn, ref mem } => {
1273                 let (opcode_rx, opcode_rxy, opcode_ril) = match op {
1274                     CmpOp::CmpS32 => (Some(0x59), Some(0xe359), Some(0xc6d)), // C(Y), CRL
1275                     CmpOp::CmpS32Ext16 => (Some(0x49), Some(0xe379), Some(0xc65)), // CH(Y), CHRL
1276                     CmpOp::CmpS64 => (None, Some(0xe320), Some(0xc68)),       // CG, CGRL
1277                     CmpOp::CmpS64Ext16 => (None, Some(0xe334), Some(0xc64)),  // CGH, CGHRL
1278                     CmpOp::CmpS64Ext32 => (None, Some(0xe330), Some(0xc6c)),  // CGF, CGFRL
1279                     CmpOp::CmpL32 => (Some(0x55), Some(0xe355), Some(0xc6f)), // CL(Y), CLRL
1280                     CmpOp::CmpL32Ext16 => (None, None, Some(0xc67)),          // CLHRL
1281                     CmpOp::CmpL64 => (None, Some(0xe321), Some(0xc6a)),       // CLG, CLGRL
1282                     CmpOp::CmpL64Ext16 => (None, None, Some(0xc66)),          // CLGHRL
1283                     CmpOp::CmpL64Ext32 => (None, Some(0xe331), Some(0xc6e)),  // CLGF, CLGFRL
1284                 };
1285                 mem_emit(
1286                     rn, mem, opcode_rx, opcode_rxy, opcode_ril, true, sink, emit_info, state,
1287                 );
1288             }
1289             &Inst::CmpRSImm16 { op, rn, imm } => {
1290                 let opcode = match op {
1291                     CmpOp::CmpS32 => 0xa7e, // CHI
1292                     CmpOp::CmpS64 => 0xa7f, // CGHI
1293                     _ => unreachable!(),
1294                 };
1295                 put(sink, &enc_ri_a(opcode, rn, imm as u16));
1296             }
1297             &Inst::CmpRSImm32 { op, rn, imm } => {
1298                 let opcode = match op {
1299                     CmpOp::CmpS32 => 0xc2d, // CFI
1300                     CmpOp::CmpS64 => 0xc2c, // CGFI
1301                     _ => unreachable!(),
1302                 };
1303                 put(sink, &enc_ril_a(opcode, rn, imm as u32));
1304             }
1305             &Inst::CmpRUImm32 { op, rn, imm } => {
1306                 let opcode = match op {
1307                     CmpOp::CmpL32 => 0xc2f, // CLFI
1308                     CmpOp::CmpL64 => 0xc2e, // CLGFI
1309                     _ => unreachable!(),
1310                 };
1311                 put(sink, &enc_ril_a(opcode, rn, imm));
1312             }
1313             &Inst::CmpTrapRR {
1314                 op,
1315                 rn,
1316                 rm,
1317                 cond,
1318                 trap_code,
1319             } => {
1320                 let opcode = match op {
1321                     CmpOp::CmpS32 => 0xb972, // CRT
1322                     CmpOp::CmpS64 => 0xb960, // CGRT
1323                     CmpOp::CmpL32 => 0xb973, // CLRT
1324                     CmpOp::CmpL64 => 0xb961, // CLGRT
1325                     _ => unreachable!(),
1326                 };
1327                 let srcloc = state.cur_srcloc();
1328                 put_with_trap(
1329                     sink,
1330                     &enc_rrf_cde(opcode, rn, rm, cond.bits(), 0),
1331                     srcloc,
1332                     trap_code,
1333                 );
1334             }
1335             &Inst::CmpTrapRSImm16 {
1336                 op,
1337                 rn,
1338                 imm,
1339                 cond,
1340                 trap_code,
1341             } => {
1342                 let opcode = match op {
1343                     CmpOp::CmpS32 => 0xec72, // CIT
1344                     CmpOp::CmpS64 => 0xec70, // CGIT
1345                     _ => unreachable!(),
1346                 };
1347                 let srcloc = state.cur_srcloc();
1348                 put_with_trap(
1349                     sink,
1350                     &enc_rie_a(opcode, rn, imm as u16, cond.bits()),
1351                     srcloc,
1352                     trap_code,
1353                 );
1354             }
1355             &Inst::CmpTrapRUImm16 {
1356                 op,
1357                 rn,
1358                 imm,
1359                 cond,
1360                 trap_code,
1361             } => {
1362                 let opcode = match op {
1363                     CmpOp::CmpL32 => 0xec73, // CLFIT
1364                     CmpOp::CmpL64 => 0xec71, // CLGIT
1365                     _ => unreachable!(),
1366                 };
1367                 let srcloc = state.cur_srcloc();
1368                 put_with_trap(
1369                     sink,
1370                     &enc_rie_a(opcode, rn, imm, cond.bits()),
1371                     srcloc,
1372                     trap_code,
1373                 );
1374             }
1375 
1376             &Inst::AtomicRmw {
1377                 alu_op,
1378                 rd,
1379                 rn,
1380                 ref mem,
1381             } => {
1382                 let opcode = match alu_op {
1383                     ALUOp::Add32 => 0xebf8, // LAA
1384                     ALUOp::Add64 => 0xebe8, // LAAG
1385                     ALUOp::And32 => 0xebf4, // LAN
1386                     ALUOp::And64 => 0xebe4, // LANG
1387                     ALUOp::Orr32 => 0xebf6, // LAO
1388                     ALUOp::Orr64 => 0xebe6, // LAOG
1389                     ALUOp::Xor32 => 0xebf7, // LAX
1390                     ALUOp::Xor64 => 0xebe7, // LAXG
1391                     _ => unreachable!(),
1392                 };
1393 
1394                 let rd = rd.to_reg();
1395                 mem_rs_emit(
1396                     rd,
1397                     rn,
1398                     mem,
1399                     None,
1400                     Some(opcode),
1401                     true,
1402                     sink,
1403                     emit_info,
1404                     state,
1405                 );
1406             }
1407             &Inst::AtomicCas32 { rd, rn, ref mem } | &Inst::AtomicCas64 { rd, rn, ref mem } => {
1408                 let (opcode_rs, opcode_rsy) = match self {
1409                     &Inst::AtomicCas32 { .. } => (Some(0xba), Some(0xeb14)), // CS(Y)
1410                     &Inst::AtomicCas64 { .. } => (None, Some(0xeb30)),       // CSG
1411                     _ => unreachable!(),
1412                 };
1413 
1414                 let rd = rd.to_reg();
1415                 mem_rs_emit(
1416                     rd, rn, mem, opcode_rs, opcode_rsy, true, sink, emit_info, state,
1417                 );
1418             }
1419             &Inst::Fence => {
1420                 put(sink, &enc_e(0x07e0));
1421             }
1422 
1423             &Inst::Load32 { rd, ref mem }
1424             | &Inst::Load32ZExt8 { rd, ref mem }
1425             | &Inst::Load32SExt8 { rd, ref mem }
1426             | &Inst::Load32ZExt16 { rd, ref mem }
1427             | &Inst::Load32SExt16 { rd, ref mem }
1428             | &Inst::Load64 { rd, ref mem }
1429             | &Inst::Load64ZExt8 { rd, ref mem }
1430             | &Inst::Load64SExt8 { rd, ref mem }
1431             | &Inst::Load64ZExt16 { rd, ref mem }
1432             | &Inst::Load64SExt16 { rd, ref mem }
1433             | &Inst::Load64ZExt32 { rd, ref mem }
1434             | &Inst::Load64SExt32 { rd, ref mem }
1435             | &Inst::LoadRev16 { rd, ref mem }
1436             | &Inst::LoadRev32 { rd, ref mem }
1437             | &Inst::LoadRev64 { rd, ref mem }
1438             | &Inst::FpuLoad32 { rd, ref mem }
1439             | &Inst::FpuLoad64 { rd, ref mem } => {
1440                 let (opcode_rx, opcode_rxy, opcode_ril) = match self {
1441                     &Inst::Load32 { .. } => (Some(0x58), Some(0xe358), Some(0xc4d)), // L(Y), LRL
1442                     &Inst::Load32ZExt8 { .. } => (None, Some(0xe394), None),         // LLC
1443                     &Inst::Load32SExt8 { .. } => (None, Some(0xe376), None),         // LB
1444                     &Inst::Load32ZExt16 { .. } => (None, Some(0xe395), Some(0xc42)), // LLH, LLHRL
1445                     &Inst::Load32SExt16 { .. } => (Some(0x48), Some(0xe378), Some(0xc45)), // LH(Y), LHRL
1446                     &Inst::Load64 { .. } => (None, Some(0xe304), Some(0xc48)), // LG, LGRL
1447                     &Inst::Load64ZExt8 { .. } => (None, Some(0xe390), None),   // LLGC
1448                     &Inst::Load64SExt8 { .. } => (None, Some(0xe377), None),   // LGB
1449                     &Inst::Load64ZExt16 { .. } => (None, Some(0xe391), Some(0xc46)), // LLGH, LLGHRL
1450                     &Inst::Load64SExt16 { .. } => (None, Some(0xe315), Some(0xc44)), // LGH, LGHRL
1451                     &Inst::Load64ZExt32 { .. } => (None, Some(0xe316), Some(0xc4e)), // LLGF, LLGFRL
1452                     &Inst::Load64SExt32 { .. } => (None, Some(0xe314), Some(0xc4c)), // LGF, LGFRL
1453                     &Inst::LoadRev16 { .. } => (None, Some(0xe31f), None),     // LRVH
1454                     &Inst::LoadRev32 { .. } => (None, Some(0xe31e), None),     // LRV
1455                     &Inst::LoadRev64 { .. } => (None, Some(0xe30f), None),     // LRVG
1456                     &Inst::FpuLoad32 { .. } => (Some(0x78), Some(0xed64), None), // LE(Y)
1457                     &Inst::FpuLoad64 { .. } => (Some(0x68), Some(0xed65), None), // LD(Y)
1458                     _ => unreachable!(),
1459                 };
1460                 let rd = rd.to_reg();
1461                 mem_emit(
1462                     rd, mem, opcode_rx, opcode_rxy, opcode_ril, true, sink, emit_info, state,
1463                 );
1464             }
1465             &Inst::FpuLoadRev32 { rd, ref mem } | &Inst::FpuLoadRev64 { rd, ref mem } => {
1466                 let opcode = match self {
1467                     &Inst::FpuLoadRev32 { .. } => 0xe603, // VLEBRF
1468                     &Inst::FpuLoadRev64 { .. } => 0xe602, // VLEBRG
1469                     _ => unreachable!(),
1470                 };
1471 
1472                 let (mem_insts, mem) = mem_finalize(mem, state, true, false, false, true);
1473                 for inst in mem_insts.into_iter() {
1474                     inst.emit(sink, emit_info, state);
1475                 }
1476 
1477                 let srcloc = state.cur_srcloc();
1478                 if srcloc != SourceLoc::default() && mem.can_trap() {
1479                     sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
1480                 }
1481 
1482                 match &mem {
1483                     &MemArg::BXD12 {
1484                         base, index, disp, ..
1485                     } => {
1486                         put(
1487                             sink,
1488                             &enc_vrx(opcode, rd.to_reg(), base, index, disp.bits(), 0),
1489                         );
1490                     }
1491                     _ => unreachable!(),
1492                 }
1493             }
1494 
1495             &Inst::Store8 { rd, ref mem }
1496             | &Inst::Store16 { rd, ref mem }
1497             | &Inst::Store32 { rd, ref mem }
1498             | &Inst::Store64 { rd, ref mem }
1499             | &Inst::StoreRev16 { rd, ref mem }
1500             | &Inst::StoreRev32 { rd, ref mem }
1501             | &Inst::StoreRev64 { rd, ref mem }
1502             | &Inst::FpuStore32 { rd, ref mem }
1503             | &Inst::FpuStore64 { rd, ref mem } => {
1504                 let (opcode_rx, opcode_rxy, opcode_ril) = match self {
1505                     &Inst::Store8 { .. } => (Some(0x42), Some(0xe372), None), // STC(Y)
1506                     &Inst::Store16 { .. } => (Some(0x40), Some(0xe370), Some(0xc47)), // STH(Y), STHRL
1507                     &Inst::Store32 { .. } => (Some(0x50), Some(0xe350), Some(0xc4f)), // ST(Y), STRL
1508                     &Inst::Store64 { .. } => (None, Some(0xe324), Some(0xc4b)),       // STG, STGRL
1509                     &Inst::StoreRev16 { .. } => (None, Some(0xe33f), None),           // STRVH
1510                     &Inst::StoreRev32 { .. } => (None, Some(0xe33e), None),           // STRV
1511                     &Inst::StoreRev64 { .. } => (None, Some(0xe32f), None),           // STRVG
1512                     &Inst::FpuStore32 { .. } => (Some(0x70), Some(0xed66), None),     // STE(Y)
1513                     &Inst::FpuStore64 { .. } => (Some(0x60), Some(0xed67), None),     // STD(Y)
1514                     _ => unreachable!(),
1515                 };
1516                 mem_emit(
1517                     rd, mem, opcode_rx, opcode_rxy, opcode_ril, true, sink, emit_info, state,
1518                 );
1519             }
1520             &Inst::StoreImm8 { imm, ref mem } => {
1521                 let opcode_si = 0x92; // MVI
1522                 let opcode_siy = 0xeb52; // MVIY
1523                 mem_imm8_emit(
1524                     imm, mem, opcode_si, opcode_siy, true, sink, emit_info, state,
1525                 );
1526             }
1527             &Inst::StoreImm16 { imm, ref mem }
1528             | &Inst::StoreImm32SExt16 { imm, ref mem }
1529             | &Inst::StoreImm64SExt16 { imm, ref mem } => {
1530                 let opcode = match self {
1531                     &Inst::StoreImm16 { .. } => 0xe544,       // MVHHI
1532                     &Inst::StoreImm32SExt16 { .. } => 0xe54c, // MVHI
1533                     &Inst::StoreImm64SExt16 { .. } => 0xe548, // MVGHI
1534                     _ => unreachable!(),
1535                 };
1536                 mem_imm16_emit(imm, mem, opcode, true, sink, emit_info, state);
1537             }
1538             &Inst::FpuStoreRev32 { rd, ref mem } | &Inst::FpuStoreRev64 { rd, ref mem } => {
1539                 let opcode = match self {
1540                     &Inst::FpuStoreRev32 { .. } => 0xe60b, // VSTEBRF
1541                     &Inst::FpuStoreRev64 { .. } => 0xe60a, // VSTEBRG
1542                     _ => unreachable!(),
1543                 };
1544 
1545                 let (mem_insts, mem) = mem_finalize(mem, state, true, false, false, true);
1546                 for inst in mem_insts.into_iter() {
1547                     inst.emit(sink, emit_info, state);
1548                 }
1549 
1550                 let srcloc = state.cur_srcloc();
1551                 if srcloc != SourceLoc::default() && mem.can_trap() {
1552                     sink.add_trap(srcloc, TrapCode::HeapOutOfBounds);
1553                 }
1554 
1555                 match &mem {
1556                     &MemArg::BXD12 {
1557                         base, index, disp, ..
1558                     } => {
1559                         put(sink, &enc_vrx(opcode, rd, base, index, disp.bits(), 0));
1560                     }
1561                     _ => unreachable!(),
1562                 }
1563             }
1564 
1565             &Inst::LoadMultiple64 {
1566                 rt,
1567                 rt2,
1568                 addr_reg,
1569                 addr_off,
1570             } => {
1571                 let opcode = 0xeb04; // LMG
1572                 let rt = rt.to_reg();
1573                 let rt2 = rt2.to_reg();
1574                 put(sink, &enc_rsy(opcode, rt, rt2, addr_reg, addr_off.bits()));
1575             }
1576             &Inst::StoreMultiple64 {
1577                 rt,
1578                 rt2,
1579                 addr_reg,
1580                 addr_off,
1581             } => {
1582                 let opcode = 0xeb24; // STMG
1583                 put(sink, &enc_rsy(opcode, rt, rt2, addr_reg, addr_off.bits()));
1584             }
1585 
1586             &Inst::LoadAddr { rd, ref mem } => {
1587                 let opcode_rx = Some(0x41); // LA
1588                 let opcode_rxy = Some(0xe371); // LAY
1589                 let opcode_ril = Some(0xc00); // LARL
1590                 let rd = rd.to_reg();
1591                 mem_emit(
1592                     rd, mem, opcode_rx, opcode_rxy, opcode_ril, false, sink, emit_info, state,
1593                 );
1594             }
1595 
1596             &Inst::Mov64 { rd, rm } => {
1597                 let opcode = 0xb904; // LGR
1598                 put(sink, &enc_rre(opcode, rd.to_reg(), rm));
1599             }
1600             &Inst::Mov32 { rd, rm } => {
1601                 let opcode = 0x18; // LR
1602                 put(sink, &enc_rr(opcode, rd.to_reg(), rm));
1603             }
1604             &Inst::Mov32Imm { rd, imm } => {
1605                 let opcode = 0xc09; // IILF
1606                 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm));
1607             }
1608             &Inst::Mov32SImm16 { rd, imm } => {
1609                 let opcode = 0xa78; // LHI
1610                 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm as u16));
1611             }
1612             &Inst::Mov64SImm16 { rd, imm } => {
1613                 let opcode = 0xa79; // LGHI
1614                 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm as u16));
1615             }
1616             &Inst::Mov64SImm32 { rd, imm } => {
1617                 let opcode = 0xc01; // LGFI
1618                 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm as u32));
1619             }
1620             &Inst::CMov32 { rd, cond, rm } => {
1621                 let opcode = 0xb9f2; // LOCR
1622                 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rm, cond.bits(), 0));
1623             }
1624             &Inst::CMov64 { rd, cond, rm } => {
1625                 let opcode = 0xb9e2; // LOCGR
1626                 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rm, cond.bits(), 0));
1627             }
1628             &Inst::CMov32SImm16 { rd, cond, imm } => {
1629                 let opcode = 0xec42; // LOCHI
1630                 put(
1631                     sink,
1632                     &enc_rie_g(opcode, rd.to_reg(), imm as u16, cond.bits()),
1633                 );
1634             }
1635             &Inst::CMov64SImm16 { rd, cond, imm } => {
1636                 let opcode = 0xec46; // LOCGHI
1637                 put(
1638                     sink,
1639                     &enc_rie_g(opcode, rd.to_reg(), imm as u16, cond.bits()),
1640                 );
1641             }
1642             &Inst::Mov64UImm16Shifted { rd, imm } => {
1643                 let opcode = match imm.shift {
1644                     0 => 0xa5f, // LLILL
1645                     1 => 0xa5e, // LLILH
1646                     2 => 0xa5d, // LLIHL
1647                     3 => 0xa5c, // LLIHH
1648                     _ => unreachable!(),
1649                 };
1650                 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm.bits));
1651             }
1652             &Inst::Mov64UImm32Shifted { rd, imm } => {
1653                 let opcode = match imm.shift {
1654                     0 => 0xc0f, // LLILF
1655                     1 => 0xc0e, // LLIHF
1656                     _ => unreachable!(),
1657                 };
1658                 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits));
1659             }
1660             &Inst::Insert64UImm16Shifted { rd, imm } => {
1661                 let opcode = match imm.shift {
1662                     0 => 0xa53, // IILL
1663                     1 => 0xa52, // IILH
1664                     2 => 0xa51, // IIHL
1665                     3 => 0xa50, // IIHH
1666                     _ => unreachable!(),
1667                 };
1668                 put(sink, &enc_ri_a(opcode, rd.to_reg(), imm.bits));
1669             }
1670             &Inst::Insert64UImm32Shifted { rd, imm } => {
1671                 let opcode = match imm.shift {
1672                     0 => 0xc09, // IILF
1673                     1 => 0xc08, // IIHF
1674                     _ => unreachable!(),
1675                 };
1676                 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits));
1677             }
1678             &Inst::LoadExtNameFar {
1679                 rd,
1680                 ref name,
1681                 offset,
1682             } => {
1683                 let opcode = 0xa75; // BRAS
1684                 let srcloc = state.cur_srcloc();
1685                 let reg = writable_spilltmp_reg().to_reg();
1686                 put(sink, &enc_ri_b(opcode, reg, 12));
1687                 sink.add_reloc(srcloc, Reloc::Abs8, name, offset);
1688                 if emit_info.flags().emit_all_ones_funcaddrs() {
1689                     sink.put8(u64::max_value());
1690                 } else {
1691                     sink.put8(0);
1692                 }
1693                 let inst = Inst::Load64 {
1694                     rd,
1695                     mem: MemArg::reg(reg, MemFlags::trusted()),
1696                 };
1697                 inst.emit(sink, emit_info, state);
1698             }
1699 
1700             &Inst::FpuMove32 { rd, rn } => {
1701                 let opcode = 0x38; // LER
1702                 put(sink, &enc_rr(opcode, rd.to_reg(), rn));
1703             }
1704             &Inst::FpuMove64 { rd, rn } => {
1705                 let opcode = 0x28; // LDR
1706                 put(sink, &enc_rr(opcode, rd.to_reg(), rn));
1707             }
1708             &Inst::FpuCMov32 { rd, cond, rm } => {
1709                 let opcode = 0xa74; // BCR
1710                 put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 2));
1711                 let opcode = 0x38; // LER
1712                 put(sink, &enc_rr(opcode, rd.to_reg(), rm));
1713             }
1714             &Inst::FpuCMov64 { rd, cond, rm } => {
1715                 let opcode = 0xa74; // BCR
1716                 put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 2));
1717                 let opcode = 0x28; // LDR
1718                 put(sink, &enc_rr(opcode, rd.to_reg(), rm));
1719             }
1720             &Inst::MovToFpr { rd, rn } => {
1721                 let opcode = 0xb3c1; // LDGR
1722                 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1723             }
1724             &Inst::MovFromFpr { rd, rn } => {
1725                 let opcode = 0xb3cd; // LGDR
1726                 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1727             }
1728             &Inst::LoadFpuConst32 { rd, const_data } => {
1729                 let opcode = 0xa75; // BRAS
1730                 let reg = writable_spilltmp_reg().to_reg();
1731                 put(sink, &enc_ri_b(opcode, reg, 8));
1732                 sink.put4(const_data.to_bits().swap_bytes());
1733                 let inst = Inst::FpuLoad32 {
1734                     rd,
1735                     mem: MemArg::reg(reg, MemFlags::trusted()),
1736                 };
1737                 inst.emit(sink, emit_info, state);
1738             }
1739             &Inst::LoadFpuConst64 { rd, const_data } => {
1740                 let opcode = 0xa75; // BRAS
1741                 let reg = writable_spilltmp_reg().to_reg();
1742                 put(sink, &enc_ri_b(opcode, reg, 12));
1743                 sink.put8(const_data.to_bits().swap_bytes());
1744                 let inst = Inst::FpuLoad64 {
1745                     rd,
1746                     mem: MemArg::reg(reg, MemFlags::trusted()),
1747                 };
1748                 inst.emit(sink, emit_info, state);
1749             }
1750 
1751             &Inst::FpuCopysign { rd, rn, rm } => {
1752                 let opcode = 0xb372; // CPSDR
1753                 put(sink, &enc_rrf_ab(opcode, rd.to_reg(), rn, rm, 0));
1754             }
1755             &Inst::FpuRR { fpu_op, rd, rn } => {
1756                 let opcode = match fpu_op {
1757                     FPUOp1::Abs32 => 0xb300,     // LPEBR
1758                     FPUOp1::Abs64 => 0xb310,     // LPDBR
1759                     FPUOp1::Neg32 => 0xb303,     // LCEBR
1760                     FPUOp1::Neg64 => 0xb313,     // LCDBR
1761                     FPUOp1::NegAbs32 => 0xb301,  // LNEBR
1762                     FPUOp1::NegAbs64 => 0xb311,  // LNDBR
1763                     FPUOp1::Sqrt32 => 0xb314,    // SQEBR
1764                     FPUOp1::Sqrt64 => 0xb315,    // SQDBR
1765                     FPUOp1::Cvt32To64 => 0xb304, // LDEBR
1766                     FPUOp1::Cvt64To32 => 0xb344, // LEDBR
1767                 };
1768                 put(sink, &enc_rre(opcode, rd.to_reg(), rn));
1769             }
1770             &Inst::FpuRRR { fpu_op, rd, rm } => {
1771                 let opcode = match fpu_op {
1772                     FPUOp2::Add32 => 0xb30a, // AEBR
1773                     FPUOp2::Add64 => 0xb31a, // ADBR
1774                     FPUOp2::Sub32 => 0xb30b, // SEBR
1775                     FPUOp2::Sub64 => 0xb31b, // SDBR
1776                     FPUOp2::Mul32 => 0xb317, // MEEBR
1777                     FPUOp2::Mul64 => 0xb31c, // MDBR
1778                     FPUOp2::Div32 => 0xb30d, // DEBR
1779                     FPUOp2::Div64 => 0xb31d, // DDBR
1780                     _ => unimplemented!(),
1781                 };
1782                 put(sink, &enc_rre(opcode, rd.to_reg(), rm));
1783             }
1784             &Inst::FpuRRRR { fpu_op, rd, rn, rm } => {
1785                 let opcode = match fpu_op {
1786                     FPUOp3::MAdd32 => 0xb30e, // MAEBR
1787                     FPUOp3::MAdd64 => 0xb31e, // MADBR
1788                     FPUOp3::MSub32 => 0xb30f, // MSEBR
1789                     FPUOp3::MSub64 => 0xb31f, // MSDBR
1790                 };
1791                 put(sink, &enc_rrd(opcode, rd.to_reg(), rm, rn));
1792             }
1793             &Inst::FpuToInt { op, rd, rn } => {
1794                 let opcode = match op {
1795                     FpuToIntOp::F32ToI32 => 0xb398, // CFEBRA
1796                     FpuToIntOp::F32ToU32 => 0xb39c, // CLFEBR
1797                     FpuToIntOp::F32ToI64 => 0xb3a8, // CGEBRA
1798                     FpuToIntOp::F32ToU64 => 0xb3ac, // CLGEBR
1799                     FpuToIntOp::F64ToI32 => 0xb399, // CFDBRA
1800                     FpuToIntOp::F64ToU32 => 0xb39d, // CLFDBR
1801                     FpuToIntOp::F64ToI64 => 0xb3a9, // CGDBRA
1802                     FpuToIntOp::F64ToU64 => 0xb3ad, // CLGDBR
1803                 };
1804                 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, 5, 0));
1805             }
1806             &Inst::IntToFpu { op, rd, rn } => {
1807                 let opcode = match op {
1808                     IntToFpuOp::I32ToF32 => 0xb394, // CEFBRA
1809                     IntToFpuOp::U32ToF32 => 0xb390, // CELFBR
1810                     IntToFpuOp::I64ToF32 => 0xb3a4, // CEGBRA
1811                     IntToFpuOp::U64ToF32 => 0xb3a0, // CELGBR
1812                     IntToFpuOp::I32ToF64 => 0xb395, // CDFBRA
1813                     IntToFpuOp::U32ToF64 => 0xb391, // CDLFBR
1814                     IntToFpuOp::I64ToF64 => 0xb3a5, // CDGBRA
1815                     IntToFpuOp::U64ToF64 => 0xb3a1, // CDLGBR
1816                 };
1817                 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, 0, 0));
1818             }
1819             &Inst::FpuRound { op, rd, rn } => {
1820                 let (opcode, m3) = match op {
1821                     FpuRoundMode::Minus32 => (0xb357, 7),   // FIEBR
1822                     FpuRoundMode::Minus64 => (0xb35f, 7),   // FIDBR
1823                     FpuRoundMode::Plus32 => (0xb357, 6),    // FIEBR
1824                     FpuRoundMode::Plus64 => (0xb35f, 6),    // FIDBR
1825                     FpuRoundMode::Zero32 => (0xb357, 5),    // FIEBR
1826                     FpuRoundMode::Zero64 => (0xb35f, 5),    // FIDBR
1827                     FpuRoundMode::Nearest32 => (0xb357, 4), // FIEBR
1828                     FpuRoundMode::Nearest64 => (0xb35f, 4), // FIDBR
1829                 };
1830                 put(sink, &enc_rrf_cde(opcode, rd.to_reg(), rn, m3, 0));
1831             }
1832             &Inst::FpuVecRRR { fpu_op, rd, rn, rm } => {
1833                 let (opcode, m4) = match fpu_op {
1834                     FPUOp2::Max32 => (0xe7ef, 2), // VFMAX
1835                     FPUOp2::Max64 => (0xe7ef, 3), // VFMAX
1836                     FPUOp2::Min32 => (0xe7ee, 2), // VFMIN
1837                     FPUOp2::Min64 => (0xe7ee, 3), // VFMIN
1838                     _ => unimplemented!(),
1839                 };
1840                 put(sink, &enc_vrr(opcode, rd.to_reg(), rn, rm, m4, 8, 1));
1841             }
1842             &Inst::FpuCmp32 { rn, rm } => {
1843                 let opcode = 0xb309; // CEBR
1844                 put(sink, &enc_rre(opcode, rn, rm));
1845             }
1846             &Inst::FpuCmp64 { rn, rm } => {
1847                 let opcode = 0xb319; // CDBR
1848                 put(sink, &enc_rre(opcode, rn, rm));
1849             }
1850 
1851             &Inst::Call { link, ref info } => {
1852                 let opcode = 0xc05; // BRASL
1853                 let reloc = Reloc::S390xPCRel32Dbl;
1854                 let srcloc = state.cur_srcloc();
1855                 if let Some(s) = state.take_stack_map() {
1856                     sink.add_stack_map(StackMapExtent::UpcomingBytes(6), s);
1857                 }
1858                 put_with_reloc(
1859                     sink,
1860                     &enc_ril_b(opcode, link.to_reg(), 0),
1861                     2,
1862                     srcloc,
1863                     reloc,
1864                     &info.dest,
1865                     0,
1866                 );
1867                 if info.opcode.is_call() {
1868                     sink.add_call_site(srcloc, info.opcode);
1869                 }
1870             }
1871             &Inst::CallInd { link, ref info } => {
1872                 let opcode = 0x0d; // BASR
1873                 let srcloc = state.cur_srcloc();
1874                 if let Some(s) = state.take_stack_map() {
1875                     sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s);
1876                 }
1877                 put(sink, &enc_rr(opcode, link.to_reg(), info.rn));
1878                 if info.opcode.is_call() {
1879                     sink.add_call_site(srcloc, info.opcode);
1880                 }
1881             }
1882             &Inst::Ret { link } => {
1883                 let opcode = 0x07; // BCR
1884                 put(sink, &enc_rr(opcode, gpr(15), link));
1885             }
1886             &Inst::EpiloguePlaceholder => {
1887                 // Noop; this is just a placeholder for epilogues.
1888             }
1889             &Inst::Jump { ref dest } => {
1890                 let off = sink.cur_offset();
1891                 // Indicate that the jump uses a label, if so, so that a fixup can occur later.
1892                 if let Some(l) = dest.as_label() {
1893                     sink.use_label_at_offset(off, l, LabelUse::BranchRIL);
1894                     sink.add_uncond_branch(off, off + 6, l);
1895                 }
1896                 // Emit the jump itself.
1897                 let opcode = 0xc04; // BCRL
1898                 put(sink, &enc_ril_c(opcode, 15, dest.as_ril_offset_or_zero()));
1899             }
1900             &Inst::IndirectBr { rn, .. } => {
1901                 let opcode = 0x07; // BCR
1902                 put(sink, &enc_rr(opcode, gpr(15), rn));
1903             }
1904             &Inst::CondBr {
1905                 ref taken,
1906                 ref not_taken,
1907                 cond,
1908             } => {
1909                 let opcode = 0xc04; // BCRL
1910 
1911                 // Conditional part first.
1912                 let cond_off = sink.cur_offset();
1913                 if let Some(l) = taken.as_label() {
1914                     sink.use_label_at_offset(cond_off, l, LabelUse::BranchRIL);
1915                     let inverted = &enc_ril_c(opcode, cond.invert().bits(), 0);
1916                     sink.add_cond_branch(cond_off, cond_off + 6, l, inverted);
1917                 }
1918                 put(
1919                     sink,
1920                     &enc_ril_c(opcode, cond.bits(), taken.as_ril_offset_or_zero()),
1921                 );
1922 
1923                 // Unconditional part next.
1924                 let uncond_off = sink.cur_offset();
1925                 if let Some(l) = not_taken.as_label() {
1926                     sink.use_label_at_offset(uncond_off, l, LabelUse::BranchRIL);
1927                     sink.add_uncond_branch(uncond_off, uncond_off + 6, l);
1928                 }
1929                 put(
1930                     sink,
1931                     &enc_ril_c(opcode, 15, not_taken.as_ril_offset_or_zero()),
1932                 );
1933             }
1934             &Inst::OneWayCondBr { ref target, cond } => {
1935                 let opcode = 0xc04; // BCRL
1936                 if let Some(l) = target.as_label() {
1937                     sink.use_label_at_offset(sink.cur_offset(), l, LabelUse::BranchRIL);
1938                 }
1939                 put(
1940                     sink,
1941                     &enc_ril_c(opcode, cond.bits(), target.as_ril_offset_or_zero()),
1942                 );
1943             }
1944             &Inst::Nop0 => {}
1945             &Inst::Nop2 => {
1946                 put(sink, &enc_e(0x0707));
1947             }
1948             &Inst::Debugtrap => {
1949                 put(sink, &enc_e(0x0001));
1950             }
1951             &Inst::Trap { trap_code } => {
1952                 if let Some(s) = state.take_stack_map() {
1953                     sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s);
1954                 }
1955                 let srcloc = state.cur_srcloc();
1956                 put_with_trap(sink, &enc_e(0x0000), srcloc, trap_code);
1957             }
1958             &Inst::TrapIf { cond, trap_code } => {
1959                 // Branch over trap if condition is false.
1960                 let opcode = 0xa74; // BCR
1961                 put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 2));
1962                 // Now emit the actual trap.
1963                 if let Some(s) = state.take_stack_map() {
1964                     sink.add_stack_map(StackMapExtent::UpcomingBytes(2), s);
1965                 }
1966                 let srcloc = state.cur_srcloc();
1967                 put_with_trap(sink, &enc_e(0x0000), srcloc, trap_code);
1968             }
1969             &Inst::JTSequence {
1970                 ridx,
1971                 rtmp1,
1972                 rtmp2,
1973                 ref info,
1974                 ..
1975             } => {
1976                 let table_label = sink.get_label();
1977 
1978                 // This sequence is *one* instruction in the vcode, and is expanded only here at
1979                 // emission time, because we cannot allow the regalloc to insert spills/reloads in
1980                 // the middle; we depend on hardcoded PC-rel addressing below.
1981 
1982                 // Bounds-check index and branch to default.
1983                 let inst = Inst::CmpRUImm32 {
1984                     op: CmpOp::CmpL64,
1985                     rn: ridx,
1986                     imm: info.targets.len() as u32,
1987                 };
1988                 inst.emit(sink, emit_info, state);
1989                 let inst = Inst::OneWayCondBr {
1990                     target: info.default_target,
1991                     cond: Cond::from_intcc(IntCC::UnsignedGreaterThanOrEqual),
1992                 };
1993                 inst.emit(sink, emit_info, state);
1994 
1995                 // Set rtmp2 to index scaled by entry size.
1996                 let inst = Inst::ShiftRR {
1997                     shift_op: ShiftOp::LShL64,
1998                     rd: rtmp2,
1999                     rn: ridx,
2000                     shift_imm: SImm20::maybe_from_i64(2).unwrap(),
2001                     shift_reg: None,
2002                 };
2003                 inst.emit(sink, emit_info, state);
2004 
2005                 // Set rtmp1 to address of jump table.
2006                 let inst = Inst::LoadAddr {
2007                     rd: rtmp1,
2008                     mem: MemArg::Label {
2009                         target: BranchTarget::Label(table_label),
2010                     },
2011                 };
2012                 inst.emit(sink, emit_info, state);
2013 
2014                 // Set rtmp2 to value loaded out of jump table.
2015                 let inst = Inst::Load64SExt32 {
2016                     rd: rtmp2,
2017                     mem: MemArg::reg_plus_reg(rtmp1.to_reg(), rtmp2.to_reg(), MemFlags::trusted()),
2018                 };
2019                 inst.emit(sink, emit_info, state);
2020 
2021                 // Set rtmp1 to target address (rtmp1 + rtmp2).
2022                 let inst = Inst::AluRRR {
2023                     alu_op: ALUOp::Add64,
2024                     rd: rtmp1,
2025                     rn: rtmp1.to_reg(),
2026                     rm: rtmp2.to_reg(),
2027                 };
2028                 inst.emit(sink, emit_info, state);
2029 
2030                 // Branch to computed address. (`targets` here is only used for successor queries
2031                 // and is not needed for emission.)
2032                 let inst = Inst::IndirectBr {
2033                     rn: rtmp1.to_reg(),
2034                     targets: vec![],
2035                 };
2036                 inst.emit(sink, emit_info, state);
2037 
2038                 // Emit jump table (table of 32-bit offsets).
2039                 sink.bind_label(table_label);
2040                 let jt_off = sink.cur_offset();
2041                 for &target in info.targets.iter() {
2042                     let word_off = sink.cur_offset();
2043                     let off_into_table = word_off - jt_off;
2044                     sink.use_label_at_offset(
2045                         word_off,
2046                         target.as_label().unwrap(),
2047                         LabelUse::PCRel32,
2048                     );
2049                     sink.put4(off_into_table.swap_bytes());
2050                 }
2051 
2052                 // Lowering produces an EmitIsland before using a JTSequence, so we can safely
2053                 // disable the worst-case-size check in this case.
2054                 start_off = sink.cur_offset();
2055             }
2056 
2057             &Inst::VirtualSPOffsetAdj { offset } => {
2058                 log::trace!(
2059                     "virtual sp offset adjusted by {} -> {}",
2060                     offset,
2061                     state.virtual_sp_offset + offset
2062                 );
2063                 state.virtual_sp_offset += offset;
2064             }
2065 
2066             &Inst::ValueLabelMarker { .. } => {
2067                 // Nothing; this is only used to compute debug info.
2068             }
2069 
2070             &Inst::Unwind { ref inst } => {
2071                 sink.add_unwind(inst.clone());
2072             }
2073         }
2074 
2075         let end_off = sink.cur_offset();
2076         debug_assert!((end_off - start_off) <= Inst::worst_case_size());
2077 
2078         state.clear_post_insn();
2079     }
2080 
pretty_print(&self, mb_rru: Option<&RealRegUniverse>, state: &mut EmitState) -> String2081     fn pretty_print(&self, mb_rru: Option<&RealRegUniverse>, state: &mut EmitState) -> String {
2082         self.print_with_state(mb_rru, state)
2083     }
2084 }
2085