1 use crate::cdsl::ast::{var, ExprBuilder, Literal};
2 use crate::cdsl::instructions::{Bindable, Instruction, InstructionGroup};
3 use crate::cdsl::xform::{TransformGroupBuilder, TransformGroups};
4
5 use crate::shared::immediates::Immediates;
6 use crate::shared::types::Float::{F32, F64};
7 use crate::shared::types::Int::{I128, I16, I32, I64, I8};
8 use cranelift_codegen_shared::condcodes::{CondCode, IntCC};
9
10 #[allow(clippy::many_single_char_names, clippy::cognitive_complexity)]
define(insts: &InstructionGroup, imm: &Immediates) -> TransformGroups11 pub(crate) fn define(insts: &InstructionGroup, imm: &Immediates) -> TransformGroups {
12 let mut narrow = TransformGroupBuilder::new(
13 "narrow",
14 r#"
15 Legalize instructions by narrowing.
16
17 The transformations in the 'narrow' group work by expressing
18 instructions in terms of smaller types. Operations on vector types are
19 expressed in terms of vector types with fewer lanes, and integer
20 operations are expressed in terms of smaller integer types.
21 "#,
22 );
23
24 let mut widen = TransformGroupBuilder::new(
25 "widen",
26 r#"
27 Legalize instructions by widening.
28
29 The transformations in the 'widen' group work by expressing
30 instructions in terms of larger types.
31 "#,
32 );
33
34 let mut expand = TransformGroupBuilder::new(
35 "expand",
36 r#"
37 Legalize instructions by expansion.
38
39 Rewrite instructions in terms of other instructions, generally
40 operating on the same types as the original instructions.
41 "#,
42 );
43
44 // List of instructions.
45 let band = insts.by_name("band");
46 let band_imm = insts.by_name("band_imm");
47 let band_not = insts.by_name("band_not");
48 let bint = insts.by_name("bint");
49 let bitrev = insts.by_name("bitrev");
50 let bnot = insts.by_name("bnot");
51 let bor = insts.by_name("bor");
52 let bor_imm = insts.by_name("bor_imm");
53 let bor_not = insts.by_name("bor_not");
54 let brnz = insts.by_name("brnz");
55 let brz = insts.by_name("brz");
56 let br_icmp = insts.by_name("br_icmp");
57 let br_table = insts.by_name("br_table");
58 let bxor = insts.by_name("bxor");
59 let bxor_imm = insts.by_name("bxor_imm");
60 let bxor_not = insts.by_name("bxor_not");
61 let cls = insts.by_name("cls");
62 let clz = insts.by_name("clz");
63 let ctz = insts.by_name("ctz");
64 let copy = insts.by_name("copy");
65 let fabs = insts.by_name("fabs");
66 let f32const = insts.by_name("f32const");
67 let f64const = insts.by_name("f64const");
68 let fcopysign = insts.by_name("fcopysign");
69 let fcvt_from_sint = insts.by_name("fcvt_from_sint");
70 let fneg = insts.by_name("fneg");
71 let iadd = insts.by_name("iadd");
72 let iadd_cin = insts.by_name("iadd_cin");
73 let iadd_cout = insts.by_name("iadd_cout");
74 let iadd_carry = insts.by_name("iadd_carry");
75 let iadd_ifcin = insts.by_name("iadd_ifcin");
76 let iadd_ifcout = insts.by_name("iadd_ifcout");
77 let iadd_imm = insts.by_name("iadd_imm");
78 let icmp = insts.by_name("icmp");
79 let icmp_imm = insts.by_name("icmp_imm");
80 let iconcat = insts.by_name("iconcat");
81 let iconst = insts.by_name("iconst");
82 let ifcmp = insts.by_name("ifcmp");
83 let ifcmp_imm = insts.by_name("ifcmp_imm");
84 let imul = insts.by_name("imul");
85 let imul_imm = insts.by_name("imul_imm");
86 let ireduce = insts.by_name("ireduce");
87 let irsub_imm = insts.by_name("irsub_imm");
88 let ishl = insts.by_name("ishl");
89 let ishl_imm = insts.by_name("ishl_imm");
90 let isplit = insts.by_name("isplit");
91 let istore8 = insts.by_name("istore8");
92 let istore16 = insts.by_name("istore16");
93 let isub = insts.by_name("isub");
94 let isub_bin = insts.by_name("isub_bin");
95 let isub_bout = insts.by_name("isub_bout");
96 let isub_borrow = insts.by_name("isub_borrow");
97 let isub_ifbin = insts.by_name("isub_ifbin");
98 let isub_ifbout = insts.by_name("isub_ifbout");
99 let jump = insts.by_name("jump");
100 let load = insts.by_name("load");
101 let popcnt = insts.by_name("popcnt");
102 let rotl = insts.by_name("rotl");
103 let rotl_imm = insts.by_name("rotl_imm");
104 let rotr = insts.by_name("rotr");
105 let rotr_imm = insts.by_name("rotr_imm");
106 let sdiv = insts.by_name("sdiv");
107 let sdiv_imm = insts.by_name("sdiv_imm");
108 let select = insts.by_name("select");
109 let sextend = insts.by_name("sextend");
110 let sshr = insts.by_name("sshr");
111 let sshr_imm = insts.by_name("sshr_imm");
112 let srem = insts.by_name("srem");
113 let srem_imm = insts.by_name("srem_imm");
114 let store = insts.by_name("store");
115 let udiv = insts.by_name("udiv");
116 let udiv_imm = insts.by_name("udiv_imm");
117 let uextend = insts.by_name("uextend");
118 let uload8 = insts.by_name("uload8");
119 let uload16 = insts.by_name("uload16");
120 let umulhi = insts.by_name("umulhi");
121 let ushr = insts.by_name("ushr");
122 let ushr_imm = insts.by_name("ushr_imm");
123 let urem = insts.by_name("urem");
124 let urem_imm = insts.by_name("urem_imm");
125 let trapif = insts.by_name("trapif");
126 let trapnz = insts.by_name("trapnz");
127 let trapz = insts.by_name("trapz");
128
129 // Custom expansions for memory objects.
130 expand.custom_legalize(insts.by_name("global_value"), "expand_global_value");
131 expand.custom_legalize(insts.by_name("heap_addr"), "expand_heap_addr");
132 expand.custom_legalize(insts.by_name("table_addr"), "expand_table_addr");
133
134 // Custom expansions for calls.
135 expand.custom_legalize(insts.by_name("call"), "expand_call");
136
137 // Custom expansions that need to change the CFG.
138 // TODO: Add sufficient XForm syntax that we don't need to hand-code these.
139 expand.custom_legalize(trapz, "expand_cond_trap");
140 expand.custom_legalize(trapnz, "expand_cond_trap");
141 expand.custom_legalize(br_table, "expand_br_table");
142 expand.custom_legalize(select, "expand_select");
143 widen.custom_legalize(select, "expand_select"); // small ints
144
145 // Custom expansions for floating point constants.
146 // These expansions require bit-casting or creating constant pool entries.
147 expand.custom_legalize(f32const, "expand_fconst");
148 expand.custom_legalize(f64const, "expand_fconst");
149
150 // Custom expansions for stack memory accesses.
151 expand.custom_legalize(insts.by_name("stack_load"), "expand_stack_load");
152 expand.custom_legalize(insts.by_name("stack_store"), "expand_stack_store");
153
154 // Custom expansions for small stack memory acccess.
155 widen.custom_legalize(insts.by_name("stack_load"), "expand_stack_load");
156 widen.custom_legalize(insts.by_name("stack_store"), "expand_stack_store");
157
158 // List of variables to reuse in patterns.
159 let x = var("x");
160 let y = var("y");
161 let z = var("z");
162 let a = var("a");
163 let a1 = var("a1");
164 let a2 = var("a2");
165 let a3 = var("a3");
166 let a4 = var("a4");
167 let b = var("b");
168 let b1 = var("b1");
169 let b2 = var("b2");
170 let b3 = var("b3");
171 let b4 = var("b4");
172 let b_in = var("b_in");
173 let b_int = var("b_int");
174 let c = var("c");
175 let c1 = var("c1");
176 let c2 = var("c2");
177 let c3 = var("c3");
178 let c4 = var("c4");
179 let c_in = var("c_in");
180 let c_int = var("c_int");
181 let d = var("d");
182 let d1 = var("d1");
183 let d2 = var("d2");
184 let d3 = var("d3");
185 let d4 = var("d4");
186 let e = var("e");
187 let e1 = var("e1");
188 let e2 = var("e2");
189 let e3 = var("e3");
190 let e4 = var("e4");
191 let f = var("f");
192 let f1 = var("f1");
193 let f2 = var("f2");
194 let xl = var("xl");
195 let xh = var("xh");
196 let yl = var("yl");
197 let yh = var("yh");
198 let al = var("al");
199 let ah = var("ah");
200 let cc = var("cc");
201 let block = var("block");
202 let block1 = var("block1");
203 let block2 = var("block2");
204 let ptr = var("ptr");
205 let flags = var("flags");
206 let offset = var("off");
207 let vararg = var("vararg");
208
209 narrow.custom_legalize(load, "narrow_load");
210 narrow.custom_legalize(store, "narrow_store");
211
212 // iconst.i64 can't be legalized in the meta langage (because integer literals can't be
213 // embedded as part of arguments), so use a custom legalization for now.
214 narrow.custom_legalize(iconst, "narrow_iconst");
215
216 for &(ty, ty_half) in &[(I128, I64), (I64, I32)] {
217 let inst = uextend.bind(ty).bind(ty_half);
218 narrow.legalize(
219 def!(a = inst(x)),
220 vec![
221 def!(ah = iconst(Literal::constant(&imm.imm64, 0))),
222 def!(a = iconcat(x, ah)),
223 ],
224 );
225 }
226
227 for &(ty, ty_half, shift) in &[(I128, I64, 63), (I64, I32, 31)] {
228 let inst = sextend.bind(ty).bind(ty_half);
229 narrow.legalize(
230 def!(a = inst(x)),
231 vec![
232 def!(ah = sshr_imm(x, Literal::constant(&imm.imm64, shift))), // splat sign bit to whole number
233 def!(a = iconcat(x, ah)),
234 ],
235 );
236 }
237
238 for &bin_op in &[band, bor, bxor, band_not, bor_not, bxor_not] {
239 narrow.legalize(
240 def!(a = bin_op(x, y)),
241 vec![
242 def!((xl, xh) = isplit(x)),
243 def!((yl, yh) = isplit(y)),
244 def!(al = bin_op(xl, yl)),
245 def!(ah = bin_op(xh, yh)),
246 def!(a = iconcat(al, ah)),
247 ],
248 );
249 }
250
251 narrow.legalize(
252 def!(a = bnot(x)),
253 vec![
254 def!((xl, xh) = isplit(x)),
255 def!(al = bnot(xl)),
256 def!(ah = bnot(xh)),
257 def!(a = iconcat(al, ah)),
258 ],
259 );
260
261 narrow.legalize(
262 def!(a = select(c, x, y)),
263 vec![
264 def!((xl, xh) = isplit(x)),
265 def!((yl, yh) = isplit(y)),
266 def!(al = select(c, xl, yl)),
267 def!(ah = select(c, xh, yh)),
268 def!(a = iconcat(al, ah)),
269 ],
270 );
271
272 narrow.legalize(
273 def!(brz.I128(x, block, vararg)),
274 vec![
275 def!((xl, xh) = isplit(x)),
276 def!(
277 a = icmp_imm(
278 Literal::enumerator_for(&imm.intcc, "eq"),
279 xl,
280 Literal::constant(&imm.imm64, 0)
281 )
282 ),
283 def!(
284 b = icmp_imm(
285 Literal::enumerator_for(&imm.intcc, "eq"),
286 xh,
287 Literal::constant(&imm.imm64, 0)
288 )
289 ),
290 def!(c = band(a, b)),
291 def!(brnz(c, block, vararg)),
292 ],
293 );
294
295 narrow.legalize(
296 def!(brnz.I128(x, block1, vararg)),
297 vec![
298 def!((xl, xh) = isplit(x)),
299 def!(brnz(xl, block1, vararg)),
300 def!(jump(block2, Literal::empty_vararg())),
301 block!(block2),
302 def!(brnz(xh, block1, vararg)),
303 ],
304 );
305
306 narrow.legalize(
307 def!(a = popcnt.I128(x)),
308 vec![
309 def!((xl, xh) = isplit(x)),
310 def!(e1 = popcnt(xl)),
311 def!(e2 = popcnt(xh)),
312 def!(e3 = iadd(e1, e2)),
313 def!(a = uextend(e3)),
314 ],
315 );
316
317 // TODO(ryzokuken): benchmark this and decide if branching is a faster
318 // approach than evaluating boolean expressions.
319
320 narrow.custom_legalize(icmp_imm, "narrow_icmp_imm");
321
322 let intcc_eq = Literal::enumerator_for(&imm.intcc, "eq");
323 let intcc_ne = Literal::enumerator_for(&imm.intcc, "ne");
324 for &(int_ty, int_ty_half) in &[(I64, I32), (I128, I64)] {
325 narrow.legalize(
326 def!(b = icmp.int_ty(intcc_eq, x, y)),
327 vec![
328 def!((xl, xh) = isplit(x)),
329 def!((yl, yh) = isplit(y)),
330 def!(b1 = icmp.int_ty_half(intcc_eq, xl, yl)),
331 def!(b2 = icmp.int_ty_half(intcc_eq, xh, yh)),
332 def!(b = band(b1, b2)),
333 ],
334 );
335
336 narrow.legalize(
337 def!(b = icmp.int_ty(intcc_ne, x, y)),
338 vec![
339 def!((xl, xh) = isplit(x)),
340 def!((yl, yh) = isplit(y)),
341 def!(b1 = icmp.int_ty_half(intcc_ne, xl, yl)),
342 def!(b2 = icmp.int_ty_half(intcc_ne, xh, yh)),
343 def!(b = bor(b1, b2)),
344 ],
345 );
346
347 use IntCC::*;
348 for cc in &[
349 SignedGreaterThan,
350 SignedGreaterThanOrEqual,
351 SignedLessThan,
352 SignedLessThanOrEqual,
353 UnsignedGreaterThan,
354 UnsignedGreaterThanOrEqual,
355 UnsignedLessThan,
356 UnsignedLessThanOrEqual,
357 ] {
358 let intcc_cc = Literal::enumerator_for(&imm.intcc, cc.to_static_str());
359 let cc1 = Literal::enumerator_for(&imm.intcc, cc.without_equal().to_static_str());
360 let cc2 =
361 Literal::enumerator_for(&imm.intcc, cc.inverse().without_equal().to_static_str());
362 let cc3 = Literal::enumerator_for(&imm.intcc, cc.unsigned().to_static_str());
363 narrow.legalize(
364 def!(b = icmp.int_ty(intcc_cc, x, y)),
365 vec![
366 def!((xl, xh) = isplit(x)),
367 def!((yl, yh) = isplit(y)),
368 // X = cc1 || (!cc2 && cc3)
369 def!(b1 = icmp.int_ty_half(cc1, xh, yh)),
370 def!(b2 = icmp.int_ty_half(cc2, xh, yh)),
371 def!(b3 = icmp.int_ty_half(cc3, xl, yl)),
372 def!(c1 = bnot(b2)),
373 def!(c2 = band(c1, b3)),
374 def!(b = bor(b1, c2)),
375 ],
376 );
377 }
378 }
379
380 // TODO(ryzokuken): explore the perf diff w/ x86_umulx and consider have a
381 // separate legalization for x86.
382 for &ty in &[I64, I128] {
383 narrow.legalize(
384 def!(a = imul.ty(x, y)),
385 vec![
386 def!((xl, xh) = isplit(x)),
387 def!((yl, yh) = isplit(y)),
388 def!(a1 = imul(xh, yl)),
389 def!(a2 = imul(xl, yh)),
390 def!(a3 = iadd(a1, a2)),
391 def!(a4 = umulhi(xl, yl)),
392 def!(ah = iadd(a3, a4)),
393 def!(al = imul(xl, yl)),
394 def!(a = iconcat(al, ah)),
395 ],
396 );
397 }
398
399 // Widen instructions with one input operand.
400 for &op in &[bnot, popcnt] {
401 for &int_ty in &[I8, I16] {
402 widen.legalize(
403 def!(a = op.int_ty(b)),
404 vec![
405 def!(x = uextend.I32(b)),
406 def!(z = op.I32(x)),
407 def!(a = ireduce.int_ty(z)),
408 ],
409 );
410 }
411 }
412
413 // Widen instructions with two input operands.
414 let mut widen_two_arg = |signed: bool, op: &Instruction| {
415 for &int_ty in &[I8, I16] {
416 let sign_ext_op = if signed { sextend } else { uextend };
417 widen.legalize(
418 def!(a = op.int_ty(b, c)),
419 vec![
420 def!(x = sign_ext_op.I32(b)),
421 def!(y = sign_ext_op.I32(c)),
422 def!(z = op.I32(x, y)),
423 def!(a = ireduce.int_ty(z)),
424 ],
425 );
426 }
427 };
428
429 for bin_op in &[
430 iadd, isub, imul, udiv, urem, band, bor, bxor, band_not, bor_not, bxor_not,
431 ] {
432 widen_two_arg(false, bin_op);
433 }
434 for bin_op in &[sdiv, srem] {
435 widen_two_arg(true, bin_op);
436 }
437
438 // Widen instructions using immediate operands.
439 let mut widen_imm = |signed: bool, op: &Instruction| {
440 for &int_ty in &[I8, I16] {
441 let sign_ext_op = if signed { sextend } else { uextend };
442 widen.legalize(
443 def!(a = op.int_ty(b, c)),
444 vec![
445 def!(x = sign_ext_op.I32(b)),
446 def!(z = op.I32(x, c)),
447 def!(a = ireduce.int_ty(z)),
448 ],
449 );
450 }
451 };
452
453 for bin_op in &[
454 iadd_imm, imul_imm, udiv_imm, urem_imm, band_imm, bor_imm, bxor_imm, irsub_imm,
455 ] {
456 widen_imm(false, bin_op);
457 }
458 for bin_op in &[sdiv_imm, srem_imm] {
459 widen_imm(true, bin_op);
460 }
461
462 for &(int_ty, num) in &[(I8, 24), (I16, 16)] {
463 let imm = Literal::constant(&imm.imm64, -num);
464
465 widen.legalize(
466 def!(a = clz.int_ty(b)),
467 vec![
468 def!(c = uextend.I32(b)),
469 def!(d = clz.I32(c)),
470 def!(e = iadd_imm(d, imm)),
471 def!(a = ireduce.int_ty(e)),
472 ],
473 );
474
475 widen.legalize(
476 def!(a = cls.int_ty(b)),
477 vec![
478 def!(c = sextend.I32(b)),
479 def!(d = cls.I32(c)),
480 def!(e = iadd_imm(d, imm)),
481 def!(a = ireduce.int_ty(e)),
482 ],
483 );
484 }
485
486 for &(int_ty, num) in &[(I8, 1 << 8), (I16, 1 << 16)] {
487 let num = Literal::constant(&imm.imm64, num);
488 widen.legalize(
489 def!(a = ctz.int_ty(b)),
490 vec![
491 def!(c = uextend.I32(b)),
492 // When `b` is zero, returns the size of x in bits.
493 def!(d = bor_imm(c, num)),
494 def!(e = ctz.I32(d)),
495 def!(a = ireduce.int_ty(e)),
496 ],
497 );
498 }
499
500 // iconst
501 for &int_ty in &[I8, I16] {
502 widen.legalize(
503 def!(a = iconst.int_ty(b)),
504 vec![def!(c = iconst.I32(b)), def!(a = ireduce.int_ty(c))],
505 );
506 }
507
508 for &extend_op in &[uextend, sextend] {
509 // The sign extension operators have two typevars: the result has one and controls the
510 // instruction, then the input has one.
511 let bound = extend_op.bind(I16).bind(I8);
512 widen.legalize(
513 def!(a = bound(b)),
514 vec![def!(c = extend_op.I32(b)), def!(a = ireduce(c))],
515 );
516 }
517
518 widen.legalize(
519 def!(store.I8(flags, a, ptr, offset)),
520 vec![
521 def!(b = uextend.I32(a)),
522 def!(istore8(flags, b, ptr, offset)),
523 ],
524 );
525
526 widen.legalize(
527 def!(store.I16(flags, a, ptr, offset)),
528 vec![
529 def!(b = uextend.I32(a)),
530 def!(istore16(flags, b, ptr, offset)),
531 ],
532 );
533
534 widen.legalize(
535 def!(a = load.I8(flags, ptr, offset)),
536 vec![
537 def!(b = uload8.I32(flags, ptr, offset)),
538 def!(a = ireduce(b)),
539 ],
540 );
541
542 widen.legalize(
543 def!(a = load.I16(flags, ptr, offset)),
544 vec![
545 def!(b = uload16.I32(flags, ptr, offset)),
546 def!(a = ireduce(b)),
547 ],
548 );
549
550 for &int_ty in &[I8, I16] {
551 widen.legalize(
552 def!(br_table.int_ty(x, y, z)),
553 vec![def!(b = uextend.I32(x)), def!(br_table(b, y, z))],
554 );
555 }
556
557 for &int_ty in &[I8, I16] {
558 widen.legalize(
559 def!(a = bint.int_ty(b)),
560 vec![def!(x = bint.I32(b)), def!(a = ireduce.int_ty(x))],
561 );
562 }
563
564 for &int_ty in &[I8, I16] {
565 for &op in &[ishl, ishl_imm, ushr, ushr_imm] {
566 widen.legalize(
567 def!(a = op.int_ty(b, c)),
568 vec![
569 def!(x = uextend.I32(b)),
570 def!(z = op.I32(x, c)),
571 def!(a = ireduce.int_ty(z)),
572 ],
573 );
574 }
575
576 for &op in &[sshr, sshr_imm] {
577 widen.legalize(
578 def!(a = op.int_ty(b, c)),
579 vec![
580 def!(x = sextend.I32(b)),
581 def!(z = op.I32(x, c)),
582 def!(a = ireduce.int_ty(z)),
583 ],
584 );
585 }
586
587 for cc in &["eq", "ne", "ugt", "ult", "uge", "ule"] {
588 let w_cc = Literal::enumerator_for(&imm.intcc, cc);
589 widen.legalize(
590 def!(a = icmp_imm.int_ty(w_cc, b, c)),
591 vec![def!(x = uextend.I32(b)), def!(a = icmp_imm(w_cc, x, c))],
592 );
593 widen.legalize(
594 def!(a = icmp.int_ty(w_cc, b, c)),
595 vec![
596 def!(x = uextend.I32(b)),
597 def!(y = uextend.I32(c)),
598 def!(a = icmp.I32(w_cc, x, y)),
599 ],
600 );
601 }
602
603 for cc in &["sgt", "slt", "sge", "sle"] {
604 let w_cc = Literal::enumerator_for(&imm.intcc, cc);
605 widen.legalize(
606 def!(a = icmp_imm.int_ty(w_cc, b, c)),
607 vec![def!(x = sextend.I32(b)), def!(a = icmp_imm(w_cc, x, c))],
608 );
609
610 widen.legalize(
611 def!(a = icmp.int_ty(w_cc, b, c)),
612 vec![
613 def!(x = sextend.I32(b)),
614 def!(y = sextend.I32(c)),
615 def!(a = icmp(w_cc, x, y)),
616 ],
617 );
618 }
619 }
620
621 for &ty in &[I8, I16] {
622 widen.legalize(
623 def!(brz.ty(x, block, vararg)),
624 vec![def!(a = uextend.I32(x)), def!(brz(a, block, vararg))],
625 );
626
627 widen.legalize(
628 def!(brnz.ty(x, block, vararg)),
629 vec![def!(a = uextend.I32(x)), def!(brnz(a, block, vararg))],
630 );
631 }
632
633 for &(ty_half, ty) in &[(I64, I128), (I32, I64)] {
634 let inst = ireduce.bind(ty_half).bind(ty);
635 expand.legalize(
636 def!(a = inst(x)),
637 vec![def!((b, c) = isplit(x)), def!(a = copy(b))],
638 );
639 }
640
641 // Expand integer operations with carry for RISC architectures that don't have
642 // the flags.
643 let intcc_ult = Literal::enumerator_for(&imm.intcc, "ult");
644 expand.legalize(
645 def!((a, c) = iadd_cout(x, y)),
646 vec![def!(a = iadd(x, y)), def!(c = icmp(intcc_ult, a, x))],
647 );
648
649 let intcc_ugt = Literal::enumerator_for(&imm.intcc, "ugt");
650 expand.legalize(
651 def!((a, b) = isub_bout(x, y)),
652 vec![def!(a = isub(x, y)), def!(b = icmp(intcc_ugt, a, x))],
653 );
654
655 expand.legalize(
656 def!(a = iadd_cin(x, y, c)),
657 vec![
658 def!(a1 = iadd(x, y)),
659 def!(c_int = bint(c)),
660 def!(a = iadd(a1, c_int)),
661 ],
662 );
663
664 expand.legalize(
665 def!(a = isub_bin(x, y, b)),
666 vec![
667 def!(a1 = isub(x, y)),
668 def!(b_int = bint(b)),
669 def!(a = isub(a1, b_int)),
670 ],
671 );
672
673 expand.legalize(
674 def!((a, c) = iadd_carry(x, y, c_in)),
675 vec![
676 def!((a1, c1) = iadd_cout(x, y)),
677 def!(c_int = bint(c_in)),
678 def!((a, c2) = iadd_cout(a1, c_int)),
679 def!(c = bor(c1, c2)),
680 ],
681 );
682
683 expand.legalize(
684 def!((a, b) = isub_borrow(x, y, b_in)),
685 vec![
686 def!((a1, b1) = isub_bout(x, y)),
687 def!(b_int = bint(b_in)),
688 def!((a, b2) = isub_bout(a1, b_int)),
689 def!(b = bor(b1, b2)),
690 ],
691 );
692
693 // Expansion for fcvt_from_sint for smaller integer types.
694 // This uses expand and not widen because the controlling type variable for
695 // this instruction is f32/f64, which is legalized as part of the expand
696 // group.
697 for &dest_ty in &[F32, F64] {
698 for &src_ty in &[I8, I16] {
699 let bound_inst = fcvt_from_sint.bind(dest_ty).bind(src_ty);
700 expand.legalize(
701 def!(a = bound_inst(b)),
702 vec![
703 def!(x = sextend.I32(b)),
704 def!(a = fcvt_from_sint.dest_ty(x)),
705 ],
706 );
707 }
708 }
709
710 // Expansions for immediate operands that are out of range.
711 for &(inst_imm, inst) in &[
712 (iadd_imm, iadd),
713 (imul_imm, imul),
714 (sdiv_imm, sdiv),
715 (udiv_imm, udiv),
716 (srem_imm, srem),
717 (urem_imm, urem),
718 (band_imm, band),
719 (bor_imm, bor),
720 (bxor_imm, bxor),
721 (ifcmp_imm, ifcmp),
722 ] {
723 expand.legalize(
724 def!(a = inst_imm(x, y)),
725 vec![def!(a1 = iconst(y)), def!(a = inst(x, a1))],
726 );
727 }
728
729 expand.legalize(
730 def!(a = irsub_imm(y, x)),
731 vec![def!(a1 = iconst(x)), def!(a = isub(a1, y))],
732 );
733
734 // Rotates and shifts.
735 for &(inst_imm, inst) in &[
736 (rotl_imm, rotl),
737 (rotr_imm, rotr),
738 (ishl_imm, ishl),
739 (sshr_imm, sshr),
740 (ushr_imm, ushr),
741 ] {
742 expand.legalize(
743 def!(a = inst_imm(x, y)),
744 vec![def!(a1 = iconst.I32(y)), def!(a = inst(x, a1))],
745 );
746 }
747
748 expand.legalize(
749 def!(a = icmp_imm(cc, x, y)),
750 vec![def!(a1 = iconst(y)), def!(a = icmp(cc, x, a1))],
751 );
752
753 //# Expansions for *_not variants of bitwise ops.
754 for &(inst_not, inst) in &[(band_not, band), (bor_not, bor), (bxor_not, bxor)] {
755 expand.legalize(
756 def!(a = inst_not(x, y)),
757 vec![def!(a1 = bnot(y)), def!(a = inst(x, a1))],
758 );
759 }
760
761 //# Expand bnot using xor.
762 let minus_one = Literal::constant(&imm.imm64, -1);
763 expand.legalize(
764 def!(a = bnot(x)),
765 vec![def!(y = iconst(minus_one)), def!(a = bxor(x, y))],
766 );
767
768 //# Expand bitrev
769 //# Adapted from Stack Overflow.
770 //# https://stackoverflow.com/questions/746171/most-efficient-algorithm-for-bit-reversal-from-msb-lsb-to-lsb-msb-in-c
771 let imm64_1 = Literal::constant(&imm.imm64, 1);
772 let imm64_2 = Literal::constant(&imm.imm64, 2);
773 let imm64_4 = Literal::constant(&imm.imm64, 4);
774
775 widen.legalize(
776 def!(a = bitrev.I8(x)),
777 vec![
778 def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaa))),
779 def!(a2 = ushr_imm(a1, imm64_1)),
780 def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x55))),
781 def!(a4 = ishl_imm(a3, imm64_1)),
782 def!(b = bor(a2, a4)),
783 def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcc))),
784 def!(b2 = ushr_imm(b1, imm64_2)),
785 def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x33))),
786 def!(b4 = ishl_imm(b3, imm64_2)),
787 def!(c = bor(b2, b4)),
788 def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0))),
789 def!(c2 = ushr_imm(c1, imm64_4)),
790 def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f))),
791 def!(c4 = ishl_imm(c3, imm64_4)),
792 def!(a = bor(c2, c4)),
793 ],
794 );
795
796 let imm64_8 = Literal::constant(&imm.imm64, 8);
797
798 widen.legalize(
799 def!(a = bitrev.I16(x)),
800 vec![
801 def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaaaa))),
802 def!(a2 = ushr_imm(a1, imm64_1)),
803 def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x5555))),
804 def!(a4 = ishl_imm(a3, imm64_1)),
805 def!(b = bor(a2, a4)),
806 def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcccc))),
807 def!(b2 = ushr_imm(b1, imm64_2)),
808 def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x3333))),
809 def!(b4 = ishl_imm(b3, imm64_2)),
810 def!(c = bor(b2, b4)),
811 def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0f0))),
812 def!(c2 = ushr_imm(c1, imm64_4)),
813 def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f0f))),
814 def!(c4 = ishl_imm(c3, imm64_4)),
815 def!(d = bor(c2, c4)),
816 def!(d1 = band_imm(d, Literal::constant(&imm.imm64, 0xff00))),
817 def!(d2 = ushr_imm(d1, imm64_8)),
818 def!(d3 = band_imm(d, Literal::constant(&imm.imm64, 0x00ff))),
819 def!(d4 = ishl_imm(d3, imm64_8)),
820 def!(a = bor(d2, d4)),
821 ],
822 );
823
824 let imm64_16 = Literal::constant(&imm.imm64, 16);
825
826 expand.legalize(
827 def!(a = bitrev.I32(x)),
828 vec![
829 def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaaaa_aaaa))),
830 def!(a2 = ushr_imm(a1, imm64_1)),
831 def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x5555_5555))),
832 def!(a4 = ishl_imm(a3, imm64_1)),
833 def!(b = bor(a2, a4)),
834 def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcccc_cccc))),
835 def!(b2 = ushr_imm(b1, imm64_2)),
836 def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x3333_3333))),
837 def!(b4 = ishl_imm(b3, imm64_2)),
838 def!(c = bor(b2, b4)),
839 def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0f0_f0f0))),
840 def!(c2 = ushr_imm(c1, imm64_4)),
841 def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f0f_0f0f))),
842 def!(c4 = ishl_imm(c3, imm64_4)),
843 def!(d = bor(c2, c4)),
844 def!(d1 = band_imm(d, Literal::constant(&imm.imm64, 0xff00_ff00))),
845 def!(d2 = ushr_imm(d1, imm64_8)),
846 def!(d3 = band_imm(d, Literal::constant(&imm.imm64, 0x00ff_00ff))),
847 def!(d4 = ishl_imm(d3, imm64_8)),
848 def!(e = bor(d2, d4)),
849 def!(e1 = ushr_imm(e, imm64_16)),
850 def!(e2 = ishl_imm(e, imm64_16)),
851 def!(a = bor(e1, e2)),
852 ],
853 );
854
855 #[allow(overflowing_literals)]
856 let imm64_0xaaaaaaaaaaaaaaaa = Literal::constant(&imm.imm64, 0xaaaa_aaaa_aaaa_aaaa);
857 let imm64_0x5555555555555555 = Literal::constant(&imm.imm64, 0x5555_5555_5555_5555);
858 #[allow(overflowing_literals)]
859 let imm64_0xcccccccccccccccc = Literal::constant(&imm.imm64, 0xcccc_cccc_cccc_cccc);
860 let imm64_0x3333333333333333 = Literal::constant(&imm.imm64, 0x3333_3333_3333_3333);
861 #[allow(overflowing_literals)]
862 let imm64_0xf0f0f0f0f0f0f0f0 = Literal::constant(&imm.imm64, 0xf0f0_f0f0_f0f0_f0f0);
863 let imm64_0x0f0f0f0f0f0f0f0f = Literal::constant(&imm.imm64, 0x0f0f_0f0f_0f0f_0f0f);
864 #[allow(overflowing_literals)]
865 let imm64_0xff00ff00ff00ff00 = Literal::constant(&imm.imm64, 0xff00_ff00_ff00_ff00);
866 let imm64_0x00ff00ff00ff00ff = Literal::constant(&imm.imm64, 0x00ff_00ff_00ff_00ff);
867 #[allow(overflowing_literals)]
868 let imm64_0xffff0000ffff0000 = Literal::constant(&imm.imm64, 0xffff_0000_ffff_0000);
869 let imm64_0x0000ffff0000ffff = Literal::constant(&imm.imm64, 0x0000_ffff_0000_ffff);
870 let imm64_32 = Literal::constant(&imm.imm64, 32);
871
872 expand.legalize(
873 def!(a = bitrev.I64(x)),
874 vec![
875 def!(a1 = band_imm(x, imm64_0xaaaaaaaaaaaaaaaa)),
876 def!(a2 = ushr_imm(a1, imm64_1)),
877 def!(a3 = band_imm(x, imm64_0x5555555555555555)),
878 def!(a4 = ishl_imm(a3, imm64_1)),
879 def!(b = bor(a2, a4)),
880 def!(b1 = band_imm(b, imm64_0xcccccccccccccccc)),
881 def!(b2 = ushr_imm(b1, imm64_2)),
882 def!(b3 = band_imm(b, imm64_0x3333333333333333)),
883 def!(b4 = ishl_imm(b3, imm64_2)),
884 def!(c = bor(b2, b4)),
885 def!(c1 = band_imm(c, imm64_0xf0f0f0f0f0f0f0f0)),
886 def!(c2 = ushr_imm(c1, imm64_4)),
887 def!(c3 = band_imm(c, imm64_0x0f0f0f0f0f0f0f0f)),
888 def!(c4 = ishl_imm(c3, imm64_4)),
889 def!(d = bor(c2, c4)),
890 def!(d1 = band_imm(d, imm64_0xff00ff00ff00ff00)),
891 def!(d2 = ushr_imm(d1, imm64_8)),
892 def!(d3 = band_imm(d, imm64_0x00ff00ff00ff00ff)),
893 def!(d4 = ishl_imm(d3, imm64_8)),
894 def!(e = bor(d2, d4)),
895 def!(e1 = band_imm(e, imm64_0xffff0000ffff0000)),
896 def!(e2 = ushr_imm(e1, imm64_16)),
897 def!(e3 = band_imm(e, imm64_0x0000ffff0000ffff)),
898 def!(e4 = ishl_imm(e3, imm64_16)),
899 def!(f = bor(e2, e4)),
900 def!(f1 = ushr_imm(f, imm64_32)),
901 def!(f2 = ishl_imm(f, imm64_32)),
902 def!(a = bor(f1, f2)),
903 ],
904 );
905
906 narrow.legalize(
907 def!(a = bitrev.I128(x)),
908 vec![
909 def!((xl, xh) = isplit(x)),
910 def!(yh = bitrev(xl)),
911 def!(yl = bitrev(xh)),
912 def!(a = iconcat(yl, yh)),
913 ],
914 );
915
916 // Floating-point sign manipulations.
917 for &(ty, const_inst, minus_zero) in &[
918 (F32, f32const, &Literal::bits(&imm.ieee32, 0x8000_0000)),
919 (
920 F64,
921 f64const,
922 &Literal::bits(&imm.ieee64, 0x8000_0000_0000_0000),
923 ),
924 ] {
925 expand.legalize(
926 def!(a = fabs.ty(x)),
927 vec![def!(b = const_inst(minus_zero)), def!(a = band_not(x, b))],
928 );
929
930 expand.legalize(
931 def!(a = fneg.ty(x)),
932 vec![def!(b = const_inst(minus_zero)), def!(a = bxor(x, b))],
933 );
934
935 expand.legalize(
936 def!(a = fcopysign.ty(x, y)),
937 vec![
938 def!(b = const_inst(minus_zero)),
939 def!(a1 = band_not(x, b)),
940 def!(a2 = band(y, b)),
941 def!(a = bor(a1, a2)),
942 ],
943 );
944 }
945
946 expand.custom_legalize(br_icmp, "expand_br_icmp");
947
948 let mut groups = TransformGroups::new();
949
950 let narrow_id = narrow.build_and_add_to(&mut groups);
951 let expand_id = expand.build_and_add_to(&mut groups);
952
953 // Expansions using CPU flags.
954 let mut expand_flags = TransformGroupBuilder::new(
955 "expand_flags",
956 r#"
957 Instruction expansions for architectures with flags.
958
959 Expand some instructions using CPU flags, then fall back to the normal
960 expansions. Not all architectures support CPU flags, so these patterns
961 are kept separate.
962 "#,
963 )
964 .chain_with(expand_id);
965
966 let imm64_0 = Literal::constant(&imm.imm64, 0);
967 let intcc_ne = Literal::enumerator_for(&imm.intcc, "ne");
968 let intcc_eq = Literal::enumerator_for(&imm.intcc, "eq");
969
970 expand_flags.legalize(
971 def!(trapnz(x, c)),
972 vec![
973 def!(a = ifcmp_imm(x, imm64_0)),
974 def!(trapif(intcc_ne, a, c)),
975 ],
976 );
977
978 expand_flags.legalize(
979 def!(trapz(x, c)),
980 vec![
981 def!(a = ifcmp_imm(x, imm64_0)),
982 def!(trapif(intcc_eq, a, c)),
983 ],
984 );
985
986 expand_flags.build_and_add_to(&mut groups);
987
988 // Narrow legalizations using CPU flags.
989 let mut narrow_flags = TransformGroupBuilder::new(
990 "narrow_flags",
991 r#"
992 Narrow instructions for architectures with flags.
993
994 Narrow some instructions using CPU flags, then fall back to the normal
995 legalizations. Not all architectures support CPU flags, so these
996 patterns are kept separate.
997 "#,
998 )
999 .chain_with(narrow_id);
1000
1001 narrow_flags.legalize(
1002 def!(a = iadd(x, y)),
1003 vec![
1004 def!((xl, xh) = isplit(x)),
1005 def!((yl, yh) = isplit(y)),
1006 def!((al, c) = iadd_ifcout(xl, yl)),
1007 def!(ah = iadd_ifcin(xh, yh, c)),
1008 def!(a = iconcat(al, ah)),
1009 ],
1010 );
1011
1012 narrow_flags.legalize(
1013 def!(a = isub(x, y)),
1014 vec![
1015 def!((xl, xh) = isplit(x)),
1016 def!((yl, yh) = isplit(y)),
1017 def!((al, b) = isub_ifbout(xl, yl)),
1018 def!(ah = isub_ifbin(xh, yh, b)),
1019 def!(a = iconcat(al, ah)),
1020 ],
1021 );
1022
1023 narrow_flags.build_and_add_to(&mut groups);
1024
1025 // TODO(ryzokuken): figure out a way to legalize iadd_c* to iadd_ifc* (and
1026 // similarly isub_b* to isub_ifb*) on expand_flags so that this isn't required.
1027 // Narrow legalizations for ISAs that don't have CPU flags.
1028 let mut narrow_no_flags = TransformGroupBuilder::new(
1029 "narrow_no_flags",
1030 r#"
1031 Narrow instructions for architectures without flags.
1032
1033 Narrow some instructions avoiding the use of CPU flags, then fall back
1034 to the normal legalizations. Not all architectures support CPU flags,
1035 so these patterns are kept separate.
1036 "#,
1037 )
1038 .chain_with(narrow_id);
1039
1040 narrow_no_flags.legalize(
1041 def!(a = iadd(x, y)),
1042 vec![
1043 def!((xl, xh) = isplit(x)),
1044 def!((yl, yh) = isplit(y)),
1045 def!((al, c) = iadd_cout(xl, yl)),
1046 def!(ah = iadd_cin(xh, yh, c)),
1047 def!(a = iconcat(al, ah)),
1048 ],
1049 );
1050
1051 narrow_no_flags.legalize(
1052 def!(a = isub(x, y)),
1053 vec![
1054 def!((xl, xh) = isplit(x)),
1055 def!((yl, yh) = isplit(y)),
1056 def!((al, b) = isub_bout(xl, yl)),
1057 def!(ah = isub_bin(xh, yh, b)),
1058 def!(a = iconcat(al, ah)),
1059 ],
1060 );
1061
1062 narrow_no_flags.build_and_add_to(&mut groups);
1063
1064 // TODO The order of declarations unfortunately matters to be compatible with the Python code.
1065 // When it's all migrated, we can put this next to the narrow/expand build_and_add_to calls
1066 // above.
1067 widen.build_and_add_to(&mut groups);
1068
1069 groups
1070 }
1071