1 use crate::cdsl::ast::{var, ExprBuilder, Literal};
2 use crate::cdsl::instructions::{Instruction, InstructionGroup};
3 use crate::cdsl::xform::{TransformGroupBuilder, TransformGroups};
4
5 use crate::shared::immediates::Immediates;
6 use crate::shared::types::Float::{F32, F64};
7 use crate::shared::types::Int::{I128, I16, I32, I64, I8};
8
define(insts: &InstructionGroup, imm: &Immediates) -> TransformGroups9 pub(crate) fn define(insts: &InstructionGroup, imm: &Immediates) -> TransformGroups {
10 let mut narrow = TransformGroupBuilder::new(
11 "narrow",
12 r#"
13 Legalize instructions by narrowing.
14
15 The transformations in the 'narrow' group work by expressing
16 instructions in terms of smaller types. Operations on vector types are
17 expressed in terms of vector types with fewer lanes, and integer
18 operations are expressed in terms of smaller integer types.
19 "#,
20 );
21
22 let mut widen = TransformGroupBuilder::new(
23 "widen",
24 r#"
25 Legalize instructions by widening.
26
27 The transformations in the 'widen' group work by expressing
28 instructions in terms of larger types.
29 "#,
30 );
31
32 let mut expand = TransformGroupBuilder::new(
33 "expand",
34 r#"
35 Legalize instructions by expansion.
36
37 Rewrite instructions in terms of other instructions, generally
38 operating on the same types as the original instructions.
39 "#,
40 );
41
42 // List of instructions.
43 let band = insts.by_name("band");
44 let band_imm = insts.by_name("band_imm");
45 let band_not = insts.by_name("band_not");
46 let bint = insts.by_name("bint");
47 let bitrev = insts.by_name("bitrev");
48 let bnot = insts.by_name("bnot");
49 let bor = insts.by_name("bor");
50 let bor_imm = insts.by_name("bor_imm");
51 let bor_not = insts.by_name("bor_not");
52 let brnz = insts.by_name("brnz");
53 let brz = insts.by_name("brz");
54 let br_icmp = insts.by_name("br_icmp");
55 let br_table = insts.by_name("br_table");
56 let bxor = insts.by_name("bxor");
57 let bxor_imm = insts.by_name("bxor_imm");
58 let bxor_not = insts.by_name("bxor_not");
59 let cls = insts.by_name("cls");
60 let clz = insts.by_name("clz");
61 let ctz = insts.by_name("ctz");
62 let fabs = insts.by_name("fabs");
63 let f32const = insts.by_name("f32const");
64 let f64const = insts.by_name("f64const");
65 let fcopysign = insts.by_name("fcopysign");
66 let fcvt_from_sint = insts.by_name("fcvt_from_sint");
67 let fneg = insts.by_name("fneg");
68 let iadd = insts.by_name("iadd");
69 let iadd_cin = insts.by_name("iadd_cin");
70 let iadd_cout = insts.by_name("iadd_cout");
71 let iadd_carry = insts.by_name("iadd_carry");
72 let iadd_ifcin = insts.by_name("iadd_ifcin");
73 let iadd_ifcout = insts.by_name("iadd_ifcout");
74 let iadd_imm = insts.by_name("iadd_imm");
75 let icmp = insts.by_name("icmp");
76 let icmp_imm = insts.by_name("icmp_imm");
77 let iconcat = insts.by_name("iconcat");
78 let iconst = insts.by_name("iconst");
79 let ifcmp = insts.by_name("ifcmp");
80 let ifcmp_imm = insts.by_name("ifcmp_imm");
81 let imul = insts.by_name("imul");
82 let imul_imm = insts.by_name("imul_imm");
83 let ireduce = insts.by_name("ireduce");
84 let irsub_imm = insts.by_name("irsub_imm");
85 let ishl = insts.by_name("ishl");
86 let ishl_imm = insts.by_name("ishl_imm");
87 let isplit = insts.by_name("isplit");
88 let istore8 = insts.by_name("istore8");
89 let istore16 = insts.by_name("istore16");
90 let isub = insts.by_name("isub");
91 let isub_bin = insts.by_name("isub_bin");
92 let isub_bout = insts.by_name("isub_bout");
93 let isub_borrow = insts.by_name("isub_borrow");
94 let isub_ifbin = insts.by_name("isub_ifbin");
95 let isub_ifbout = insts.by_name("isub_ifbout");
96 let jump = insts.by_name("jump");
97 let load = insts.by_name("load");
98 let popcnt = insts.by_name("popcnt");
99 let rotl = insts.by_name("rotl");
100 let rotl_imm = insts.by_name("rotl_imm");
101 let rotr = insts.by_name("rotr");
102 let rotr_imm = insts.by_name("rotr_imm");
103 let sdiv = insts.by_name("sdiv");
104 let sdiv_imm = insts.by_name("sdiv_imm");
105 let select = insts.by_name("select");
106 let sextend = insts.by_name("sextend");
107 let sshr = insts.by_name("sshr");
108 let sshr_imm = insts.by_name("sshr_imm");
109 let srem = insts.by_name("srem");
110 let srem_imm = insts.by_name("srem_imm");
111 let store = insts.by_name("store");
112 let udiv = insts.by_name("udiv");
113 let udiv_imm = insts.by_name("udiv_imm");
114 let uextend = insts.by_name("uextend");
115 let uload8 = insts.by_name("uload8");
116 let uload16 = insts.by_name("uload16");
117 let ushr = insts.by_name("ushr");
118 let ushr_imm = insts.by_name("ushr_imm");
119 let urem = insts.by_name("urem");
120 let urem_imm = insts.by_name("urem_imm");
121 let trapif = insts.by_name("trapif");
122 let trapnz = insts.by_name("trapnz");
123 let trapz = insts.by_name("trapz");
124
125 // Custom expansions for memory objects.
126 expand.custom_legalize(insts.by_name("global_value"), "expand_global_value");
127 expand.custom_legalize(insts.by_name("heap_addr"), "expand_heap_addr");
128 expand.custom_legalize(insts.by_name("table_addr"), "expand_table_addr");
129
130 // Custom expansions for calls.
131 expand.custom_legalize(insts.by_name("call"), "expand_call");
132
133 // Custom expansions that need to change the CFG.
134 // TODO: Add sufficient XForm syntax that we don't need to hand-code these.
135 expand.custom_legalize(trapz, "expand_cond_trap");
136 expand.custom_legalize(trapnz, "expand_cond_trap");
137 expand.custom_legalize(br_table, "expand_br_table");
138 expand.custom_legalize(select, "expand_select");
139
140 // Custom expansions for floating point constants.
141 // These expansions require bit-casting or creating constant pool entries.
142 expand.custom_legalize(f32const, "expand_fconst");
143 expand.custom_legalize(f64const, "expand_fconst");
144
145 // Custom expansions for stack memory accesses.
146 expand.custom_legalize(insts.by_name("stack_load"), "expand_stack_load");
147 expand.custom_legalize(insts.by_name("stack_store"), "expand_stack_store");
148
149 // List of variables to reuse in patterns.
150 let x = var("x");
151 let y = var("y");
152 let z = var("z");
153 let a = var("a");
154 let a1 = var("a1");
155 let a2 = var("a2");
156 let a3 = var("a3");
157 let a4 = var("a4");
158 let b = var("b");
159 let b1 = var("b1");
160 let b2 = var("b2");
161 let b3 = var("b3");
162 let b4 = var("b4");
163 let b_in = var("b_in");
164 let b_int = var("b_int");
165 let c = var("c");
166 let c1 = var("c1");
167 let c2 = var("c2");
168 let c3 = var("c3");
169 let c4 = var("c4");
170 let c_in = var("c_in");
171 let c_int = var("c_int");
172 let d = var("d");
173 let d1 = var("d1");
174 let d2 = var("d2");
175 let d3 = var("d3");
176 let d4 = var("d4");
177 let e = var("e");
178 let e1 = var("e1");
179 let e2 = var("e2");
180 let e3 = var("e3");
181 let e4 = var("e4");
182 let f = var("f");
183 let f1 = var("f1");
184 let f2 = var("f2");
185 let xl = var("xl");
186 let xh = var("xh");
187 let yl = var("yl");
188 let yh = var("yh");
189 let al = var("al");
190 let ah = var("ah");
191 let cc = var("cc");
192 let ebb = var("ebb");
193 let ebb1 = var("ebb1");
194 let ebb2 = var("ebb2");
195 let ptr = var("ptr");
196 let flags = var("flags");
197 let offset = var("off");
198 let vararg = var("vararg");
199
200 narrow.custom_legalize(load, "narrow_load");
201 narrow.custom_legalize(store, "narrow_store");
202
203 // iconst.i64 can't be legalized in the meta langage (because integer literals can't be
204 // embedded as part of arguments), so use a custom legalization for now.
205 narrow.custom_legalize(iconst, "narrow_iconst");
206
207 for &bin_op in &[band, bor, bxor, band_not, bor_not, bxor_not] {
208 narrow.legalize(
209 def!(a = bin_op(x, y)),
210 vec![
211 def!((xl, xh) = isplit(x)),
212 def!((yl, yh) = isplit(y)),
213 def!(al = bin_op(xl, yl)),
214 def!(ah = bin_op(xh, yh)),
215 def!(a = iconcat(al, ah)),
216 ],
217 );
218 }
219
220 narrow.legalize(
221 def!(a = bnot(x)),
222 vec![
223 def!((xl, xh) = isplit(x)),
224 def!(al = bnot(xl)),
225 def!(ah = bnot(xh)),
226 def!(a = iconcat(al, ah)),
227 ],
228 );
229
230 narrow.legalize(
231 def!(a = select(c, x, y)),
232 vec![
233 def!((xl, xh) = isplit(x)),
234 def!((yl, yh) = isplit(y)),
235 def!(al = select(c, xl, yl)),
236 def!(ah = select(c, xh, yh)),
237 def!(a = iconcat(al, ah)),
238 ],
239 );
240
241 narrow.legalize(
242 def!(brz.I128(x, ebb, vararg)),
243 vec![
244 def!((xl, xh) = isplit(x)),
245 def!(
246 a = icmp_imm(
247 Literal::enumerator_for(&imm.intcc, "eq"),
248 xl,
249 Literal::constant(&imm.imm64, 0)
250 )
251 ),
252 def!(
253 b = icmp_imm(
254 Literal::enumerator_for(&imm.intcc, "eq"),
255 xh,
256 Literal::constant(&imm.imm64, 0)
257 )
258 ),
259 def!(c = band(a, b)),
260 def!(brz(c, ebb, vararg)),
261 ],
262 );
263
264 narrow.legalize(
265 def!(brnz.I128(x, ebb1, vararg)),
266 vec![
267 def!((xl, xh) = isplit(x)),
268 def!(brnz(xl, ebb1, vararg)),
269 def!(jump(ebb2, Literal::empty_vararg())),
270 ebb!(ebb2),
271 def!(brnz(xh, ebb1, vararg)),
272 ],
273 );
274
275 // Widen instructions with one input operand.
276 for &op in &[bnot, popcnt] {
277 for &int_ty in &[I8, I16] {
278 widen.legalize(
279 def!(a = op.int_ty(b)),
280 vec![
281 def!(x = uextend.I32(b)),
282 def!(z = op.I32(x)),
283 def!(a = ireduce.int_ty(z)),
284 ],
285 );
286 }
287 }
288
289 // Widen instructions with two input operands.
290 let mut widen_two_arg = |signed: bool, op: &Instruction| {
291 for &int_ty in &[I8, I16] {
292 let sign_ext_op = if signed { sextend } else { uextend };
293 widen.legalize(
294 def!(a = op.int_ty(b, c)),
295 vec![
296 def!(x = sign_ext_op.I32(b)),
297 def!(y = sign_ext_op.I32(c)),
298 def!(z = op.I32(x, y)),
299 def!(a = ireduce.int_ty(z)),
300 ],
301 );
302 }
303 };
304
305 for bin_op in &[
306 iadd, isub, imul, udiv, urem, band, bor, bxor, band_not, bor_not, bxor_not,
307 ] {
308 widen_two_arg(false, bin_op);
309 }
310 for bin_op in &[sdiv, srem] {
311 widen_two_arg(true, bin_op);
312 }
313
314 // Widen instructions using immediate operands.
315 let mut widen_imm = |signed: bool, op: &Instruction| {
316 for &int_ty in &[I8, I16] {
317 let sign_ext_op = if signed { sextend } else { uextend };
318 widen.legalize(
319 def!(a = op.int_ty(b, c)),
320 vec![
321 def!(x = sign_ext_op.I32(b)),
322 def!(z = op.I32(x, c)),
323 def!(a = ireduce.int_ty(z)),
324 ],
325 );
326 }
327 };
328
329 for bin_op in &[
330 iadd_imm, imul_imm, udiv_imm, urem_imm, band_imm, bor_imm, bxor_imm, irsub_imm,
331 ] {
332 widen_imm(false, bin_op);
333 }
334 for bin_op in &[sdiv_imm, srem_imm] {
335 widen_imm(true, bin_op);
336 }
337
338 for &(int_ty, num) in &[(I8, 24), (I16, 16)] {
339 let imm = Literal::constant(&imm.imm64, -num);
340
341 widen.legalize(
342 def!(a = clz.int_ty(b)),
343 vec![
344 def!(c = uextend.I32(b)),
345 def!(d = clz.I32(c)),
346 def!(e = iadd_imm(d, imm)),
347 def!(a = ireduce.int_ty(e)),
348 ],
349 );
350
351 widen.legalize(
352 def!(a = cls.int_ty(b)),
353 vec![
354 def!(c = sextend.I32(b)),
355 def!(d = cls.I32(c)),
356 def!(e = iadd_imm(d, imm)),
357 def!(a = ireduce.int_ty(e)),
358 ],
359 );
360 }
361
362 for &(int_ty, num) in &[(I8, 1 << 8), (I16, 1 << 16)] {
363 let num = Literal::constant(&imm.imm64, num);
364 widen.legalize(
365 def!(a = ctz.int_ty(b)),
366 vec![
367 def!(c = uextend.I32(b)),
368 // When `b` is zero, returns the size of x in bits.
369 def!(d = bor_imm(c, num)),
370 def!(e = ctz.I32(d)),
371 def!(a = ireduce.int_ty(e)),
372 ],
373 );
374 }
375
376 // iconst
377 for &int_ty in &[I8, I16] {
378 widen.legalize(
379 def!(a = iconst.int_ty(b)),
380 vec![def!(c = iconst.I32(b)), def!(a = ireduce.int_ty(c))],
381 );
382 }
383
384 for &extend_op in &[uextend, sextend] {
385 // The sign extension operators have two typevars: the result has one and controls the
386 // instruction, then the input has one.
387 let bound = extend_op.bind(I16).bind(I8);
388 widen.legalize(
389 def!(a = bound(b)),
390 vec![def!(c = extend_op.I32(b)), def!(a = ireduce(c))],
391 );
392 }
393
394 widen.legalize(
395 def!(store.I8(flags, a, ptr, offset)),
396 vec![
397 def!(b = uextend.I32(a)),
398 def!(istore8(flags, b, ptr, offset)),
399 ],
400 );
401
402 widen.legalize(
403 def!(store.I16(flags, a, ptr, offset)),
404 vec![
405 def!(b = uextend.I32(a)),
406 def!(istore16(flags, b, ptr, offset)),
407 ],
408 );
409
410 widen.legalize(
411 def!(a = load.I8(flags, ptr, offset)),
412 vec![
413 def!(b = uload8.I32(flags, ptr, offset)),
414 def!(a = ireduce(b)),
415 ],
416 );
417
418 widen.legalize(
419 def!(a = load.I16(flags, ptr, offset)),
420 vec![
421 def!(b = uload16.I32(flags, ptr, offset)),
422 def!(a = ireduce(b)),
423 ],
424 );
425
426 for &int_ty in &[I8, I16] {
427 widen.legalize(
428 def!(br_table.int_ty(x, y, z)),
429 vec![def!(b = uextend.I32(x)), def!(br_table(b, y, z))],
430 );
431 }
432
433 for &int_ty in &[I8, I16] {
434 widen.legalize(
435 def!(a = bint.int_ty(b)),
436 vec![def!(x = bint.I32(b)), def!(a = ireduce.int_ty(x))],
437 );
438 }
439
440 for &int_ty in &[I8, I16] {
441 for &op in &[ishl, ishl_imm, ushr, ushr_imm] {
442 widen.legalize(
443 def!(a = op.int_ty(b, c)),
444 vec![
445 def!(x = uextend.I32(b)),
446 def!(z = op.I32(x, c)),
447 def!(a = ireduce.int_ty(z)),
448 ],
449 );
450 }
451
452 for &op in &[sshr, sshr_imm] {
453 widen.legalize(
454 def!(a = op.int_ty(b, c)),
455 vec![
456 def!(x = sextend.I32(b)),
457 def!(z = op.I32(x, c)),
458 def!(a = ireduce.int_ty(z)),
459 ],
460 );
461 }
462
463 for cc in &["eq", "ne", "ugt", "ult", "uge", "ule"] {
464 let w_cc = Literal::enumerator_for(&imm.intcc, cc);
465 widen.legalize(
466 def!(a = icmp_imm.int_ty(w_cc, b, c)),
467 vec![def!(x = uextend.I32(b)), def!(a = icmp_imm(w_cc, x, c))],
468 );
469 widen.legalize(
470 def!(a = icmp.int_ty(w_cc, b, c)),
471 vec![
472 def!(x = uextend.I32(b)),
473 def!(y = uextend.I32(c)),
474 def!(a = icmp.I32(w_cc, x, y)),
475 ],
476 );
477 }
478
479 for cc in &["sgt", "slt", "sge", "sle"] {
480 let w_cc = Literal::enumerator_for(&imm.intcc, cc);
481 widen.legalize(
482 def!(a = icmp_imm.int_ty(w_cc, b, c)),
483 vec![def!(x = sextend.I32(b)), def!(a = icmp_imm(w_cc, x, c))],
484 );
485
486 widen.legalize(
487 def!(a = icmp.int_ty(w_cc, b, c)),
488 vec![
489 def!(x = sextend.I32(b)),
490 def!(y = sextend.I32(c)),
491 def!(a = icmp(w_cc, x, y)),
492 ],
493 );
494 }
495 }
496
497 // Expand integer operations with carry for RISC architectures that don't have
498 // the flags.
499 let intcc_ult = Literal::enumerator_for(&imm.intcc, "ult");
500 expand.legalize(
501 def!((a, c) = iadd_cout(x, y)),
502 vec![def!(a = iadd(x, y)), def!(c = icmp(intcc_ult, a, x))],
503 );
504
505 let intcc_ugt = Literal::enumerator_for(&imm.intcc, "ugt");
506 expand.legalize(
507 def!((a, b) = isub_bout(x, y)),
508 vec![def!(a = isub(x, y)), def!(b = icmp(intcc_ugt, a, x))],
509 );
510
511 expand.legalize(
512 def!(a = iadd_cin(x, y, c)),
513 vec![
514 def!(a1 = iadd(x, y)),
515 def!(c_int = bint(c)),
516 def!(a = iadd(a1, c_int)),
517 ],
518 );
519
520 expand.legalize(
521 def!(a = isub_bin(x, y, b)),
522 vec![
523 def!(a1 = isub(x, y)),
524 def!(b_int = bint(b)),
525 def!(a = isub(a1, b_int)),
526 ],
527 );
528
529 expand.legalize(
530 def!((a, c) = iadd_carry(x, y, c_in)),
531 vec![
532 def!((a1, c1) = iadd_cout(x, y)),
533 def!(c_int = bint(c_in)),
534 def!((a, c2) = iadd_cout(a1, c_int)),
535 def!(c = bor(c1, c2)),
536 ],
537 );
538
539 expand.legalize(
540 def!((a, b) = isub_borrow(x, y, b_in)),
541 vec![
542 def!((a1, b1) = isub_bout(x, y)),
543 def!(b_int = bint(b_in)),
544 def!((a, b2) = isub_bout(a1, b_int)),
545 def!(b = bor(b1, b2)),
546 ],
547 );
548
549 // Expansion for fcvt_from_sint for smaller integer types.
550 // This uses expand and not widen because the controlling type variable for
551 // this instruction is f32/f64, which is legalized as part of the expand
552 // group.
553 for &dest_ty in &[F32, F64] {
554 for &src_ty in &[I8, I16] {
555 let bound_inst = fcvt_from_sint.bind(dest_ty).bind(src_ty);
556 expand.legalize(
557 def!(a = bound_inst(b)),
558 vec![
559 def!(x = sextend.I32(b)),
560 def!(a = fcvt_from_sint.dest_ty(x)),
561 ],
562 );
563 }
564 }
565
566 // Expansions for immediate operands that are out of range.
567 for &(inst_imm, inst) in &[
568 (iadd_imm, iadd),
569 (imul_imm, imul),
570 (sdiv_imm, sdiv),
571 (udiv_imm, udiv),
572 (srem_imm, srem),
573 (urem_imm, urem),
574 (band_imm, band),
575 (bor_imm, bor),
576 (bxor_imm, bxor),
577 (ifcmp_imm, ifcmp),
578 ] {
579 expand.legalize(
580 def!(a = inst_imm(x, y)),
581 vec![def!(a1 = iconst(y)), def!(a = inst(x, a1))],
582 );
583 }
584
585 expand.legalize(
586 def!(a = irsub_imm(y, x)),
587 vec![def!(a1 = iconst(x)), def!(a = isub(a1, y))],
588 );
589
590 // Rotates and shifts.
591 for &(inst_imm, inst) in &[
592 (rotl_imm, rotl),
593 (rotr_imm, rotr),
594 (ishl_imm, ishl),
595 (sshr_imm, sshr),
596 (ushr_imm, ushr),
597 ] {
598 expand.legalize(
599 def!(a = inst_imm(x, y)),
600 vec![def!(a1 = iconst.I32(y)), def!(a = inst(x, a1))],
601 );
602 }
603
604 expand.legalize(
605 def!(a = icmp_imm(cc, x, y)),
606 vec![def!(a1 = iconst(y)), def!(a = icmp(cc, x, a1))],
607 );
608
609 //# Expansions for *_not variants of bitwise ops.
610 for &(inst_not, inst) in &[(band_not, band), (bor_not, bor), (bxor_not, bxor)] {
611 expand.legalize(
612 def!(a = inst_not(x, y)),
613 vec![def!(a1 = bnot(y)), def!(a = inst(x, a1))],
614 );
615 }
616
617 //# Expand bnot using xor.
618 let minus_one = Literal::constant(&imm.imm64, -1);
619 expand.legalize(
620 def!(a = bnot(x)),
621 vec![def!(y = iconst(minus_one)), def!(a = bxor(x, y))],
622 );
623
624 //# Expand bitrev
625 //# Adapted from Stack Overflow.
626 //# https://stackoverflow.com/questions/746171/most-efficient-algorithm-for-bit-reversal-from-msb-lsb-to-lsb-msb-in-c
627 let imm64_1 = Literal::constant(&imm.imm64, 1);
628 let imm64_2 = Literal::constant(&imm.imm64, 2);
629 let imm64_4 = Literal::constant(&imm.imm64, 4);
630
631 widen.legalize(
632 def!(a = bitrev.I8(x)),
633 vec![
634 def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaa))),
635 def!(a2 = ushr_imm(a1, imm64_1)),
636 def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x55))),
637 def!(a4 = ishl_imm(a3, imm64_1)),
638 def!(b = bor(a2, a4)),
639 def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcc))),
640 def!(b2 = ushr_imm(b1, imm64_2)),
641 def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x33))),
642 def!(b4 = ishl_imm(b3, imm64_2)),
643 def!(c = bor(b2, b4)),
644 def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0))),
645 def!(c2 = ushr_imm(c1, imm64_4)),
646 def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f))),
647 def!(c4 = ishl_imm(c3, imm64_4)),
648 def!(a = bor(c2, c4)),
649 ],
650 );
651
652 let imm64_8 = Literal::constant(&imm.imm64, 8);
653
654 widen.legalize(
655 def!(a = bitrev.I16(x)),
656 vec![
657 def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaaaa))),
658 def!(a2 = ushr_imm(a1, imm64_1)),
659 def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x5555))),
660 def!(a4 = ishl_imm(a3, imm64_1)),
661 def!(b = bor(a2, a4)),
662 def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcccc))),
663 def!(b2 = ushr_imm(b1, imm64_2)),
664 def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x3333))),
665 def!(b4 = ishl_imm(b3, imm64_2)),
666 def!(c = bor(b2, b4)),
667 def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0f0))),
668 def!(c2 = ushr_imm(c1, imm64_4)),
669 def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f0f))),
670 def!(c4 = ishl_imm(c3, imm64_4)),
671 def!(d = bor(c2, c4)),
672 def!(d1 = band_imm(d, Literal::constant(&imm.imm64, 0xff00))),
673 def!(d2 = ushr_imm(d1, imm64_8)),
674 def!(d3 = band_imm(d, Literal::constant(&imm.imm64, 0x00ff))),
675 def!(d4 = ishl_imm(d3, imm64_8)),
676 def!(a = bor(d2, d4)),
677 ],
678 );
679
680 let imm64_16 = Literal::constant(&imm.imm64, 16);
681
682 expand.legalize(
683 def!(a = bitrev.I32(x)),
684 vec![
685 def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaaaaaaaa))),
686 def!(a2 = ushr_imm(a1, imm64_1)),
687 def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x55555555))),
688 def!(a4 = ishl_imm(a3, imm64_1)),
689 def!(b = bor(a2, a4)),
690 def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcccccccc))),
691 def!(b2 = ushr_imm(b1, imm64_2)),
692 def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x33333333))),
693 def!(b4 = ishl_imm(b3, imm64_2)),
694 def!(c = bor(b2, b4)),
695 def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0f0f0f0))),
696 def!(c2 = ushr_imm(c1, imm64_4)),
697 def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f0f0f0f))),
698 def!(c4 = ishl_imm(c3, imm64_4)),
699 def!(d = bor(c2, c4)),
700 def!(d1 = band_imm(d, Literal::constant(&imm.imm64, 0xff00ff00))),
701 def!(d2 = ushr_imm(d1, imm64_8)),
702 def!(d3 = band_imm(d, Literal::constant(&imm.imm64, 0x00ff00ff))),
703 def!(d4 = ishl_imm(d3, imm64_8)),
704 def!(e = bor(d2, d4)),
705 def!(e1 = ushr_imm(e, imm64_16)),
706 def!(e2 = ishl_imm(e, imm64_16)),
707 def!(a = bor(e1, e2)),
708 ],
709 );
710
711 #[allow(overflowing_literals)]
712 let imm64_0xaaaaaaaaaaaaaaaa = Literal::constant(&imm.imm64, 0xaaaaaaaaaaaaaaaa);
713 let imm64_0x5555555555555555 = Literal::constant(&imm.imm64, 0x5555555555555555);
714 #[allow(overflowing_literals)]
715 let imm64_0xcccccccccccccccc = Literal::constant(&imm.imm64, 0xcccccccccccccccc);
716 let imm64_0x3333333333333333 = Literal::constant(&imm.imm64, 0x3333333333333333);
717 #[allow(overflowing_literals)]
718 let imm64_0xf0f0f0f0f0f0f0f0 = Literal::constant(&imm.imm64, 0xf0f0f0f0f0f0f0f0);
719 let imm64_0x0f0f0f0f0f0f0f0f = Literal::constant(&imm.imm64, 0x0f0f0f0f0f0f0f0f);
720 #[allow(overflowing_literals)]
721 let imm64_0xff00ff00ff00ff00 = Literal::constant(&imm.imm64, 0xff00ff00ff00ff00);
722 let imm64_0x00ff00ff00ff00ff = Literal::constant(&imm.imm64, 0x00ff00ff00ff00ff);
723 #[allow(overflowing_literals)]
724 let imm64_0xffff0000ffff0000 = Literal::constant(&imm.imm64, 0xffff0000ffff0000);
725 let imm64_0x0000ffff0000ffff = Literal::constant(&imm.imm64, 0x0000ffff0000ffff);
726 let imm64_32 = Literal::constant(&imm.imm64, 32);
727
728 expand.legalize(
729 def!(a = bitrev.I64(x)),
730 vec![
731 def!(a1 = band_imm(x, imm64_0xaaaaaaaaaaaaaaaa)),
732 def!(a2 = ushr_imm(a1, imm64_1)),
733 def!(a3 = band_imm(x, imm64_0x5555555555555555)),
734 def!(a4 = ishl_imm(a3, imm64_1)),
735 def!(b = bor(a2, a4)),
736 def!(b1 = band_imm(b, imm64_0xcccccccccccccccc)),
737 def!(b2 = ushr_imm(b1, imm64_2)),
738 def!(b3 = band_imm(b, imm64_0x3333333333333333)),
739 def!(b4 = ishl_imm(b3, imm64_2)),
740 def!(c = bor(b2, b4)),
741 def!(c1 = band_imm(c, imm64_0xf0f0f0f0f0f0f0f0)),
742 def!(c2 = ushr_imm(c1, imm64_4)),
743 def!(c3 = band_imm(c, imm64_0x0f0f0f0f0f0f0f0f)),
744 def!(c4 = ishl_imm(c3, imm64_4)),
745 def!(d = bor(c2, c4)),
746 def!(d1 = band_imm(d, imm64_0xff00ff00ff00ff00)),
747 def!(d2 = ushr_imm(d1, imm64_8)),
748 def!(d3 = band_imm(d, imm64_0x00ff00ff00ff00ff)),
749 def!(d4 = ishl_imm(d3, imm64_8)),
750 def!(e = bor(d2, d4)),
751 def!(e1 = band_imm(e, imm64_0xffff0000ffff0000)),
752 def!(e2 = ushr_imm(e1, imm64_16)),
753 def!(e3 = band_imm(e, imm64_0x0000ffff0000ffff)),
754 def!(e4 = ishl_imm(e3, imm64_16)),
755 def!(f = bor(e2, e4)),
756 def!(f1 = ushr_imm(f, imm64_32)),
757 def!(f2 = ishl_imm(f, imm64_32)),
758 def!(a = bor(f1, f2)),
759 ],
760 );
761
762 // Floating-point sign manipulations.
763 for &(ty, const_inst, minus_zero) in &[
764 (F32, f32const, &Literal::bits(&imm.ieee32, 0x80000000)),
765 (
766 F64,
767 f64const,
768 &Literal::bits(&imm.ieee64, 0x8000000000000000),
769 ),
770 ] {
771 expand.legalize(
772 def!(a = fabs.ty(x)),
773 vec![def!(b = const_inst(minus_zero)), def!(a = band_not(x, b))],
774 );
775
776 expand.legalize(
777 def!(a = fneg.ty(x)),
778 vec![def!(b = const_inst(minus_zero)), def!(a = bxor(x, b))],
779 );
780
781 expand.legalize(
782 def!(a = fcopysign.ty(x, y)),
783 vec![
784 def!(b = const_inst(minus_zero)),
785 def!(a1 = band_not(x, b)),
786 def!(a2 = band(y, b)),
787 def!(a = bor(a1, a2)),
788 ],
789 );
790 }
791
792 expand.custom_legalize(br_icmp, "expand_br_icmp");
793
794 let mut groups = TransformGroups::new();
795
796 let narrow_id = narrow.build_and_add_to(&mut groups);
797 let expand_id = expand.build_and_add_to(&mut groups);
798
799 // Expansions using CPU flags.
800 let mut expand_flags = TransformGroupBuilder::new(
801 "expand_flags",
802 r#"
803 Instruction expansions for architectures with flags.
804
805 Expand some instructions using CPU flags, then fall back to the normal
806 expansions. Not all architectures support CPU flags, so these patterns
807 are kept separate.
808 "#,
809 )
810 .chain_with(expand_id);
811
812 let imm64_0 = Literal::constant(&imm.imm64, 0);
813 let intcc_ne = Literal::enumerator_for(&imm.intcc, "ne");
814 let intcc_eq = Literal::enumerator_for(&imm.intcc, "eq");
815
816 expand_flags.legalize(
817 def!(trapnz(x, c)),
818 vec![
819 def!(a = ifcmp_imm(x, imm64_0)),
820 def!(trapif(intcc_ne, a, c)),
821 ],
822 );
823
824 expand_flags.legalize(
825 def!(trapz(x, c)),
826 vec![
827 def!(a = ifcmp_imm(x, imm64_0)),
828 def!(trapif(intcc_eq, a, c)),
829 ],
830 );
831
832 expand_flags.build_and_add_to(&mut groups);
833
834 // Narrow legalizations using CPU flags.
835 let mut narrow_flags = TransformGroupBuilder::new(
836 "narrow_flags",
837 r#"
838 Narrow instructions for architectures with flags.
839
840 Narrow some instructions using CPU flags, then fall back to the normal
841 legalizations. Not all architectures support CPU flags, so these
842 patterns are kept separate.
843 "#,
844 )
845 .chain_with(narrow_id);
846
847 narrow_flags.legalize(
848 def!(a = iadd(x, y)),
849 vec![
850 def!((xl, xh) = isplit(x)),
851 def!((yl, yh) = isplit(y)),
852 def!((al, c) = iadd_ifcout(xl, yl)),
853 def!(ah = iadd_ifcin(xh, yh, c)),
854 def!(a = iconcat(al, ah)),
855 ],
856 );
857
858 narrow_flags.legalize(
859 def!(a = isub(x, y)),
860 vec![
861 def!((xl, xh) = isplit(x)),
862 def!((yl, yh) = isplit(y)),
863 def!((al, b) = isub_ifbout(xl, yl)),
864 def!(ah = isub_ifbin(xh, yh, b)),
865 def!(a = iconcat(al, ah)),
866 ],
867 );
868
869 narrow_flags.build_and_add_to(&mut groups);
870
871 // TODO(ryzokuken): figure out a way to legalize iadd_c* to iadd_ifc* (and
872 // similarly isub_b* to isub_ifb*) on expand_flags so that this isn't required.
873 // Narrow legalizations for ISAs that don't have CPU flags.
874 let mut narrow_no_flags = TransformGroupBuilder::new(
875 "narrow_no_flags",
876 r#"
877 Narrow instructions for architectures without flags.
878
879 Narrow some instructions avoiding the use of CPU flags, then fall back
880 to the normal legalizations. Not all architectures support CPU flags,
881 so these patterns are kept separate.
882 "#,
883 )
884 .chain_with(narrow_id);
885
886 narrow_no_flags.legalize(
887 def!(a = iadd(x, y)),
888 vec![
889 def!((xl, xh) = isplit(x)),
890 def!((yl, yh) = isplit(y)),
891 def!((al, c) = iadd_cout(xl, yl)),
892 def!(ah = iadd_cin(xh, yh, c)),
893 def!(a = iconcat(al, ah)),
894 ],
895 );
896
897 narrow_no_flags.legalize(
898 def!(a = isub(x, y)),
899 vec![
900 def!((xl, xh) = isplit(x)),
901 def!((yl, yh) = isplit(y)),
902 def!((al, b) = isub_bout(xl, yl)),
903 def!(ah = isub_bin(xh, yh, b)),
904 def!(a = iconcat(al, ah)),
905 ],
906 );
907
908 narrow_no_flags.build_and_add_to(&mut groups);
909
910 // TODO The order of declarations unfortunately matters to be compatible with the Python code.
911 // When it's all migrated, we can put this next to the narrow/expand build_and_add_to calls
912 // above.
913 widen.build_and_add_to(&mut groups);
914
915 groups
916 }
917