1 use crate::cdsl::ast::{var, ExprBuilder, Literal};
2 use crate::cdsl::instructions::{Bindable, Instruction, InstructionGroup};
3 use crate::cdsl::xform::{TransformGroupBuilder, TransformGroups};
4
5 use crate::shared::immediates::Immediates;
6 use crate::shared::types::Float::{F32, F64};
7 use crate::shared::types::Int::{I128, I16, I32, I64, I8};
8 use cranelift_codegen_shared::condcodes::{CondCode, IntCC};
9
10 #[allow(clippy::many_single_char_names, clippy::cognitive_complexity)]
define(insts: &InstructionGroup, imm: &Immediates) -> TransformGroups11 pub(crate) fn define(insts: &InstructionGroup, imm: &Immediates) -> TransformGroups {
12 let mut narrow = TransformGroupBuilder::new(
13 "narrow",
14 r#"
15 Legalize instructions by narrowing.
16
17 The transformations in the 'narrow' group work by expressing
18 instructions in terms of smaller types. Operations on vector types are
19 expressed in terms of vector types with fewer lanes, and integer
20 operations are expressed in terms of smaller integer types.
21 "#,
22 );
23
24 let mut widen = TransformGroupBuilder::new(
25 "widen",
26 r#"
27 Legalize instructions by widening.
28
29 The transformations in the 'widen' group work by expressing
30 instructions in terms of larger types.
31 "#,
32 );
33
34 let mut expand = TransformGroupBuilder::new(
35 "expand",
36 r#"
37 Legalize instructions by expansion.
38
39 Rewrite instructions in terms of other instructions, generally
40 operating on the same types as the original instructions.
41 "#,
42 );
43
44 // List of instructions.
45 let band = insts.by_name("band");
46 let band_imm = insts.by_name("band_imm");
47 let band_not = insts.by_name("band_not");
48 let bint = insts.by_name("bint");
49 let bitrev = insts.by_name("bitrev");
50 let bnot = insts.by_name("bnot");
51 let bor = insts.by_name("bor");
52 let bor_imm = insts.by_name("bor_imm");
53 let bor_not = insts.by_name("bor_not");
54 let brnz = insts.by_name("brnz");
55 let brz = insts.by_name("brz");
56 let br_icmp = insts.by_name("br_icmp");
57 let br_table = insts.by_name("br_table");
58 let bxor = insts.by_name("bxor");
59 let bxor_imm = insts.by_name("bxor_imm");
60 let bxor_not = insts.by_name("bxor_not");
61 let cls = insts.by_name("cls");
62 let clz = insts.by_name("clz");
63 let ctz = insts.by_name("ctz");
64 let fabs = insts.by_name("fabs");
65 let f32const = insts.by_name("f32const");
66 let f64const = insts.by_name("f64const");
67 let fcopysign = insts.by_name("fcopysign");
68 let fcvt_from_sint = insts.by_name("fcvt_from_sint");
69 let fneg = insts.by_name("fneg");
70 let iadd = insts.by_name("iadd");
71 let iadd_cin = insts.by_name("iadd_cin");
72 let iadd_cout = insts.by_name("iadd_cout");
73 let iadd_carry = insts.by_name("iadd_carry");
74 let iadd_ifcin = insts.by_name("iadd_ifcin");
75 let iadd_ifcout = insts.by_name("iadd_ifcout");
76 let iadd_imm = insts.by_name("iadd_imm");
77 let icmp = insts.by_name("icmp");
78 let icmp_imm = insts.by_name("icmp_imm");
79 let iconcat = insts.by_name("iconcat");
80 let iconst = insts.by_name("iconst");
81 let ifcmp = insts.by_name("ifcmp");
82 let ifcmp_imm = insts.by_name("ifcmp_imm");
83 let imul = insts.by_name("imul");
84 let imul_imm = insts.by_name("imul_imm");
85 let ireduce = insts.by_name("ireduce");
86 let irsub_imm = insts.by_name("irsub_imm");
87 let ishl = insts.by_name("ishl");
88 let ishl_imm = insts.by_name("ishl_imm");
89 let isplit = insts.by_name("isplit");
90 let istore8 = insts.by_name("istore8");
91 let istore16 = insts.by_name("istore16");
92 let isub = insts.by_name("isub");
93 let isub_bin = insts.by_name("isub_bin");
94 let isub_bout = insts.by_name("isub_bout");
95 let isub_borrow = insts.by_name("isub_borrow");
96 let isub_ifbin = insts.by_name("isub_ifbin");
97 let isub_ifbout = insts.by_name("isub_ifbout");
98 let jump = insts.by_name("jump");
99 let load = insts.by_name("load");
100 let popcnt = insts.by_name("popcnt");
101 let rotl = insts.by_name("rotl");
102 let rotl_imm = insts.by_name("rotl_imm");
103 let rotr = insts.by_name("rotr");
104 let rotr_imm = insts.by_name("rotr_imm");
105 let sdiv = insts.by_name("sdiv");
106 let sdiv_imm = insts.by_name("sdiv_imm");
107 let select = insts.by_name("select");
108 let sextend = insts.by_name("sextend");
109 let sshr = insts.by_name("sshr");
110 let sshr_imm = insts.by_name("sshr_imm");
111 let srem = insts.by_name("srem");
112 let srem_imm = insts.by_name("srem_imm");
113 let store = insts.by_name("store");
114 let udiv = insts.by_name("udiv");
115 let udiv_imm = insts.by_name("udiv_imm");
116 let uextend = insts.by_name("uextend");
117 let uload8 = insts.by_name("uload8");
118 let uload16 = insts.by_name("uload16");
119 let umulhi = insts.by_name("umulhi");
120 let ushr = insts.by_name("ushr");
121 let ushr_imm = insts.by_name("ushr_imm");
122 let urem = insts.by_name("urem");
123 let urem_imm = insts.by_name("urem_imm");
124 let trapif = insts.by_name("trapif");
125 let trapnz = insts.by_name("trapnz");
126 let trapz = insts.by_name("trapz");
127
128 // Custom expansions for memory objects.
129 expand.custom_legalize(insts.by_name("global_value"), "expand_global_value");
130 expand.custom_legalize(insts.by_name("heap_addr"), "expand_heap_addr");
131 expand.custom_legalize(insts.by_name("table_addr"), "expand_table_addr");
132
133 // Custom expansions for calls.
134 expand.custom_legalize(insts.by_name("call"), "expand_call");
135
136 // Custom expansions that need to change the CFG.
137 // TODO: Add sufficient XForm syntax that we don't need to hand-code these.
138 expand.custom_legalize(trapz, "expand_cond_trap");
139 expand.custom_legalize(trapnz, "expand_cond_trap");
140 expand.custom_legalize(br_table, "expand_br_table");
141 expand.custom_legalize(select, "expand_select");
142 widen.custom_legalize(select, "expand_select"); // small ints
143
144 // Custom expansions for floating point constants.
145 // These expansions require bit-casting or creating constant pool entries.
146 expand.custom_legalize(f32const, "expand_fconst");
147 expand.custom_legalize(f64const, "expand_fconst");
148
149 // Custom expansions for stack memory accesses.
150 expand.custom_legalize(insts.by_name("stack_load"), "expand_stack_load");
151 expand.custom_legalize(insts.by_name("stack_store"), "expand_stack_store");
152
153 // Custom expansions for small stack memory acccess.
154 widen.custom_legalize(insts.by_name("stack_load"), "expand_stack_load");
155 widen.custom_legalize(insts.by_name("stack_store"), "expand_stack_store");
156
157 // List of variables to reuse in patterns.
158 let x = var("x");
159 let y = var("y");
160 let z = var("z");
161 let a = var("a");
162 let a1 = var("a1");
163 let a2 = var("a2");
164 let a3 = var("a3");
165 let a4 = var("a4");
166 let b = var("b");
167 let b1 = var("b1");
168 let b2 = var("b2");
169 let b3 = var("b3");
170 let b4 = var("b4");
171 let b_in = var("b_in");
172 let b_int = var("b_int");
173 let c = var("c");
174 let c1 = var("c1");
175 let c2 = var("c2");
176 let c3 = var("c3");
177 let c4 = var("c4");
178 let c_in = var("c_in");
179 let c_int = var("c_int");
180 let d = var("d");
181 let d1 = var("d1");
182 let d2 = var("d2");
183 let d3 = var("d3");
184 let d4 = var("d4");
185 let e = var("e");
186 let e1 = var("e1");
187 let e2 = var("e2");
188 let e3 = var("e3");
189 let e4 = var("e4");
190 let f = var("f");
191 let f1 = var("f1");
192 let f2 = var("f2");
193 let xl = var("xl");
194 let xh = var("xh");
195 let yl = var("yl");
196 let yh = var("yh");
197 let al = var("al");
198 let ah = var("ah");
199 let cc = var("cc");
200 let block = var("block");
201 let block1 = var("block1");
202 let block2 = var("block2");
203 let ptr = var("ptr");
204 let flags = var("flags");
205 let offset = var("off");
206 let vararg = var("vararg");
207
208 narrow.custom_legalize(load, "narrow_load");
209 narrow.custom_legalize(store, "narrow_store");
210
211 // iconst.i64 can't be legalized in the meta langage (because integer literals can't be
212 // embedded as part of arguments), so use a custom legalization for now.
213 narrow.custom_legalize(iconst, "narrow_iconst");
214
215 {
216 let inst = uextend.bind(I128).bind(I64);
217 narrow.legalize(
218 def!(a = inst(x)),
219 vec![
220 def!(ah = iconst(Literal::constant(&imm.imm64, 0))),
221 def!(a = iconcat(x, ah)),
222 ],
223 );
224 }
225
226 {
227 let inst = sextend.bind(I128).bind(I64);
228 narrow.legalize(
229 def!(a = inst(x)),
230 vec![
231 def!(ah = sshr_imm(x, Literal::constant(&imm.imm64, 63))), // splat sign bit to whole number
232 def!(a = iconcat(x, ah)),
233 ],
234 );
235 }
236
237 for &bin_op in &[band, bor, bxor, band_not, bor_not, bxor_not] {
238 narrow.legalize(
239 def!(a = bin_op(x, y)),
240 vec![
241 def!((xl, xh) = isplit(x)),
242 def!((yl, yh) = isplit(y)),
243 def!(al = bin_op(xl, yl)),
244 def!(ah = bin_op(xh, yh)),
245 def!(a = iconcat(al, ah)),
246 ],
247 );
248 }
249
250 narrow.legalize(
251 def!(a = bnot(x)),
252 vec![
253 def!((xl, xh) = isplit(x)),
254 def!(al = bnot(xl)),
255 def!(ah = bnot(xh)),
256 def!(a = iconcat(al, ah)),
257 ],
258 );
259
260 narrow.legalize(
261 def!(a = select(c, x, y)),
262 vec![
263 def!((xl, xh) = isplit(x)),
264 def!((yl, yh) = isplit(y)),
265 def!(al = select(c, xl, yl)),
266 def!(ah = select(c, xh, yh)),
267 def!(a = iconcat(al, ah)),
268 ],
269 );
270
271 narrow.legalize(
272 def!(brz.I128(x, block, vararg)),
273 vec![
274 def!((xl, xh) = isplit(x)),
275 def!(
276 a = icmp_imm(
277 Literal::enumerator_for(&imm.intcc, "eq"),
278 xl,
279 Literal::constant(&imm.imm64, 0)
280 )
281 ),
282 def!(
283 b = icmp_imm(
284 Literal::enumerator_for(&imm.intcc, "eq"),
285 xh,
286 Literal::constant(&imm.imm64, 0)
287 )
288 ),
289 def!(c = band(a, b)),
290 def!(brnz(c, block, vararg)),
291 ],
292 );
293
294 narrow.legalize(
295 def!(brnz.I128(x, block1, vararg)),
296 vec![
297 def!((xl, xh) = isplit(x)),
298 def!(brnz(xl, block1, vararg)),
299 def!(jump(block2, Literal::empty_vararg())),
300 block!(block2),
301 def!(brnz(xh, block1, vararg)),
302 ],
303 );
304
305 narrow.legalize(
306 def!(a = popcnt.I128(x)),
307 vec![
308 def!((xl, xh) = isplit(x)),
309 def!(e1 = popcnt(xl)),
310 def!(e2 = popcnt(xh)),
311 def!(e3 = iadd(e1, e2)),
312 def!(a = uextend(e3)),
313 ],
314 );
315
316 // TODO(ryzokuken): benchmark this and decide if branching is a faster
317 // approach than evaluating boolean expressions.
318
319 narrow.custom_legalize(icmp_imm, "narrow_icmp_imm");
320
321 let intcc_eq = Literal::enumerator_for(&imm.intcc, "eq");
322 let intcc_ne = Literal::enumerator_for(&imm.intcc, "ne");
323 for &(int_ty, int_ty_half) in &[(I64, I32), (I128, I64)] {
324 narrow.legalize(
325 def!(b = icmp.int_ty(intcc_eq, x, y)),
326 vec![
327 def!((xl, xh) = isplit(x)),
328 def!((yl, yh) = isplit(y)),
329 def!(b1 = icmp.int_ty_half(intcc_eq, xl, yl)),
330 def!(b2 = icmp.int_ty_half(intcc_eq, xh, yh)),
331 def!(b = band(b1, b2)),
332 ],
333 );
334
335 narrow.legalize(
336 def!(b = icmp.int_ty(intcc_ne, x, y)),
337 vec![
338 def!((xl, xh) = isplit(x)),
339 def!((yl, yh) = isplit(y)),
340 def!(b1 = icmp.int_ty_half(intcc_ne, xl, yl)),
341 def!(b2 = icmp.int_ty_half(intcc_ne, xh, yh)),
342 def!(b = bor(b1, b2)),
343 ],
344 );
345
346 use IntCC::*;
347 for cc in &[
348 SignedGreaterThan,
349 SignedGreaterThanOrEqual,
350 SignedLessThan,
351 SignedLessThanOrEqual,
352 UnsignedGreaterThan,
353 UnsignedGreaterThanOrEqual,
354 UnsignedLessThan,
355 UnsignedLessThanOrEqual,
356 ] {
357 let intcc_cc = Literal::enumerator_for(&imm.intcc, cc.to_static_str());
358 let cc1 = Literal::enumerator_for(&imm.intcc, cc.without_equal().to_static_str());
359 let cc2 =
360 Literal::enumerator_for(&imm.intcc, cc.inverse().without_equal().to_static_str());
361 let cc3 = Literal::enumerator_for(&imm.intcc, cc.unsigned().to_static_str());
362 narrow.legalize(
363 def!(b = icmp.int_ty(intcc_cc, x, y)),
364 vec![
365 def!((xl, xh) = isplit(x)),
366 def!((yl, yh) = isplit(y)),
367 // X = cc1 || (!cc2 && cc3)
368 def!(b1 = icmp.int_ty_half(cc1, xh, yh)),
369 def!(b2 = icmp.int_ty_half(cc2, xh, yh)),
370 def!(b3 = icmp.int_ty_half(cc3, xl, yl)),
371 def!(c1 = bnot(b2)),
372 def!(c2 = band(c1, b3)),
373 def!(b = bor(b1, c2)),
374 ],
375 );
376 }
377 }
378
379 // TODO(ryzokuken): explore the perf diff w/ x86_umulx and consider have a
380 // separate legalization for x86.
381 for &ty in &[I64, I128] {
382 narrow.legalize(
383 def!(a = imul.ty(x, y)),
384 vec![
385 def!((xl, xh) = isplit(x)),
386 def!((yl, yh) = isplit(y)),
387 def!(a1 = imul(xh, yl)),
388 def!(a2 = imul(xl, yh)),
389 def!(a3 = iadd(a1, a2)),
390 def!(a4 = umulhi(xl, yl)),
391 def!(ah = iadd(a3, a4)),
392 def!(al = imul(xl, yl)),
393 def!(a = iconcat(al, ah)),
394 ],
395 );
396 }
397
398 // Widen instructions with one input operand.
399 for &op in &[bnot, popcnt] {
400 for &int_ty in &[I8, I16] {
401 widen.legalize(
402 def!(a = op.int_ty(b)),
403 vec![
404 def!(x = uextend.I32(b)),
405 def!(z = op.I32(x)),
406 def!(a = ireduce.int_ty(z)),
407 ],
408 );
409 }
410 }
411
412 // Widen instructions with two input operands.
413 let mut widen_two_arg = |signed: bool, op: &Instruction| {
414 for &int_ty in &[I8, I16] {
415 let sign_ext_op = if signed { sextend } else { uextend };
416 widen.legalize(
417 def!(a = op.int_ty(b, c)),
418 vec![
419 def!(x = sign_ext_op.I32(b)),
420 def!(y = sign_ext_op.I32(c)),
421 def!(z = op.I32(x, y)),
422 def!(a = ireduce.int_ty(z)),
423 ],
424 );
425 }
426 };
427
428 for bin_op in &[
429 iadd, isub, imul, udiv, urem, band, bor, bxor, band_not, bor_not, bxor_not,
430 ] {
431 widen_two_arg(false, bin_op);
432 }
433 for bin_op in &[sdiv, srem] {
434 widen_two_arg(true, bin_op);
435 }
436
437 // Widen instructions using immediate operands.
438 let mut widen_imm = |signed: bool, op: &Instruction| {
439 for &int_ty in &[I8, I16] {
440 let sign_ext_op = if signed { sextend } else { uextend };
441 widen.legalize(
442 def!(a = op.int_ty(b, c)),
443 vec![
444 def!(x = sign_ext_op.I32(b)),
445 def!(z = op.I32(x, c)),
446 def!(a = ireduce.int_ty(z)),
447 ],
448 );
449 }
450 };
451
452 for bin_op in &[
453 iadd_imm, imul_imm, udiv_imm, urem_imm, band_imm, bor_imm, bxor_imm, irsub_imm,
454 ] {
455 widen_imm(false, bin_op);
456 }
457 for bin_op in &[sdiv_imm, srem_imm] {
458 widen_imm(true, bin_op);
459 }
460
461 for &(int_ty, num) in &[(I8, 24), (I16, 16)] {
462 let imm = Literal::constant(&imm.imm64, -num);
463
464 widen.legalize(
465 def!(a = clz.int_ty(b)),
466 vec![
467 def!(c = uextend.I32(b)),
468 def!(d = clz.I32(c)),
469 def!(e = iadd_imm(d, imm)),
470 def!(a = ireduce.int_ty(e)),
471 ],
472 );
473
474 widen.legalize(
475 def!(a = cls.int_ty(b)),
476 vec![
477 def!(c = sextend.I32(b)),
478 def!(d = cls.I32(c)),
479 def!(e = iadd_imm(d, imm)),
480 def!(a = ireduce.int_ty(e)),
481 ],
482 );
483 }
484
485 for &(int_ty, num) in &[(I8, 1 << 8), (I16, 1 << 16)] {
486 let num = Literal::constant(&imm.imm64, num);
487 widen.legalize(
488 def!(a = ctz.int_ty(b)),
489 vec![
490 def!(c = uextend.I32(b)),
491 // When `b` is zero, returns the size of x in bits.
492 def!(d = bor_imm(c, num)),
493 def!(e = ctz.I32(d)),
494 def!(a = ireduce.int_ty(e)),
495 ],
496 );
497 }
498
499 // iconst
500 for &int_ty in &[I8, I16] {
501 widen.legalize(
502 def!(a = iconst.int_ty(b)),
503 vec![def!(c = iconst.I32(b)), def!(a = ireduce.int_ty(c))],
504 );
505 }
506
507 for &extend_op in &[uextend, sextend] {
508 // The sign extension operators have two typevars: the result has one and controls the
509 // instruction, then the input has one.
510 let bound = extend_op.bind(I16).bind(I8);
511 widen.legalize(
512 def!(a = bound(b)),
513 vec![def!(c = extend_op.I32(b)), def!(a = ireduce(c))],
514 );
515 }
516
517 widen.legalize(
518 def!(store.I8(flags, a, ptr, offset)),
519 vec![
520 def!(b = uextend.I32(a)),
521 def!(istore8(flags, b, ptr, offset)),
522 ],
523 );
524
525 widen.legalize(
526 def!(store.I16(flags, a, ptr, offset)),
527 vec![
528 def!(b = uextend.I32(a)),
529 def!(istore16(flags, b, ptr, offset)),
530 ],
531 );
532
533 widen.legalize(
534 def!(a = load.I8(flags, ptr, offset)),
535 vec![
536 def!(b = uload8.I32(flags, ptr, offset)),
537 def!(a = ireduce(b)),
538 ],
539 );
540
541 widen.legalize(
542 def!(a = load.I16(flags, ptr, offset)),
543 vec![
544 def!(b = uload16.I32(flags, ptr, offset)),
545 def!(a = ireduce(b)),
546 ],
547 );
548
549 for &int_ty in &[I8, I16] {
550 widen.legalize(
551 def!(br_table.int_ty(x, y, z)),
552 vec![def!(b = uextend.I32(x)), def!(br_table(b, y, z))],
553 );
554 }
555
556 for &int_ty in &[I8, I16] {
557 widen.legalize(
558 def!(a = bint.int_ty(b)),
559 vec![def!(x = bint.I32(b)), def!(a = ireduce.int_ty(x))],
560 );
561 }
562
563 for &int_ty in &[I8, I16] {
564 for &op in &[ishl, ishl_imm, ushr, ushr_imm] {
565 widen.legalize(
566 def!(a = op.int_ty(b, c)),
567 vec![
568 def!(x = uextend.I32(b)),
569 def!(z = op.I32(x, c)),
570 def!(a = ireduce.int_ty(z)),
571 ],
572 );
573 }
574
575 for &op in &[sshr, sshr_imm] {
576 widen.legalize(
577 def!(a = op.int_ty(b, c)),
578 vec![
579 def!(x = sextend.I32(b)),
580 def!(z = op.I32(x, c)),
581 def!(a = ireduce.int_ty(z)),
582 ],
583 );
584 }
585
586 for cc in &["eq", "ne", "ugt", "ult", "uge", "ule"] {
587 let w_cc = Literal::enumerator_for(&imm.intcc, cc);
588 widen.legalize(
589 def!(a = icmp_imm.int_ty(w_cc, b, c)),
590 vec![def!(x = uextend.I32(b)), def!(a = icmp_imm(w_cc, x, c))],
591 );
592 widen.legalize(
593 def!(a = icmp.int_ty(w_cc, b, c)),
594 vec![
595 def!(x = uextend.I32(b)),
596 def!(y = uextend.I32(c)),
597 def!(a = icmp.I32(w_cc, x, y)),
598 ],
599 );
600 }
601
602 for cc in &["sgt", "slt", "sge", "sle"] {
603 let w_cc = Literal::enumerator_for(&imm.intcc, cc);
604 widen.legalize(
605 def!(a = icmp_imm.int_ty(w_cc, b, c)),
606 vec![def!(x = sextend.I32(b)), def!(a = icmp_imm(w_cc, x, c))],
607 );
608
609 widen.legalize(
610 def!(a = icmp.int_ty(w_cc, b, c)),
611 vec![
612 def!(x = sextend.I32(b)),
613 def!(y = sextend.I32(c)),
614 def!(a = icmp(w_cc, x, y)),
615 ],
616 );
617 }
618 }
619
620 for &ty in &[I8, I16] {
621 widen.legalize(
622 def!(brz.ty(x, block, vararg)),
623 vec![def!(a = uextend.I32(x)), def!(brz(a, block, vararg))],
624 );
625
626 widen.legalize(
627 def!(brnz.ty(x, block, vararg)),
628 vec![def!(a = uextend.I32(x)), def!(brnz(a, block, vararg))],
629 );
630 }
631
632 // Expand integer operations with carry for RISC architectures that don't have
633 // the flags.
634 let intcc_ult = Literal::enumerator_for(&imm.intcc, "ult");
635 expand.legalize(
636 def!((a, c) = iadd_cout(x, y)),
637 vec![def!(a = iadd(x, y)), def!(c = icmp(intcc_ult, a, x))],
638 );
639
640 let intcc_ugt = Literal::enumerator_for(&imm.intcc, "ugt");
641 expand.legalize(
642 def!((a, b) = isub_bout(x, y)),
643 vec![def!(a = isub(x, y)), def!(b = icmp(intcc_ugt, a, x))],
644 );
645
646 expand.legalize(
647 def!(a = iadd_cin(x, y, c)),
648 vec![
649 def!(a1 = iadd(x, y)),
650 def!(c_int = bint(c)),
651 def!(a = iadd(a1, c_int)),
652 ],
653 );
654
655 expand.legalize(
656 def!(a = isub_bin(x, y, b)),
657 vec![
658 def!(a1 = isub(x, y)),
659 def!(b_int = bint(b)),
660 def!(a = isub(a1, b_int)),
661 ],
662 );
663
664 expand.legalize(
665 def!((a, c) = iadd_carry(x, y, c_in)),
666 vec![
667 def!((a1, c1) = iadd_cout(x, y)),
668 def!(c_int = bint(c_in)),
669 def!((a, c2) = iadd_cout(a1, c_int)),
670 def!(c = bor(c1, c2)),
671 ],
672 );
673
674 expand.legalize(
675 def!((a, b) = isub_borrow(x, y, b_in)),
676 vec![
677 def!((a1, b1) = isub_bout(x, y)),
678 def!(b_int = bint(b_in)),
679 def!((a, b2) = isub_bout(a1, b_int)),
680 def!(b = bor(b1, b2)),
681 ],
682 );
683
684 // Expansion for fcvt_from_sint for smaller integer types.
685 // This uses expand and not widen because the controlling type variable for
686 // this instruction is f32/f64, which is legalized as part of the expand
687 // group.
688 for &dest_ty in &[F32, F64] {
689 for &src_ty in &[I8, I16] {
690 let bound_inst = fcvt_from_sint.bind(dest_ty).bind(src_ty);
691 expand.legalize(
692 def!(a = bound_inst(b)),
693 vec![
694 def!(x = sextend.I32(b)),
695 def!(a = fcvt_from_sint.dest_ty(x)),
696 ],
697 );
698 }
699 }
700
701 // Expansions for immediate operands that are out of range.
702 for &(inst_imm, inst) in &[
703 (iadd_imm, iadd),
704 (imul_imm, imul),
705 (sdiv_imm, sdiv),
706 (udiv_imm, udiv),
707 (srem_imm, srem),
708 (urem_imm, urem),
709 (band_imm, band),
710 (bor_imm, bor),
711 (bxor_imm, bxor),
712 (ifcmp_imm, ifcmp),
713 ] {
714 expand.legalize(
715 def!(a = inst_imm(x, y)),
716 vec![def!(a1 = iconst(y)), def!(a = inst(x, a1))],
717 );
718 }
719
720 expand.legalize(
721 def!(a = irsub_imm(y, x)),
722 vec![def!(a1 = iconst(x)), def!(a = isub(a1, y))],
723 );
724
725 // Rotates and shifts.
726 for &(inst_imm, inst) in &[
727 (rotl_imm, rotl),
728 (rotr_imm, rotr),
729 (ishl_imm, ishl),
730 (sshr_imm, sshr),
731 (ushr_imm, ushr),
732 ] {
733 expand.legalize(
734 def!(a = inst_imm(x, y)),
735 vec![def!(a1 = iconst.I32(y)), def!(a = inst(x, a1))],
736 );
737 }
738
739 expand.legalize(
740 def!(a = icmp_imm(cc, x, y)),
741 vec![def!(a1 = iconst(y)), def!(a = icmp(cc, x, a1))],
742 );
743
744 //# Expansions for *_not variants of bitwise ops.
745 for &(inst_not, inst) in &[(band_not, band), (bor_not, bor), (bxor_not, bxor)] {
746 expand.legalize(
747 def!(a = inst_not(x, y)),
748 vec![def!(a1 = bnot(y)), def!(a = inst(x, a1))],
749 );
750 }
751
752 //# Expand bnot using xor.
753 let minus_one = Literal::constant(&imm.imm64, -1);
754 expand.legalize(
755 def!(a = bnot(x)),
756 vec![def!(y = iconst(minus_one)), def!(a = bxor(x, y))],
757 );
758
759 //# Expand bitrev
760 //# Adapted from Stack Overflow.
761 //# https://stackoverflow.com/questions/746171/most-efficient-algorithm-for-bit-reversal-from-msb-lsb-to-lsb-msb-in-c
762 let imm64_1 = Literal::constant(&imm.imm64, 1);
763 let imm64_2 = Literal::constant(&imm.imm64, 2);
764 let imm64_4 = Literal::constant(&imm.imm64, 4);
765
766 widen.legalize(
767 def!(a = bitrev.I8(x)),
768 vec![
769 def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaa))),
770 def!(a2 = ushr_imm(a1, imm64_1)),
771 def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x55))),
772 def!(a4 = ishl_imm(a3, imm64_1)),
773 def!(b = bor(a2, a4)),
774 def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcc))),
775 def!(b2 = ushr_imm(b1, imm64_2)),
776 def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x33))),
777 def!(b4 = ishl_imm(b3, imm64_2)),
778 def!(c = bor(b2, b4)),
779 def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0))),
780 def!(c2 = ushr_imm(c1, imm64_4)),
781 def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f))),
782 def!(c4 = ishl_imm(c3, imm64_4)),
783 def!(a = bor(c2, c4)),
784 ],
785 );
786
787 let imm64_8 = Literal::constant(&imm.imm64, 8);
788
789 widen.legalize(
790 def!(a = bitrev.I16(x)),
791 vec![
792 def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaaaa))),
793 def!(a2 = ushr_imm(a1, imm64_1)),
794 def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x5555))),
795 def!(a4 = ishl_imm(a3, imm64_1)),
796 def!(b = bor(a2, a4)),
797 def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcccc))),
798 def!(b2 = ushr_imm(b1, imm64_2)),
799 def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x3333))),
800 def!(b4 = ishl_imm(b3, imm64_2)),
801 def!(c = bor(b2, b4)),
802 def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0f0))),
803 def!(c2 = ushr_imm(c1, imm64_4)),
804 def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f0f))),
805 def!(c4 = ishl_imm(c3, imm64_4)),
806 def!(d = bor(c2, c4)),
807 def!(d1 = band_imm(d, Literal::constant(&imm.imm64, 0xff00))),
808 def!(d2 = ushr_imm(d1, imm64_8)),
809 def!(d3 = band_imm(d, Literal::constant(&imm.imm64, 0x00ff))),
810 def!(d4 = ishl_imm(d3, imm64_8)),
811 def!(a = bor(d2, d4)),
812 ],
813 );
814
815 let imm64_16 = Literal::constant(&imm.imm64, 16);
816
817 expand.legalize(
818 def!(a = bitrev.I32(x)),
819 vec![
820 def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaaaa_aaaa))),
821 def!(a2 = ushr_imm(a1, imm64_1)),
822 def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x5555_5555))),
823 def!(a4 = ishl_imm(a3, imm64_1)),
824 def!(b = bor(a2, a4)),
825 def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcccc_cccc))),
826 def!(b2 = ushr_imm(b1, imm64_2)),
827 def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x3333_3333))),
828 def!(b4 = ishl_imm(b3, imm64_2)),
829 def!(c = bor(b2, b4)),
830 def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0f0_f0f0))),
831 def!(c2 = ushr_imm(c1, imm64_4)),
832 def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f0f_0f0f))),
833 def!(c4 = ishl_imm(c3, imm64_4)),
834 def!(d = bor(c2, c4)),
835 def!(d1 = band_imm(d, Literal::constant(&imm.imm64, 0xff00_ff00))),
836 def!(d2 = ushr_imm(d1, imm64_8)),
837 def!(d3 = band_imm(d, Literal::constant(&imm.imm64, 0x00ff_00ff))),
838 def!(d4 = ishl_imm(d3, imm64_8)),
839 def!(e = bor(d2, d4)),
840 def!(e1 = ushr_imm(e, imm64_16)),
841 def!(e2 = ishl_imm(e, imm64_16)),
842 def!(a = bor(e1, e2)),
843 ],
844 );
845
846 #[allow(overflowing_literals)]
847 let imm64_0xaaaaaaaaaaaaaaaa = Literal::constant(&imm.imm64, 0xaaaa_aaaa_aaaa_aaaa);
848 let imm64_0x5555555555555555 = Literal::constant(&imm.imm64, 0x5555_5555_5555_5555);
849 #[allow(overflowing_literals)]
850 let imm64_0xcccccccccccccccc = Literal::constant(&imm.imm64, 0xcccc_cccc_cccc_cccc);
851 let imm64_0x3333333333333333 = Literal::constant(&imm.imm64, 0x3333_3333_3333_3333);
852 #[allow(overflowing_literals)]
853 let imm64_0xf0f0f0f0f0f0f0f0 = Literal::constant(&imm.imm64, 0xf0f0_f0f0_f0f0_f0f0);
854 let imm64_0x0f0f0f0f0f0f0f0f = Literal::constant(&imm.imm64, 0x0f0f_0f0f_0f0f_0f0f);
855 #[allow(overflowing_literals)]
856 let imm64_0xff00ff00ff00ff00 = Literal::constant(&imm.imm64, 0xff00_ff00_ff00_ff00);
857 let imm64_0x00ff00ff00ff00ff = Literal::constant(&imm.imm64, 0x00ff_00ff_00ff_00ff);
858 #[allow(overflowing_literals)]
859 let imm64_0xffff0000ffff0000 = Literal::constant(&imm.imm64, 0xffff_0000_ffff_0000);
860 let imm64_0x0000ffff0000ffff = Literal::constant(&imm.imm64, 0x0000_ffff_0000_ffff);
861 let imm64_32 = Literal::constant(&imm.imm64, 32);
862
863 expand.legalize(
864 def!(a = bitrev.I64(x)),
865 vec![
866 def!(a1 = band_imm(x, imm64_0xaaaaaaaaaaaaaaaa)),
867 def!(a2 = ushr_imm(a1, imm64_1)),
868 def!(a3 = band_imm(x, imm64_0x5555555555555555)),
869 def!(a4 = ishl_imm(a3, imm64_1)),
870 def!(b = bor(a2, a4)),
871 def!(b1 = band_imm(b, imm64_0xcccccccccccccccc)),
872 def!(b2 = ushr_imm(b1, imm64_2)),
873 def!(b3 = band_imm(b, imm64_0x3333333333333333)),
874 def!(b4 = ishl_imm(b3, imm64_2)),
875 def!(c = bor(b2, b4)),
876 def!(c1 = band_imm(c, imm64_0xf0f0f0f0f0f0f0f0)),
877 def!(c2 = ushr_imm(c1, imm64_4)),
878 def!(c3 = band_imm(c, imm64_0x0f0f0f0f0f0f0f0f)),
879 def!(c4 = ishl_imm(c3, imm64_4)),
880 def!(d = bor(c2, c4)),
881 def!(d1 = band_imm(d, imm64_0xff00ff00ff00ff00)),
882 def!(d2 = ushr_imm(d1, imm64_8)),
883 def!(d3 = band_imm(d, imm64_0x00ff00ff00ff00ff)),
884 def!(d4 = ishl_imm(d3, imm64_8)),
885 def!(e = bor(d2, d4)),
886 def!(e1 = band_imm(e, imm64_0xffff0000ffff0000)),
887 def!(e2 = ushr_imm(e1, imm64_16)),
888 def!(e3 = band_imm(e, imm64_0x0000ffff0000ffff)),
889 def!(e4 = ishl_imm(e3, imm64_16)),
890 def!(f = bor(e2, e4)),
891 def!(f1 = ushr_imm(f, imm64_32)),
892 def!(f2 = ishl_imm(f, imm64_32)),
893 def!(a = bor(f1, f2)),
894 ],
895 );
896
897 narrow.legalize(
898 def!(a = bitrev.I128(x)),
899 vec![
900 def!((xl, xh) = isplit(x)),
901 def!(yh = bitrev(xl)),
902 def!(yl = bitrev(xh)),
903 def!(a = iconcat(yl, yh)),
904 ],
905 );
906
907 // Floating-point sign manipulations.
908 for &(ty, const_inst, minus_zero) in &[
909 (F32, f32const, &Literal::bits(&imm.ieee32, 0x8000_0000)),
910 (
911 F64,
912 f64const,
913 &Literal::bits(&imm.ieee64, 0x8000_0000_0000_0000),
914 ),
915 ] {
916 expand.legalize(
917 def!(a = fabs.ty(x)),
918 vec![def!(b = const_inst(minus_zero)), def!(a = band_not(x, b))],
919 );
920
921 expand.legalize(
922 def!(a = fneg.ty(x)),
923 vec![def!(b = const_inst(minus_zero)), def!(a = bxor(x, b))],
924 );
925
926 expand.legalize(
927 def!(a = fcopysign.ty(x, y)),
928 vec![
929 def!(b = const_inst(minus_zero)),
930 def!(a1 = band_not(x, b)),
931 def!(a2 = band(y, b)),
932 def!(a = bor(a1, a2)),
933 ],
934 );
935 }
936
937 expand.custom_legalize(br_icmp, "expand_br_icmp");
938
939 let mut groups = TransformGroups::new();
940
941 let narrow_id = narrow.build_and_add_to(&mut groups);
942 let expand_id = expand.build_and_add_to(&mut groups);
943
944 // Expansions using CPU flags.
945 let mut expand_flags = TransformGroupBuilder::new(
946 "expand_flags",
947 r#"
948 Instruction expansions for architectures with flags.
949
950 Expand some instructions using CPU flags, then fall back to the normal
951 expansions. Not all architectures support CPU flags, so these patterns
952 are kept separate.
953 "#,
954 )
955 .chain_with(expand_id);
956
957 let imm64_0 = Literal::constant(&imm.imm64, 0);
958 let intcc_ne = Literal::enumerator_for(&imm.intcc, "ne");
959 let intcc_eq = Literal::enumerator_for(&imm.intcc, "eq");
960
961 expand_flags.legalize(
962 def!(trapnz(x, c)),
963 vec![
964 def!(a = ifcmp_imm(x, imm64_0)),
965 def!(trapif(intcc_ne, a, c)),
966 ],
967 );
968
969 expand_flags.legalize(
970 def!(trapz(x, c)),
971 vec![
972 def!(a = ifcmp_imm(x, imm64_0)),
973 def!(trapif(intcc_eq, a, c)),
974 ],
975 );
976
977 expand_flags.build_and_add_to(&mut groups);
978
979 // Narrow legalizations using CPU flags.
980 let mut narrow_flags = TransformGroupBuilder::new(
981 "narrow_flags",
982 r#"
983 Narrow instructions for architectures with flags.
984
985 Narrow some instructions using CPU flags, then fall back to the normal
986 legalizations. Not all architectures support CPU flags, so these
987 patterns are kept separate.
988 "#,
989 )
990 .chain_with(narrow_id);
991
992 narrow_flags.legalize(
993 def!(a = iadd(x, y)),
994 vec![
995 def!((xl, xh) = isplit(x)),
996 def!((yl, yh) = isplit(y)),
997 def!((al, c) = iadd_ifcout(xl, yl)),
998 def!(ah = iadd_ifcin(xh, yh, c)),
999 def!(a = iconcat(al, ah)),
1000 ],
1001 );
1002
1003 narrow_flags.legalize(
1004 def!(a = isub(x, y)),
1005 vec![
1006 def!((xl, xh) = isplit(x)),
1007 def!((yl, yh) = isplit(y)),
1008 def!((al, b) = isub_ifbout(xl, yl)),
1009 def!(ah = isub_ifbin(xh, yh, b)),
1010 def!(a = iconcat(al, ah)),
1011 ],
1012 );
1013
1014 narrow_flags.build_and_add_to(&mut groups);
1015
1016 // TODO(ryzokuken): figure out a way to legalize iadd_c* to iadd_ifc* (and
1017 // similarly isub_b* to isub_ifb*) on expand_flags so that this isn't required.
1018 // Narrow legalizations for ISAs that don't have CPU flags.
1019 let mut narrow_no_flags = TransformGroupBuilder::new(
1020 "narrow_no_flags",
1021 r#"
1022 Narrow instructions for architectures without flags.
1023
1024 Narrow some instructions avoiding the use of CPU flags, then fall back
1025 to the normal legalizations. Not all architectures support CPU flags,
1026 so these patterns are kept separate.
1027 "#,
1028 )
1029 .chain_with(narrow_id);
1030
1031 narrow_no_flags.legalize(
1032 def!(a = iadd(x, y)),
1033 vec![
1034 def!((xl, xh) = isplit(x)),
1035 def!((yl, yh) = isplit(y)),
1036 def!((al, c) = iadd_cout(xl, yl)),
1037 def!(ah = iadd_cin(xh, yh, c)),
1038 def!(a = iconcat(al, ah)),
1039 ],
1040 );
1041
1042 narrow_no_flags.legalize(
1043 def!(a = isub(x, y)),
1044 vec![
1045 def!((xl, xh) = isplit(x)),
1046 def!((yl, yh) = isplit(y)),
1047 def!((al, b) = isub_bout(xl, yl)),
1048 def!(ah = isub_bin(xh, yh, b)),
1049 def!(a = iconcat(al, ah)),
1050 ],
1051 );
1052
1053 narrow_no_flags.build_and_add_to(&mut groups);
1054
1055 // TODO The order of declarations unfortunately matters to be compatible with the Python code.
1056 // When it's all migrated, we can put this next to the narrow/expand build_and_add_to calls
1057 // above.
1058 widen.build_and_add_to(&mut groups);
1059
1060 groups
1061 }
1062