1// Copyright 2016 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// Lowering arithmetic
6(Add(Ptr|32|16|8) x y) -> (ADDL  x y)
7(Add(32|64)F x y) -> (ADDS(S|D) x y)
8(Add32carry x y) -> (ADDLcarry x y)
9(Add32withcarry x y c) -> (ADCL x y c)
10
11(Sub(Ptr|32|16|8) x y) -> (SUBL  x y)
12(Sub(32|64)F x y) -> (SUBS(S|D) x y)
13(Sub32carry x y) -> (SUBLcarry x y)
14(Sub32withcarry x y c) -> (SBBL x y c)
15
16(Mul(32|16|8)  x y) -> (MULL  x y)
17(Mul(32|64)F x y) -> (MULS(S|D) x y)
18(Mul32uhilo x y) -> (MULLQU x y)
19
20(Select0 (Mul32uover x y)) -> (Select0 <typ.UInt32> (MULLU x y))
21(Select1 (Mul32uover x y)) -> (SETO (Select1 <types.TypeFlags> (MULLU x y)))
22
23(Avg32u x y) -> (AVGLU x y)
24
25(Div32F x y) -> (DIVSS x y)
26(Div64F x y) -> (DIVSD x y)
27
28(Div32 [a] x y) -> (DIVL [a] x y)
29(Div32u x y) -> (DIVLU x y)
30(Div16 [a] x y) -> (DIVW [a] x y)
31(Div16u x y) -> (DIVWU x y)
32(Div8   x y) -> (DIVW  (SignExt8to16 x) (SignExt8to16 y))
33(Div8u  x y) -> (DIVWU (ZeroExt8to16 x) (ZeroExt8to16 y))
34
35(Hmul32  x y) -> (HMULL  x y)
36(Hmul32u x y) -> (HMULLU x y)
37
38(Mod32 [a] x y) -> (MODL [a] x y)
39(Mod32u x y) -> (MODLU x y)
40(Mod16 [a] x y) -> (MODW [a] x y)
41(Mod16u x y) -> (MODWU x y)
42(Mod8   x y) -> (MODW  (SignExt8to16 x) (SignExt8to16 y))
43(Mod8u  x y) -> (MODWU (ZeroExt8to16 x) (ZeroExt8to16 y))
44
45(And(32|16|8) x y) -> (ANDL x y)
46(Or(32|16|8) x y) -> (ORL x y)
47(Xor(32|16|8) x y) -> (XORL x y)
48
49(Neg(32|16|8)  x) -> (NEGL x)
50(Neg32F x) && !config.use387 -> (PXOR x (MOVSSconst <typ.Float32> [auxFrom32F(float32(math.Copysign(0, -1)))]))
51(Neg64F x) && !config.use387 -> (PXOR x (MOVSDconst <typ.Float64> [auxFrom64F(math.Copysign(0, -1))]))
52(Neg32F x) && config.use387 -> (FCHS x)
53(Neg64F x) && config.use387 -> (FCHS x)
54
55(Com(32|16|8) x) -> (NOTL x)
56
57// Lowering boolean ops
58(AndB x y) -> (ANDL x y)
59(OrB x y) -> (ORL x y)
60(Not x) -> (XORLconst [1] x)
61
62// Lowering pointer arithmetic
63(OffPtr [off] ptr) -> (ADDLconst [off] ptr)
64
65(Bswap32 x) -> (BSWAPL x)
66
67(Sqrt x) -> (SQRTSD x)
68
69(Ctz16 x) -> (BSFL (ORLconst <typ.UInt32> [0x10000] x))
70(Ctz16NonZero x) -> (BSFL x)
71
72// Lowering extension
73(SignExt8to16  x) -> (MOVBLSX x)
74(SignExt8to32  x) -> (MOVBLSX x)
75(SignExt16to32 x) -> (MOVWLSX x)
76
77(ZeroExt8to16  x) -> (MOVBLZX x)
78(ZeroExt8to32  x) -> (MOVBLZX x)
79(ZeroExt16to32 x) -> (MOVWLZX x)
80
81(Signmask x) -> (SARLconst x [31])
82(Zeromask <t> x) -> (XORLconst [-1] (SBBLcarrymask <t> (CMPLconst x [1])))
83(Slicemask <t> x) -> (SARLconst (NEGL <t> x) [31])
84
85// Lowering truncation
86// Because we ignore high parts of registers, truncates are just copies.
87(Trunc16to8  x) -> x
88(Trunc32to8  x) -> x
89(Trunc32to16 x) -> x
90
91// Lowering float <-> int
92(Cvt32to32F x) -> (CVTSL2SS x)
93(Cvt32to64F x) -> (CVTSL2SD x)
94
95(Cvt32Fto32 x) -> (CVTTSS2SL x)
96(Cvt64Fto32 x) -> (CVTTSD2SL x)
97
98(Cvt32Fto64F x) -> (CVTSS2SD x)
99(Cvt64Fto32F x) -> (CVTSD2SS x)
100
101(Round32F x) -> x
102(Round64F x) -> x
103
104// Lowering shifts
105// Unsigned shifts need to return 0 if shift amount is >= width of shifted value.
106//   result = (arg << shift) & (shift >= argbits ? 0 : 0xffffffffffffffff)
107(Lsh32x32 <t> x y) -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPLconst y [32])))
108(Lsh32x16 <t> x y) -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPWconst y [32])))
109(Lsh32x8  <t> x y) -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPBconst y [32])))
110
111(Lsh16x32 <t> x y) -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPLconst y [32])))
112(Lsh16x16 <t> x y) -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPWconst y [32])))
113(Lsh16x8  <t> x y) -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPBconst y [32])))
114
115(Lsh8x32 <t> x y)  -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPLconst y [32])))
116(Lsh8x16 <t> x y)  -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPWconst y [32])))
117(Lsh8x8  <t> x y)  -> (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMPBconst y [32])))
118
119(Rsh32Ux32 <t> x y) -> (ANDL (SHRL <t> x y) (SBBLcarrymask <t> (CMPLconst y [32])))
120(Rsh32Ux16 <t> x y) -> (ANDL (SHRL <t> x y) (SBBLcarrymask <t> (CMPWconst y [32])))
121(Rsh32Ux8  <t> x y) -> (ANDL (SHRL <t> x y) (SBBLcarrymask <t> (CMPBconst y [32])))
122
123(Rsh16Ux32 <t> x y) -> (ANDL (SHRW <t> x y) (SBBLcarrymask <t> (CMPLconst y [16])))
124(Rsh16Ux16 <t> x y) -> (ANDL (SHRW <t> x y) (SBBLcarrymask <t> (CMPWconst y [16])))
125(Rsh16Ux8  <t> x y) -> (ANDL (SHRW <t> x y) (SBBLcarrymask <t> (CMPBconst y [16])))
126
127(Rsh8Ux32 <t> x y)  -> (ANDL (SHRB <t> x y) (SBBLcarrymask <t> (CMPLconst y [8])))
128(Rsh8Ux16 <t> x y)  -> (ANDL (SHRB <t> x y) (SBBLcarrymask <t> (CMPWconst y [8])))
129(Rsh8Ux8  <t> x y)  -> (ANDL (SHRB <t> x y) (SBBLcarrymask <t> (CMPBconst y [8])))
130
131// Signed right shift needs to return 0/-1 if shift amount is >= width of shifted value.
132// We implement this by setting the shift value to -1 (all ones) if the shift value is >= width.
133
134(Rsh32x32 <t> x y) -> (SARL <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPLconst y [32])))))
135(Rsh32x16 <t> x y) -> (SARL <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPWconst y [32])))))
136(Rsh32x8  <t> x y) -> (SARL <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPBconst y [32])))))
137
138(Rsh16x32 <t> x y) -> (SARW <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPLconst y [16])))))
139(Rsh16x16 <t> x y) -> (SARW <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPWconst y [16])))))
140(Rsh16x8  <t> x y) -> (SARW <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPBconst y [16])))))
141
142(Rsh8x32 <t> x y)  -> (SARB <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPLconst y [8])))))
143(Rsh8x16 <t> x y)  -> (SARB <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPWconst y [8])))))
144(Rsh8x8  <t> x y)  -> (SARB <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMPBconst y [8])))))
145
146// constant shifts
147// generic opt rewrites all constant shifts to shift by Const64
148(Lsh32x64 x (Const64 [c])) && uint64(c) < 32 -> (SHLLconst x [c])
149(Rsh32x64 x (Const64 [c])) && uint64(c) < 32 -> (SARLconst x [c])
150(Rsh32Ux64 x (Const64 [c])) && uint64(c) < 32 -> (SHRLconst x [c])
151(Lsh16x64 x (Const64 [c])) && uint64(c) < 16 -> (SHLLconst x [c])
152(Rsh16x64 x (Const64 [c])) && uint64(c) < 16 -> (SARWconst x [c])
153(Rsh16Ux64 x (Const64 [c])) && uint64(c) < 16 -> (SHRWconst x [c])
154(Lsh8x64 x (Const64 [c])) && uint64(c) < 8 -> (SHLLconst x [c])
155(Rsh8x64 x (Const64 [c])) && uint64(c) < 8 -> (SARBconst x [c])
156(Rsh8Ux64 x (Const64 [c])) && uint64(c) < 8 -> (SHRBconst x [c])
157
158// large constant shifts
159(Lsh32x64 _ (Const64 [c])) && uint64(c) >= 32 -> (Const32 [0])
160(Rsh32Ux64 _ (Const64 [c])) && uint64(c) >= 32 -> (Const32 [0])
161(Lsh16x64 _ (Const64 [c])) && uint64(c) >= 16 -> (Const16 [0])
162(Rsh16Ux64 _ (Const64 [c])) && uint64(c) >= 16 -> (Const16 [0])
163(Lsh8x64 _ (Const64 [c])) && uint64(c) >= 8 -> (Const8 [0])
164(Rsh8Ux64 _ (Const64 [c])) && uint64(c) >= 8 -> (Const8 [0])
165
166// large constant signed right shift, we leave the sign bit
167(Rsh32x64 x (Const64 [c])) && uint64(c) >= 32 -> (SARLconst x [31])
168(Rsh16x64 x (Const64 [c])) && uint64(c) >= 16 -> (SARWconst x [15])
169(Rsh8x64 x (Const64 [c])) && uint64(c) >= 8 -> (SARBconst x [7])
170
171// constant rotates
172(RotateLeft32 x (MOVLconst [c])) -> (ROLLconst [c&31] x)
173(RotateLeft16 x (MOVLconst [c])) -> (ROLWconst [c&15] x)
174(RotateLeft8 x (MOVLconst [c]))  -> (ROLBconst [c&7] x)
175
176// Lowering comparisons
177(Less32  x y) -> (SETL (CMPL x y))
178(Less16  x y) -> (SETL (CMPW x y))
179(Less8   x y) -> (SETL (CMPB x y))
180(Less32U x y) -> (SETB (CMPL x y))
181(Less16U x y) -> (SETB (CMPW x y))
182(Less8U  x y) -> (SETB (CMPB x y))
183// Use SETGF with reversed operands to dodge NaN case
184(Less64F x y) -> (SETGF (UCOMISD y x))
185(Less32F x y) -> (SETGF (UCOMISS y x))
186
187(Leq32  x y) -> (SETLE (CMPL x y))
188(Leq16  x y) -> (SETLE (CMPW x y))
189(Leq8   x y) -> (SETLE (CMPB x y))
190(Leq32U x y) -> (SETBE (CMPL x y))
191(Leq16U x y) -> (SETBE (CMPW x y))
192(Leq8U  x y) -> (SETBE (CMPB x y))
193// Use SETGEF with reversed operands to dodge NaN case
194(Leq64F x y) -> (SETGEF (UCOMISD y x))
195(Leq32F x y) -> (SETGEF (UCOMISS y x))
196
197(Greater32  x y) -> (SETG (CMPL x y))
198(Greater16  x y) -> (SETG (CMPW x y))
199(Greater8   x y) -> (SETG (CMPB x y))
200(Greater32U x y) -> (SETA (CMPL x y))
201(Greater16U x y) -> (SETA (CMPW x y))
202(Greater8U  x y) -> (SETA (CMPB x y))
203// Note Go assembler gets UCOMISx operand order wrong, but it is right here
204// Bug is accommodated at generation of assembly language.
205(Greater64F x y) -> (SETGF (UCOMISD x y))
206(Greater32F x y) -> (SETGF (UCOMISS x y))
207
208(Geq32  x y) -> (SETGE (CMPL x y))
209(Geq16  x y) -> (SETGE (CMPW x y))
210(Geq8   x y) -> (SETGE (CMPB x y))
211(Geq32U x y) -> (SETAE (CMPL x y))
212(Geq16U x y) -> (SETAE (CMPW x y))
213(Geq8U  x y) -> (SETAE (CMPB x y))
214// Note Go assembler gets UCOMISx operand order wrong, but it is right here
215// Bug is accommodated at generation of assembly language.
216(Geq64F x y) -> (SETGEF (UCOMISD x y))
217(Geq32F x y) -> (SETGEF (UCOMISS x y))
218
219(Eq32  x y) -> (SETEQ (CMPL x y))
220(Eq16  x y) -> (SETEQ (CMPW x y))
221(Eq8   x y) -> (SETEQ (CMPB x y))
222(EqB   x y) -> (SETEQ (CMPB x y))
223(EqPtr x y) -> (SETEQ (CMPL x y))
224(Eq64F x y) -> (SETEQF (UCOMISD x y))
225(Eq32F x y) -> (SETEQF (UCOMISS x y))
226
227(Neq32  x y) -> (SETNE (CMPL x y))
228(Neq16  x y) -> (SETNE (CMPW x y))
229(Neq8   x y) -> (SETNE (CMPB x y))
230(NeqB   x y) -> (SETNE (CMPB x y))
231(NeqPtr x y) -> (SETNE (CMPL x y))
232(Neq64F x y) -> (SETNEF (UCOMISD x y))
233(Neq32F x y) -> (SETNEF (UCOMISS x y))
234
235// Lowering loads
236(Load <t> ptr mem) && (is32BitInt(t) || isPtr(t)) -> (MOVLload ptr mem)
237(Load <t> ptr mem) && is16BitInt(t) -> (MOVWload ptr mem)
238(Load <t> ptr mem) && (t.IsBoolean() || is8BitInt(t)) -> (MOVBload ptr mem)
239(Load <t> ptr mem) && is32BitFloat(t) -> (MOVSSload ptr mem)
240(Load <t> ptr mem) && is64BitFloat(t) -> (MOVSDload ptr mem)
241
242// Lowering stores
243// These more-specific FP versions of Store pattern should come first.
244(Store {t} ptr val mem) && t.(*types.Type).Size() == 8 && is64BitFloat(val.Type) -> (MOVSDstore ptr val mem)
245(Store {t} ptr val mem) && t.(*types.Type).Size() == 4 && is32BitFloat(val.Type) -> (MOVSSstore ptr val mem)
246
247(Store {t} ptr val mem) && t.(*types.Type).Size() == 4 -> (MOVLstore ptr val mem)
248(Store {t} ptr val mem) && t.(*types.Type).Size() == 2 -> (MOVWstore ptr val mem)
249(Store {t} ptr val mem) && t.(*types.Type).Size() == 1 -> (MOVBstore ptr val mem)
250
251// Lowering moves
252(Move [0] _ _ mem) -> mem
253(Move [1] dst src mem) -> (MOVBstore dst (MOVBload src mem) mem)
254(Move [2] dst src mem) -> (MOVWstore dst (MOVWload src mem) mem)
255(Move [4] dst src mem) -> (MOVLstore dst (MOVLload src mem) mem)
256(Move [3] dst src mem) ->
257	(MOVBstore [2] dst (MOVBload [2] src mem)
258		(MOVWstore dst (MOVWload src mem) mem))
259(Move [5] dst src mem) ->
260	(MOVBstore [4] dst (MOVBload [4] src mem)
261		(MOVLstore dst (MOVLload src mem) mem))
262(Move [6] dst src mem) ->
263	(MOVWstore [4] dst (MOVWload [4] src mem)
264		(MOVLstore dst (MOVLload src mem) mem))
265(Move [7] dst src mem) ->
266	(MOVLstore [3] dst (MOVLload [3] src mem)
267		(MOVLstore dst (MOVLload src mem) mem))
268(Move [8] dst src mem) ->
269	(MOVLstore [4] dst (MOVLload [4] src mem)
270		(MOVLstore dst (MOVLload src mem) mem))
271
272// Adjust moves to be a multiple of 4 bytes.
273(Move [s] dst src mem)
274	&& s > 8 && s%4 != 0 ->
275	(Move [s-s%4]
276		(ADDLconst <dst.Type> dst [s%4])
277		(ADDLconst <src.Type> src [s%4])
278		(MOVLstore dst (MOVLload src mem) mem))
279
280// Medium copying uses a duff device.
281(Move [s] dst src mem)
282	&& s > 8 && s <= 4*128 && s%4 == 0
283	&& !config.noDuffDevice ->
284	(DUFFCOPY [10*(128-s/4)] dst src mem)
285// 10 and 128 are magic constants.  10 is the number of bytes to encode:
286//	MOVL	(SI), CX
287//	ADDL	$4, SI
288//	MOVL	CX, (DI)
289//	ADDL	$4, DI
290// and 128 is the number of such blocks. See src/runtime/duff_386.s:duffcopy.
291
292// Large copying uses REP MOVSL.
293(Move [s] dst src mem) && (s > 4*128 || config.noDuffDevice) && s%4 == 0 ->
294	(REPMOVSL dst src (MOVLconst [s/4]) mem)
295
296// Lowering Zero instructions
297(Zero [0] _ mem) -> mem
298(Zero [1] destptr mem) -> (MOVBstoreconst [0] destptr mem)
299(Zero [2] destptr mem) -> (MOVWstoreconst [0] destptr mem)
300(Zero [4] destptr mem) -> (MOVLstoreconst [0] destptr mem)
301
302(Zero [3] destptr mem) ->
303	(MOVBstoreconst [makeValAndOff(0,2)] destptr
304		(MOVWstoreconst [0] destptr mem))
305(Zero [5] destptr mem) ->
306	(MOVBstoreconst [makeValAndOff(0,4)] destptr
307		(MOVLstoreconst [0] destptr mem))
308(Zero [6] destptr mem) ->
309	(MOVWstoreconst [makeValAndOff(0,4)] destptr
310		(MOVLstoreconst [0] destptr mem))
311(Zero [7] destptr mem) ->
312	(MOVLstoreconst [makeValAndOff(0,3)] destptr
313		(MOVLstoreconst [0] destptr mem))
314
315// Strip off any fractional word zeroing.
316(Zero [s] destptr mem) && s%4 != 0 && s > 4 ->
317	(Zero [s-s%4] (ADDLconst destptr [s%4])
318		(MOVLstoreconst [0] destptr mem))
319
320// Zero small numbers of words directly.
321(Zero [8] destptr mem) ->
322	(MOVLstoreconst [makeValAndOff(0,4)] destptr
323		(MOVLstoreconst [0] destptr mem))
324(Zero [12] destptr mem) ->
325	(MOVLstoreconst [makeValAndOff(0,8)] destptr
326		(MOVLstoreconst [makeValAndOff(0,4)] destptr
327			(MOVLstoreconst [0] destptr mem)))
328(Zero [16] destptr mem) ->
329	(MOVLstoreconst [makeValAndOff(0,12)] destptr
330		(MOVLstoreconst [makeValAndOff(0,8)] destptr
331			(MOVLstoreconst [makeValAndOff(0,4)] destptr
332				(MOVLstoreconst [0] destptr mem))))
333
334// Medium zeroing uses a duff device.
335(Zero [s] destptr mem)
336  && s > 16 && s <= 4*128 && s%4 == 0
337  && !config.noDuffDevice ->
338	(DUFFZERO [1*(128-s/4)] destptr (MOVLconst [0]) mem)
339// 1 and 128 are magic constants.  1 is the number of bytes to encode STOSL.
340// 128 is the number of STOSL instructions in duffzero.
341// See src/runtime/duff_386.s:duffzero.
342
343// Large zeroing uses REP STOSQ.
344(Zero [s] destptr mem)
345  && (s > 4*128 || (config.noDuffDevice && s > 16))
346  && s%4 == 0 ->
347	(REPSTOSL destptr (MOVLconst [s/4]) (MOVLconst [0]) mem)
348
349// Lowering constants
350(Const(8|16|32)   [val]) -> (MOVLconst [val])
351(Const(32|64)F [val]) -> (MOVS(S|D)const [val])
352(ConstNil) -> (MOVLconst [0])
353(ConstBool [b]) -> (MOVLconst [b])
354
355// Lowering calls
356(StaticCall [argwid] {target} mem) -> (CALLstatic [argwid] {target} mem)
357(ClosureCall [argwid] entry closure mem) -> (CALLclosure [argwid] entry closure mem)
358(InterCall [argwid] entry mem) -> (CALLinter [argwid] entry mem)
359
360// Miscellaneous
361(IsNonNil p) -> (SETNE (TESTL p p))
362(IsInBounds idx len) -> (SETB (CMPL idx len))
363(IsSliceInBounds idx len) -> (SETBE (CMPL idx len))
364(NilCheck ptr mem) -> (LoweredNilCheck ptr mem)
365(GetG mem) -> (LoweredGetG mem)
366(GetClosurePtr) -> (LoweredGetClosurePtr)
367(GetCallerPC) -> (LoweredGetCallerPC)
368(GetCallerSP) -> (LoweredGetCallerSP)
369(Addr {sym} base) -> (LEAL {sym} base)
370(LocalAddr {sym} base _) -> (LEAL {sym} base)
371
372// block rewrites
373(If (SETL  cmp) yes no) -> (LT  cmp yes no)
374(If (SETLE cmp) yes no) -> (LE  cmp yes no)
375(If (SETG  cmp) yes no) -> (GT  cmp yes no)
376(If (SETGE cmp) yes no) -> (GE  cmp yes no)
377(If (SETEQ cmp) yes no) -> (EQ  cmp yes no)
378(If (SETNE cmp) yes no) -> (NE  cmp yes no)
379(If (SETB  cmp) yes no) -> (ULT cmp yes no)
380(If (SETBE cmp) yes no) -> (ULE cmp yes no)
381(If (SETA  cmp) yes no) -> (UGT cmp yes no)
382(If (SETAE cmp) yes no) -> (UGE cmp yes no)
383(If (SETO  cmp) yes no) -> (OS cmp yes no)
384
385// Special case for floating point - LF/LEF not generated
386(If (SETGF  cmp) yes no) -> (UGT  cmp yes no)
387(If (SETGEF cmp) yes no) -> (UGE  cmp yes no)
388(If (SETEQF cmp) yes no) -> (EQF  cmp yes no)
389(If (SETNEF cmp) yes no) -> (NEF  cmp yes no)
390
391(If cond yes no) -> (NE (TESTB cond cond) yes no)
392
393// Write barrier.
394(WB {fn} destptr srcptr mem) -> (LoweredWB {fn} destptr srcptr mem)
395
396(PanicBounds [kind] x y mem) && boundsABI(kind) == 0 -> (LoweredPanicBoundsA [kind] x y mem)
397(PanicBounds [kind] x y mem) && boundsABI(kind) == 1 -> (LoweredPanicBoundsB [kind] x y mem)
398(PanicBounds [kind] x y mem) && boundsABI(kind) == 2 -> (LoweredPanicBoundsC [kind] x y mem)
399
400(PanicExtend [kind] hi lo y mem) && boundsABI(kind) == 0 -> (LoweredPanicExtendA [kind] hi lo y mem)
401(PanicExtend [kind] hi lo y mem) && boundsABI(kind) == 1 -> (LoweredPanicExtendB [kind] hi lo y mem)
402(PanicExtend [kind] hi lo y mem) && boundsABI(kind) == 2 -> (LoweredPanicExtendC [kind] hi lo y mem)
403
404// ***************************
405// Above: lowering rules
406// Below: optimizations
407// ***************************
408// TODO: Should the optimizations be a separate pass?
409
410// Fold boolean tests into blocks
411(NE (TESTB (SETL  cmp) (SETL  cmp)) yes no) -> (LT  cmp yes no)
412(NE (TESTB (SETLE cmp) (SETLE cmp)) yes no) -> (LE  cmp yes no)
413(NE (TESTB (SETG  cmp) (SETG  cmp)) yes no) -> (GT  cmp yes no)
414(NE (TESTB (SETGE cmp) (SETGE cmp)) yes no) -> (GE  cmp yes no)
415(NE (TESTB (SETEQ cmp) (SETEQ cmp)) yes no) -> (EQ  cmp yes no)
416(NE (TESTB (SETNE cmp) (SETNE cmp)) yes no) -> (NE  cmp yes no)
417(NE (TESTB (SETB  cmp) (SETB  cmp)) yes no) -> (ULT cmp yes no)
418(NE (TESTB (SETBE cmp) (SETBE cmp)) yes no) -> (ULE cmp yes no)
419(NE (TESTB (SETA  cmp) (SETA  cmp)) yes no) -> (UGT cmp yes no)
420(NE (TESTB (SETAE cmp) (SETAE cmp)) yes no) -> (UGE cmp yes no)
421(NE (TESTB (SETO cmp) (SETO cmp)) yes no) -> (OS cmp yes no)
422
423// Special case for floating point - LF/LEF not generated
424(NE (TESTB (SETGF  cmp) (SETGF  cmp)) yes no) -> (UGT  cmp yes no)
425(NE (TESTB (SETGEF cmp) (SETGEF cmp)) yes no) -> (UGE  cmp yes no)
426(NE (TESTB (SETEQF cmp) (SETEQF cmp)) yes no) -> (EQF  cmp yes no)
427(NE (TESTB (SETNEF cmp) (SETNEF cmp)) yes no) -> (NEF  cmp yes no)
428
429// fold constants into instructions
430(ADDL x (MOVLconst [c])) -> (ADDLconst [c] x)
431(ADDLcarry x (MOVLconst [c])) -> (ADDLconstcarry [c] x)
432(ADCL x (MOVLconst [c]) f) -> (ADCLconst [c] x f)
433(ADCL (MOVLconst [c]) x f) -> (ADCLconst [c] x f)
434
435(SUBL x (MOVLconst [c])) -> (SUBLconst x [c])
436(SUBL (MOVLconst [c]) x) -> (NEGL (SUBLconst <v.Type> x [c]))
437(SUBLcarry x (MOVLconst [c])) -> (SUBLconstcarry [c] x)
438(SBBL x (MOVLconst [c]) f) -> (SBBLconst [c] x f)
439
440(MULL x (MOVLconst [c])) -> (MULLconst [c] x)
441
442(ANDL x (MOVLconst [c])) -> (ANDLconst [c] x)
443
444(ANDLconst [c] (ANDLconst [d] x)) -> (ANDLconst [c & d] x)
445
446(XORLconst [c] (XORLconst [d] x)) -> (XORLconst [c ^ d] x)
447
448(MULLconst [c] (MULLconst [d] x)) -> (MULLconst [int64(int32(c * d))] x)
449
450(ORL x (MOVLconst [c])) -> (ORLconst [c] x)
451
452(XORL x (MOVLconst [c])) -> (XORLconst [c] x)
453
454(SHLL x (MOVLconst [c])) -> (SHLLconst [c&31] x)
455(SHRL x (MOVLconst [c])) -> (SHRLconst [c&31] x)
456(SHRW x (MOVLconst [c])) && c&31 < 16 -> (SHRWconst [c&31] x)
457(SHRW _ (MOVLconst [c])) && c&31 >= 16 -> (MOVLconst [0])
458(SHRB x (MOVLconst [c])) && c&31 < 8 -> (SHRBconst [c&31] x)
459(SHRB _ (MOVLconst [c])) && c&31 >= 8 -> (MOVLconst [0])
460
461(SARL x (MOVLconst [c])) -> (SARLconst [c&31] x)
462(SARW x (MOVLconst [c])) -> (SARWconst [min(c&31,15)] x)
463(SARB x (MOVLconst [c])) -> (SARBconst [min(c&31,7)] x)
464
465(SARL x (ANDLconst [31] y)) -> (SARL x y)
466
467(SHLL x (ANDLconst [31] y)) -> (SHLL x y)
468
469(SHRL x (ANDLconst [31] y)) -> (SHRL x y)
470
471// Rotate instructions
472
473(ADDL (SHLLconst [c] x) (SHRLconst [d] x)) && d == 32-c -> (ROLLconst [c] x)
474( ORL (SHLLconst [c] x) (SHRLconst [d] x)) && d == 32-c -> (ROLLconst [c] x)
475(XORL (SHLLconst [c] x) (SHRLconst [d] x)) && d == 32-c -> (ROLLconst [c] x)
476
477(ADDL <t> (SHLLconst x [c]) (SHRWconst x [d])) && c < 16 && d == 16-c && t.Size() == 2 -> (ROLWconst x [c])
478( ORL <t> (SHLLconst x [c]) (SHRWconst x [d])) && c < 16 && d == 16-c && t.Size() == 2 -> (ROLWconst x [c])
479(XORL <t> (SHLLconst x [c]) (SHRWconst x [d])) && c < 16 && d == 16-c && t.Size() == 2 -> (ROLWconst x [c])
480
481(ADDL <t> (SHLLconst x [c]) (SHRBconst x [d])) && c < 8 && d == 8-c && t.Size() == 1 -> (ROLBconst x [c])
482( ORL <t> (SHLLconst x [c]) (SHRBconst x [d])) && c < 8 && d == 8-c && t.Size() == 1 -> (ROLBconst x [c])
483(XORL <t> (SHLLconst x [c]) (SHRBconst x [d])) && c < 8 && d == 8-c && t.Size() == 1 -> (ROLBconst x [c])
484
485(ROLLconst [c] (ROLLconst [d] x)) -> (ROLLconst [(c+d)&31] x)
486(ROLWconst [c] (ROLWconst [d] x)) -> (ROLWconst [(c+d)&15] x)
487(ROLBconst [c] (ROLBconst [d] x)) -> (ROLBconst [(c+d)& 7] x)
488
489// Constant shift simplifications
490
491(SHLLconst x [0]) -> x
492(SHRLconst x [0]) -> x
493(SARLconst x [0]) -> x
494
495(SHRWconst x [0]) -> x
496(SARWconst x [0]) -> x
497
498(SHRBconst x [0]) -> x
499(SARBconst x [0]) -> x
500
501(ROLLconst [0] x) -> x
502(ROLWconst [0] x) -> x
503(ROLBconst [0] x) -> x
504
505// Note: the word and byte shifts keep the low 5 bits (not the low 4 or 3 bits)
506// because the x86 instructions are defined to use all 5 bits of the shift even
507// for the small shifts. I don't think we'll ever generate a weird shift (e.g.
508// (SHRW x (MOVLconst [24])), but just in case.
509
510(CMPL x (MOVLconst [c])) -> (CMPLconst x [c])
511(CMPL (MOVLconst [c]) x) -> (InvertFlags (CMPLconst x [c]))
512(CMPW x (MOVLconst [c])) -> (CMPWconst x [int64(int16(c))])
513(CMPW (MOVLconst [c]) x) -> (InvertFlags (CMPWconst x [int64(int16(c))]))
514(CMPB x (MOVLconst [c])) -> (CMPBconst x [int64(int8(c))])
515(CMPB (MOVLconst [c]) x) -> (InvertFlags (CMPBconst x [int64(int8(c))]))
516
517// strength reduction
518// Assumes that the following costs from https://gmplib.org/~tege/x86-timing.pdf:
519//    1 - addl, shll, leal, negl, subl
520//    3 - imull
521// This limits the rewrites to two instructions.
522// Note that negl always operates in-place,
523// which can require a register-register move
524// to preserve the original value,
525// so it must be used with care.
526(MULLconst [-9] x) -> (NEGL (LEAL8 <v.Type> x x))
527(MULLconst [-5] x) -> (NEGL (LEAL4 <v.Type> x x))
528(MULLconst [-3] x) -> (NEGL (LEAL2 <v.Type> x x))
529(MULLconst [-1] x) -> (NEGL x)
530(MULLconst [0] _) -> (MOVLconst [0])
531(MULLconst [1] x) -> x
532(MULLconst [3] x) -> (LEAL2 x x)
533(MULLconst [5] x) -> (LEAL4 x x)
534(MULLconst [7] x) -> (LEAL2 x (LEAL2 <v.Type> x x))
535(MULLconst [9] x) -> (LEAL8 x x)
536(MULLconst [11] x) -> (LEAL2 x (LEAL4 <v.Type> x x))
537(MULLconst [13] x) -> (LEAL4 x (LEAL2 <v.Type> x x))
538(MULLconst [19] x) -> (LEAL2 x (LEAL8 <v.Type> x x))
539(MULLconst [21] x) -> (LEAL4 x (LEAL4 <v.Type> x x))
540(MULLconst [25] x) -> (LEAL8 x (LEAL2 <v.Type> x x))
541(MULLconst [27] x) -> (LEAL8 (LEAL2 <v.Type> x x) (LEAL2 <v.Type> x x))
542(MULLconst [37] x) -> (LEAL4 x (LEAL8 <v.Type> x x))
543(MULLconst [41] x) -> (LEAL8 x (LEAL4 <v.Type> x x))
544(MULLconst [45] x) -> (LEAL8 (LEAL4 <v.Type> x x) (LEAL4 <v.Type> x x))
545(MULLconst [73] x) -> (LEAL8 x (LEAL8 <v.Type> x x))
546(MULLconst [81] x) -> (LEAL8 (LEAL8 <v.Type> x x) (LEAL8 <v.Type> x x))
547
548(MULLconst [c] x) && isPowerOfTwo(c+1) && c >= 15 -> (SUBL (SHLLconst <v.Type> [log2(c+1)] x) x)
549(MULLconst [c] x) && isPowerOfTwo(c-1) && c >= 17 -> (LEAL1 (SHLLconst <v.Type> [log2(c-1)] x) x)
550(MULLconst [c] x) && isPowerOfTwo(c-2) && c >= 34 -> (LEAL2 (SHLLconst <v.Type> [log2(c-2)] x) x)
551(MULLconst [c] x) && isPowerOfTwo(c-4) && c >= 68 -> (LEAL4 (SHLLconst <v.Type> [log2(c-4)] x) x)
552(MULLconst [c] x) && isPowerOfTwo(c-8) && c >= 136 -> (LEAL8 (SHLLconst <v.Type> [log2(c-8)] x) x)
553(MULLconst [c] x) && c%3 == 0 && isPowerOfTwo(c/3) -> (SHLLconst [log2(c/3)] (LEAL2 <v.Type> x x))
554(MULLconst [c] x) && c%5 == 0 && isPowerOfTwo(c/5) -> (SHLLconst [log2(c/5)] (LEAL4 <v.Type> x x))
555(MULLconst [c] x) && c%9 == 0 && isPowerOfTwo(c/9) -> (SHLLconst [log2(c/9)] (LEAL8 <v.Type> x x))
556
557// combine add/shift into LEAL
558(ADDL x (SHLLconst [3] y)) -> (LEAL8 x y)
559(ADDL x (SHLLconst [2] y)) -> (LEAL4 x y)
560(ADDL x (SHLLconst [1] y)) -> (LEAL2 x y)
561(ADDL x (ADDL y y)) -> (LEAL2 x y)
562(ADDL x (ADDL x y)) -> (LEAL2 y x)
563
564// combine ADDL/ADDLconst into LEAL1
565(ADDLconst [c] (ADDL x y)) -> (LEAL1 [c] x y)
566(ADDL (ADDLconst [c] x) y) -> (LEAL1 [c] x y)
567
568// fold ADDL into LEAL
569(ADDLconst [c] (LEAL [d] {s} x)) && is32Bit(c+d) -> (LEAL [c+d] {s} x)
570(LEAL [c] {s} (ADDLconst [d] x)) && is32Bit(c+d) -> (LEAL [c+d] {s} x)
571(LEAL [c] {s} (ADDL x y)) && x.Op != OpSB && y.Op != OpSB -> (LEAL1 [c] {s} x y)
572(ADDL x (LEAL [c] {s} y)) && x.Op != OpSB && y.Op != OpSB -> (LEAL1 [c] {s} x y)
573
574// fold ADDLconst into LEALx
575(ADDLconst [c] (LEAL1 [d] {s} x y)) && is32Bit(c+d) -> (LEAL1 [c+d] {s} x y)
576(ADDLconst [c] (LEAL2 [d] {s} x y)) && is32Bit(c+d) -> (LEAL2 [c+d] {s} x y)
577(ADDLconst [c] (LEAL4 [d] {s} x y)) && is32Bit(c+d) -> (LEAL4 [c+d] {s} x y)
578(ADDLconst [c] (LEAL8 [d] {s} x y)) && is32Bit(c+d) -> (LEAL8 [c+d] {s} x y)
579(LEAL1 [c] {s} (ADDLconst [d] x) y) && is32Bit(c+d)   && x.Op != OpSB -> (LEAL1 [c+d] {s} x y)
580(LEAL2 [c] {s} (ADDLconst [d] x) y) && is32Bit(c+d)   && x.Op != OpSB -> (LEAL2 [c+d] {s} x y)
581(LEAL2 [c] {s} x (ADDLconst [d] y)) && is32Bit(c+2*d) && y.Op != OpSB -> (LEAL2 [c+2*d] {s} x y)
582(LEAL4 [c] {s} (ADDLconst [d] x) y) && is32Bit(c+d)   && x.Op != OpSB -> (LEAL4 [c+d] {s} x y)
583(LEAL4 [c] {s} x (ADDLconst [d] y)) && is32Bit(c+4*d) && y.Op != OpSB -> (LEAL4 [c+4*d] {s} x y)
584(LEAL8 [c] {s} (ADDLconst [d] x) y) && is32Bit(c+d)   && x.Op != OpSB -> (LEAL8 [c+d] {s} x y)
585(LEAL8 [c] {s} x (ADDLconst [d] y)) && is32Bit(c+8*d) && y.Op != OpSB -> (LEAL8 [c+8*d] {s} x y)
586
587// fold shifts into LEALx
588(LEAL1 [c] {s} x (SHLLconst [1] y)) -> (LEAL2 [c] {s} x y)
589(LEAL1 [c] {s} x (SHLLconst [2] y)) -> (LEAL4 [c] {s} x y)
590(LEAL1 [c] {s} x (SHLLconst [3] y)) -> (LEAL8 [c] {s} x y)
591(LEAL2 [c] {s} x (SHLLconst [1] y)) -> (LEAL4 [c] {s} x y)
592(LEAL2 [c] {s} x (SHLLconst [2] y)) -> (LEAL8 [c] {s} x y)
593(LEAL4 [c] {s} x (SHLLconst [1] y)) -> (LEAL8 [c] {s} x y)
594
595// reverse ordering of compare instruction
596(SETL (InvertFlags x)) -> (SETG x)
597(SETG (InvertFlags x)) -> (SETL x)
598(SETB (InvertFlags x)) -> (SETA x)
599(SETA (InvertFlags x)) -> (SETB x)
600(SETLE (InvertFlags x)) -> (SETGE x)
601(SETGE (InvertFlags x)) -> (SETLE x)
602(SETBE (InvertFlags x)) -> (SETAE x)
603(SETAE (InvertFlags x)) -> (SETBE x)
604(SETEQ (InvertFlags x)) -> (SETEQ x)
605(SETNE (InvertFlags x)) -> (SETNE x)
606
607// sign extended loads
608// Note: The combined instruction must end up in the same block
609// as the original load. If not, we end up making a value with
610// memory type live in two different blocks, which can lead to
611// multiple memory values alive simultaneously.
612// Make sure we don't combine these ops if the load has another use.
613// This prevents a single load from being split into multiple loads
614// which then might return different values.  See test/atomicload.go.
615(MOVBLSX x:(MOVBload [off] {sym} ptr mem)) && x.Uses == 1 && clobber(x) -> @x.Block (MOVBLSXload <v.Type> [off] {sym} ptr mem)
616(MOVBLZX x:(MOVBload [off] {sym} ptr mem)) && x.Uses == 1 && clobber(x) -> @x.Block (MOVBload <v.Type> [off] {sym} ptr mem)
617(MOVWLSX x:(MOVWload [off] {sym} ptr mem)) && x.Uses == 1 && clobber(x) -> @x.Block (MOVWLSXload <v.Type> [off] {sym} ptr mem)
618(MOVWLZX x:(MOVWload [off] {sym} ptr mem)) && x.Uses == 1 && clobber(x) -> @x.Block (MOVWload <v.Type> [off] {sym} ptr mem)
619
620(MOVBLZX x:(MOVBloadidx1 [off] {sym} ptr idx mem)) && x.Uses == 1 && clobber(x) -> @x.Block (MOVBloadidx1 <v.Type> [off] {sym} ptr idx mem)
621(MOVWLZX x:(MOVWloadidx1 [off] {sym} ptr idx mem)) && x.Uses == 1 && clobber(x) -> @x.Block (MOVWloadidx1 <v.Type> [off] {sym} ptr idx mem)
622(MOVWLZX x:(MOVWloadidx2 [off] {sym} ptr idx mem)) && x.Uses == 1 && clobber(x) -> @x.Block (MOVWloadidx2 <v.Type> [off] {sym} ptr idx mem)
623
624// replace load from same location as preceding store with zero/sign extension (or copy in case of full width)
625(MOVBload [off] {sym} ptr (MOVBstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && isSamePtr(ptr, ptr2) -> (MOVBLZX x)
626(MOVWload [off] {sym} ptr (MOVWstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && isSamePtr(ptr, ptr2) -> (MOVWLZX x)
627(MOVLload [off] {sym} ptr (MOVLstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && isSamePtr(ptr, ptr2) -> x
628(MOVBLSXload [off] {sym} ptr (MOVBstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && isSamePtr(ptr, ptr2) -> (MOVBLSX x)
629(MOVWLSXload [off] {sym} ptr (MOVWstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && isSamePtr(ptr, ptr2) -> (MOVWLSX x)
630
631// Fold extensions and ANDs together.
632(MOVBLZX (ANDLconst [c] x)) -> (ANDLconst [c & 0xff] x)
633(MOVWLZX (ANDLconst [c] x)) -> (ANDLconst [c & 0xffff] x)
634(MOVBLSX (ANDLconst [c] x)) && c & 0x80 == 0 -> (ANDLconst [c & 0x7f] x)
635(MOVWLSX (ANDLconst [c] x)) && c & 0x8000 == 0 -> (ANDLconst [c & 0x7fff] x)
636
637// Don't extend before storing
638(MOVWstore [off] {sym} ptr (MOVWL(S|Z)X x) mem) -> (MOVWstore [off] {sym} ptr x mem)
639(MOVBstore [off] {sym} ptr (MOVBL(S|Z)X x) mem) -> (MOVBstore [off] {sym} ptr x mem)
640
641// fold constants into memory operations
642// Note that this is not always a good idea because if not all the uses of
643// the ADDQconst get eliminated, we still have to compute the ADDQconst and we now
644// have potentially two live values (ptr and (ADDQconst [off] ptr)) instead of one.
645// Nevertheless, let's do it!
646(MOV(L|W|B|SS|SD)load  [off1] {sym} (ADDLconst [off2] ptr) mem) && is32Bit(off1+off2) -> (MOV(L|W|B|SS|SD)load  [off1+off2] {sym} ptr mem)
647(MOV(L|W|B|SS|SD)store  [off1] {sym} (ADDLconst [off2] ptr) val mem) && is32Bit(off1+off2) -> (MOV(L|W|B|SS|SD)store  [off1+off2] {sym} ptr val mem)
648
649((ADD|SUB|MUL|AND|OR|XOR)Lload [off1] {sym} val (ADDLconst [off2] base) mem) && is32Bit(off1+off2) ->
650	((ADD|SUB|MUL|AND|OR|XOR)Lload [off1+off2] {sym} val base mem)
651((ADD|SUB|MUL|AND|OR|XOR)Lloadidx4 [off1] {sym} val (ADDLconst [off2] base) idx mem) && is32Bit(off1+off2) ->
652	((ADD|SUB|MUL|AND|OR|XOR)Lloadidx4 [off1+off2] {sym} val base idx mem)
653((ADD|SUB|MUL|AND|OR|XOR)Lloadidx4 [off1] {sym} val base (ADDLconst [off2] idx) mem) && is32Bit(off1+off2*4) ->
654	((ADD|SUB|MUL|AND|OR|XOR)Lloadidx4 [off1+off2*4] {sym} val base idx mem)
655((ADD|SUB|MUL|DIV)SSload [off1] {sym} val (ADDLconst [off2] base) mem) && is32Bit(off1+off2) ->
656	((ADD|SUB|MUL|DIV)SSload [off1+off2] {sym} val base mem)
657((ADD|SUB|MUL|DIV)SDload [off1] {sym} val (ADDLconst [off2] base) mem) && is32Bit(off1+off2) ->
658	((ADD|SUB|MUL|DIV)SDload [off1+off2] {sym} val base mem)
659((ADD|SUB|AND|OR|XOR)Lmodify [off1] {sym} (ADDLconst [off2] base) val mem) && is32Bit(off1+off2) ->
660	((ADD|SUB|AND|OR|XOR)Lmodify [off1+off2] {sym} base val mem)
661((ADD|SUB|AND|OR|XOR)Lmodifyidx4 [off1] {sym} (ADDLconst [off2] base) idx val mem) && is32Bit(off1+off2) ->
662	((ADD|SUB|AND|OR|XOR)Lmodifyidx4 [off1+off2] {sym} base idx val mem)
663((ADD|SUB|AND|OR|XOR)Lmodifyidx4 [off1] {sym} base (ADDLconst [off2] idx) val mem) && is32Bit(off1+off2*4) ->
664	((ADD|SUB|AND|OR|XOR)Lmodifyidx4 [off1+off2*4] {sym} base idx val mem)
665((ADD|AND|OR|XOR)Lconstmodify [valoff1] {sym} (ADDLconst [off2] base) mem) && ValAndOff(valoff1).canAdd(off2) ->
666	((ADD|AND|OR|XOR)Lconstmodify [ValAndOff(valoff1).add(off2)] {sym} base mem)
667((ADD|AND|OR|XOR)Lconstmodifyidx4 [valoff1] {sym} (ADDLconst [off2] base) idx mem) && ValAndOff(valoff1).canAdd(off2) ->
668	((ADD|AND|OR|XOR)Lconstmodifyidx4 [ValAndOff(valoff1).add(off2)] {sym} base idx mem)
669((ADD|AND|OR|XOR)Lconstmodifyidx4 [valoff1] {sym} base (ADDLconst [off2] idx) mem) && ValAndOff(valoff1).canAdd(off2*4) ->
670	((ADD|AND|OR|XOR)Lconstmodifyidx4 [ValAndOff(valoff1).add(off2*4)] {sym} base idx mem)
671
672// Fold constants into stores.
673(MOVLstore [off] {sym} ptr (MOVLconst [c]) mem) && validOff(off) ->
674	(MOVLstoreconst [makeValAndOff(int64(int32(c)),off)] {sym} ptr mem)
675(MOVWstore [off] {sym} ptr (MOVLconst [c]) mem) && validOff(off) ->
676	(MOVWstoreconst [makeValAndOff(int64(int16(c)),off)] {sym} ptr mem)
677(MOVBstore [off] {sym} ptr (MOVLconst [c]) mem) && validOff(off) ->
678	(MOVBstoreconst [makeValAndOff(int64(int8(c)),off)] {sym} ptr mem)
679
680// Fold address offsets into constant stores.
681(MOV(L|W|B)storeconst [sc] {s} (ADDLconst [off] ptr) mem) && ValAndOff(sc).canAdd(off) ->
682	(MOV(L|W|B)storeconst [ValAndOff(sc).add(off)] {s} ptr mem)
683
684// We need to fold LEAQ into the MOVx ops so that the live variable analysis knows
685// what variables are being read/written by the ops.
686// Note: we turn off this merging for operations on globals when building
687// position-independent code (when Flag_shared is set).
688// PIC needs a spare register to load the PC into.  Having the LEAL be
689// a separate instruction gives us that register.  Having the LEAL be
690// a separate instruction also allows it to be CSEd (which is good because
691// it compiles to a thunk call).
692(MOV(L|W|B|SS|SD|BLSX|WLSX)load  [off1] {sym1} (LEAL [off2] {sym2} base) mem) && is32Bit(off1+off2) && canMergeSym(sym1, sym2)
693  && (base.Op != OpSB || !config.ctxt.Flag_shared) ->
694        (MOV(L|W|B|SS|SD|BLSX|WLSX)load  [off1+off2] {mergeSym(sym1,sym2)} base mem)
695
696(MOV(L|W|B|SS|SD)store  [off1] {sym1} (LEAL [off2] {sym2} base) val mem) && is32Bit(off1+off2) && canMergeSym(sym1, sym2)
697  && (base.Op != OpSB || !config.ctxt.Flag_shared) ->
698	(MOV(L|W|B|SS|SD)store  [off1+off2] {mergeSym(sym1,sym2)} base val mem)
699
700(MOV(L|W|B)storeconst [sc] {sym1} (LEAL [off] {sym2} ptr) mem) && canMergeSym(sym1, sym2) && ValAndOff(sc).canAdd(off)
701  && (ptr.Op != OpSB || !config.ctxt.Flag_shared) ->
702	(MOV(L|W|B)storeconst [ValAndOff(sc).add(off)] {mergeSym(sym1, sym2)} ptr mem)
703
704// generating indexed loads and stores
705(MOV(B|W|L|SS|SD)load [off1] {sym1} (LEAL1 [off2] {sym2} ptr idx) mem) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
706	(MOV(B|W|L|SS|SD)loadidx1 [off1+off2] {mergeSym(sym1,sym2)} ptr idx mem)
707(MOVWload [off1] {sym1} (LEAL2 [off2] {sym2} ptr idx) mem) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
708	(MOVWloadidx2 [off1+off2] {mergeSym(sym1,sym2)} ptr idx mem)
709(MOV(L|SS)load [off1] {sym1} (LEAL4 [off2] {sym2} ptr idx) mem) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
710	(MOV(L|SS)loadidx4 [off1+off2] {mergeSym(sym1,sym2)} ptr idx mem)
711(MOVSDload [off1] {sym1} (LEAL8 [off2] {sym2} ptr idx) mem) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
712	(MOVSDloadidx8 [off1+off2] {mergeSym(sym1,sym2)} ptr idx mem)
713
714(MOV(B|W|L|SS|SD)store [off1] {sym1} (LEAL1 [off2] {sym2} ptr idx) val mem) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
715	(MOV(B|W|L|SS|SD)storeidx1 [off1+off2] {mergeSym(sym1,sym2)} ptr idx val mem)
716(MOVWstore [off1] {sym1} (LEAL2 [off2] {sym2} ptr idx) val mem) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
717	(MOVWstoreidx2 [off1+off2] {mergeSym(sym1,sym2)} ptr idx val mem)
718(MOV(L|SS)store [off1] {sym1} (LEAL4 [off2] {sym2} ptr idx) val mem) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
719	(MOV(L|SS)storeidx4 [off1+off2] {mergeSym(sym1,sym2)} ptr idx val mem)
720(MOVSDstore [off1] {sym1} (LEAL8 [off2] {sym2} ptr idx) val mem) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
721	(MOVSDstoreidx8 [off1+off2] {mergeSym(sym1,sym2)} ptr idx val mem)
722
723((ADD|SUB|MUL|AND|OR|XOR)Lload [off1] {sym1} val (LEAL [off2] {sym2} base) mem)
724	&& is32Bit(off1+off2) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) ->
725	((ADD|SUB|MUL|AND|OR|XOR)Lload [off1+off2] {mergeSym(sym1,sym2)} val base mem)
726((ADD|SUB|MUL|AND|OR|XOR)Lloadidx4 [off1] {sym1} val (LEAL [off2] {sym2} base) idx mem)
727	&& is32Bit(off1+off2) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) ->
728	((ADD|SUB|MUL|AND|OR|XOR)Lloadidx4 [off1+off2] {mergeSym(sym1,sym2)} val base idx mem)
729((ADD|SUB|MUL|DIV)SSload [off1] {sym1} val (LEAL [off2] {sym2} base) mem)
730	&& is32Bit(off1+off2) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) ->
731	((ADD|SUB|MUL|DIV)SSload [off1+off2] {mergeSym(sym1,sym2)} val base mem)
732((ADD|SUB|MUL|DIV)SDload [off1] {sym1} val (LEAL [off2] {sym2} base) mem)
733	&& is32Bit(off1+off2) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) ->
734	((ADD|SUB|MUL|DIV)SDload [off1+off2] {mergeSym(sym1,sym2)} val base mem)
735((ADD|SUB|AND|OR|XOR)Lmodify [off1] {sym1} (LEAL [off2] {sym2} base) val mem)
736	&& is32Bit(off1+off2) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) ->
737	((ADD|SUB|AND|OR|XOR)Lmodify [off1+off2] {mergeSym(sym1,sym2)} base val mem)
738((ADD|SUB|AND|OR|XOR)Lmodifyidx4 [off1] {sym1} (LEAL [off2] {sym2} base) idx val mem)
739	&& is32Bit(off1+off2) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) ->
740	((ADD|SUB|AND|OR|XOR)Lmodifyidx4 [off1+off2] {mergeSym(sym1,sym2)} base idx val mem)
741((ADD|AND|OR|XOR)Lconstmodify [valoff1] {sym1} (LEAL [off2] {sym2} base) mem)
742	&& ValAndOff(valoff1).canAdd(off2) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) ->
743	((ADD|AND|OR|XOR)Lconstmodify [ValAndOff(valoff1).add(off2)] {mergeSym(sym1,sym2)} base mem)
744((ADD|AND|OR|XOR)Lconstmodifyidx4 [valoff1] {sym1} (LEAL [off2] {sym2} base) idx mem)
745	&& ValAndOff(valoff1).canAdd(off2) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) ->
746	((ADD|AND|OR|XOR)Lconstmodifyidx4 [ValAndOff(valoff1).add(off2)] {mergeSym(sym1,sym2)} base idx mem)
747
748(MOV(B|W|L|SS|SD)load [off] {sym} (ADDL ptr idx) mem) && ptr.Op != OpSB -> (MOV(B|W|L|SS|SD)loadidx1 [off] {sym} ptr idx mem)
749(MOV(B|W|L|SS|SD)store [off] {sym} (ADDL ptr idx) val mem) && ptr.Op != OpSB -> (MOV(B|W|L|SS|SD)storeidx1 [off] {sym} ptr idx val mem)
750
751(MOV(B|W|L)storeconst [x] {sym1} (LEAL1 [off] {sym2} ptr idx) mem) && canMergeSym(sym1, sym2) ->
752	(MOV(B|W|L)storeconstidx1 [ValAndOff(x).add(off)] {mergeSym(sym1,sym2)} ptr idx mem)
753(MOVWstoreconst [x] {sym1} (LEAL2 [off] {sym2} ptr idx) mem) && canMergeSym(sym1, sym2) ->
754	(MOVWstoreconstidx2 [ValAndOff(x).add(off)] {mergeSym(sym1,sym2)} ptr idx mem)
755(MOVLstoreconst [x] {sym1} (LEAL4 [off] {sym2} ptr idx) mem) && canMergeSym(sym1, sym2) ->
756	(MOVLstoreconstidx4 [ValAndOff(x).add(off)] {mergeSym(sym1,sym2)} ptr idx mem)
757
758(MOV(B|W|L)storeconst [x] {sym} (ADDL ptr idx) mem) -> (MOV(B|W|L)storeconstidx1 [x] {sym} ptr idx mem)
759
760// combine SHLL into indexed loads and stores
761(MOVWloadidx1 [c] {sym} ptr (SHLLconst [1] idx) mem) -> (MOVWloadidx2 [c] {sym} ptr idx mem)
762(MOVLloadidx1 [c] {sym} ptr (SHLLconst [2] idx) mem) -> (MOVLloadidx4 [c] {sym} ptr idx mem)
763(MOVWstoreidx1 [c] {sym} ptr (SHLLconst [1] idx) val mem) -> (MOVWstoreidx2 [c] {sym} ptr idx val mem)
764(MOVLstoreidx1 [c] {sym} ptr (SHLLconst [2] idx) val mem) -> (MOVLstoreidx4 [c] {sym} ptr idx val mem)
765(MOVWstoreconstidx1 [c] {sym} ptr (SHLLconst [1] idx) mem) -> (MOVWstoreconstidx2 [c] {sym} ptr idx mem)
766(MOVLstoreconstidx1 [c] {sym} ptr (SHLLconst [2] idx) mem) -> (MOVLstoreconstidx4 [c] {sym} ptr idx mem)
767
768// combine ADDL into indexed loads and stores
769(MOV(B|W|L|SS|SD)loadidx1 [c] {sym} (ADDLconst [d] ptr) idx mem) -> (MOV(B|W|L|SS|SD)loadidx1 [int64(int32(c+d))] {sym} ptr idx mem)
770(MOVWloadidx2 [c] {sym} (ADDLconst [d] ptr) idx mem) -> (MOVWloadidx2 [int64(int32(c+d))] {sym} ptr idx mem)
771(MOV(L|SS)loadidx4 [c] {sym} (ADDLconst [d] ptr) idx mem) -> (MOV(L|SS)loadidx4 [int64(int32(c+d))] {sym} ptr idx mem)
772(MOVSDloadidx8 [c] {sym} (ADDLconst [d] ptr) idx mem) -> (MOVSDloadidx8 [int64(int32(c+d))] {sym} ptr idx mem)
773
774(MOV(B|W|L|SS|SD)storeidx1 [c] {sym} (ADDLconst [d] ptr) idx val mem) -> (MOV(B|W|L|SS|SD)storeidx1 [int64(int32(c+d))] {sym} ptr idx val mem)
775(MOVWstoreidx2 [c] {sym} (ADDLconst [d] ptr) idx val mem) -> (MOVWstoreidx2 [int64(int32(c+d))] {sym} ptr idx val mem)
776(MOV(L|SS)storeidx4 [c] {sym} (ADDLconst [d] ptr) idx val mem) -> (MOV(L|SS)storeidx4 [int64(int32(c+d))] {sym} ptr idx val mem)
777(MOVSDstoreidx8 [c] {sym} (ADDLconst [d] ptr) idx val mem) -> (MOVSDstoreidx8 [int64(int32(c+d))] {sym} ptr idx val mem)
778
779(MOV(B|W|L|SS|SD)loadidx1 [c] {sym} ptr (ADDLconst [d] idx) mem) -> (MOV(B|W|L|SS|SD)loadidx1 [int64(int32(c+d))] {sym} ptr idx mem)
780(MOVWloadidx2 [c] {sym} ptr (ADDLconst [d] idx) mem) -> (MOVWloadidx2 [int64(int32(c+2*d))] {sym} ptr idx mem)
781(MOV(L|SS)loadidx4 [c] {sym} ptr (ADDLconst [d] idx) mem) -> (MOV(L|SS)loadidx4 [int64(int32(c+4*d))] {sym} ptr idx mem)
782(MOVSDloadidx8 [c] {sym} ptr (ADDLconst [d] idx) mem) -> (MOVSDloadidx8 [int64(int32(c+8*d))] {sym} ptr idx mem)
783
784(MOV(B|W|L|SS|SD)storeidx1 [c] {sym} ptr (ADDLconst [d] idx) val mem) -> (MOV(B|W|L|SS|SD)storeidx1  [int64(int32(c+d))]   {sym} ptr idx val mem)
785(MOVWstoreidx2 [c] {sym} ptr (ADDLconst [d] idx) val mem) -> (MOVWstoreidx2  [int64(int32(c+2*d))] {sym} ptr idx val mem)
786(MOV(L|SS)storeidx4 [c] {sym} ptr (ADDLconst [d] idx) val mem) -> (MOV(L|SS)storeidx4 [int64(int32(c+4*d))] {sym} ptr idx val mem)
787(MOVSDstoreidx8 [c] {sym} ptr (ADDLconst [d] idx) val mem) -> (MOVSDstoreidx8 [int64(int32(c+8*d))] {sym} ptr idx val mem)
788
789// Merge load/store to op
790((ADD|AND|OR|XOR|SUB|MUL)L x l:(MOVLload [off] {sym} ptr mem)) && canMergeLoadClobber(v, l, x) && clobber(l) -> ((ADD|AND|OR|XOR|SUB|MUL)Lload x [off] {sym} ptr mem)
791((ADD|AND|OR|XOR|SUB|MUL)L x l:(MOVLloadidx4 [off] {sym} ptr idx mem)) && canMergeLoadClobber(v, l, x) && clobber(l) ->
792	((ADD|AND|OR|XOR|SUB|MUL)Lloadidx4 x [off] {sym} ptr idx mem)
793((ADD|SUB|MUL|AND|OR|XOR)Lload [off1] {sym1} val (LEAL4 [off2] {sym2} ptr idx) mem)
794	&& is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
795	((ADD|SUB|MUL|AND|OR|XOR)Lloadidx4 [off1+off2] {mergeSym(sym1,sym2)} val ptr idx mem)
796((ADD|SUB|MUL|DIV)SD x l:(MOVSDload [off] {sym} ptr mem)) && canMergeLoadClobber(v, l, x) && !config.use387 && clobber(l) -> ((ADD|SUB|MUL|DIV)SDload x [off] {sym} ptr mem)
797((ADD|SUB|MUL|DIV)SS x l:(MOVSSload [off] {sym} ptr mem)) && canMergeLoadClobber(v, l, x) && !config.use387 && clobber(l) -> ((ADD|SUB|MUL|DIV)SSload x [off] {sym} ptr mem)
798(MOVLstore {sym} [off] ptr y:((ADD|AND|OR|XOR)Lload x [off] {sym} ptr mem) mem) && y.Uses==1 && clobber(y) -> ((ADD|AND|OR|XOR)Lmodify [off] {sym} ptr x mem)
799(MOVLstore {sym} [off] ptr y:((ADD|SUB|AND|OR|XOR)L l:(MOVLload [off] {sym} ptr mem) x) mem) && y.Uses==1 && l.Uses==1 && clobber(y) && clobber(l) ->
800	((ADD|SUB|AND|OR|XOR)Lmodify [off] {sym} ptr x mem)
801(MOVLstoreidx4 {sym} [off] ptr idx y:((ADD|AND|OR|XOR)Lloadidx4 x [off] {sym} ptr idx mem) mem) && y.Uses==1 && clobber(y) ->
802	((ADD|AND|OR|XOR)Lmodifyidx4 [off] {sym} ptr idx x mem)
803(MOVLstoreidx4 {sym} [off] ptr idx y:((ADD|SUB|AND|OR|XOR)L l:(MOVLloadidx4 [off] {sym} ptr idx mem) x) mem) && y.Uses==1 && l.Uses==1 && clobber(y) && clobber(l) ->
804	((ADD|SUB|AND|OR|XOR)Lmodifyidx4 [off] {sym} ptr idx x mem)
805(MOVLstore {sym} [off] ptr y:((ADD|AND|OR|XOR)Lconst [c] l:(MOVLload [off] {sym} ptr mem)) mem)
806	&& y.Uses==1 && l.Uses==1 && clobber(y) && clobber(l) && validValAndOff(c,off) ->
807	((ADD|AND|OR|XOR)Lconstmodify [makeValAndOff(c,off)] {sym} ptr mem)
808(MOVLstoreidx4 {sym} [off] ptr idx y:((ADD|AND|OR|XOR)Lconst [c] l:(MOVLloadidx4 [off] {sym} ptr idx mem)) mem)
809	&& y.Uses==1 && l.Uses==1 && clobber(y) && clobber(l) && validValAndOff(c,off) ->
810	((ADD|AND|OR|XOR)Lconstmodifyidx4 [makeValAndOff(c,off)] {sym} ptr idx mem)
811((ADD|AND|OR|XOR)Lmodifyidx4 [off] {sym} ptr idx (MOVLconst [c]) mem) && validValAndOff(c,off) ->
812	((ADD|AND|OR|XOR)Lconstmodifyidx4 [makeValAndOff(c,off)] {sym} ptr idx mem)
813(SUBLmodifyidx4 [off] {sym} ptr idx (MOVLconst [c]) mem) && validValAndOff(-c,off) ->
814	(ADDLconstmodifyidx4 [makeValAndOff(-c,off)] {sym} ptr idx mem)
815
816(MOV(B|W|L)storeconstidx1 [x] {sym} (ADDLconst [c] ptr) idx mem) ->
817	(MOV(B|W|L)storeconstidx1 [ValAndOff(x).add(c)] {sym} ptr idx mem)
818(MOVWstoreconstidx2 [x] {sym} (ADDLconst [c] ptr) idx mem) ->
819	(MOVWstoreconstidx2 [ValAndOff(x).add(c)] {sym} ptr idx mem)
820(MOVLstoreconstidx4 [x] {sym} (ADDLconst [c] ptr) idx mem) ->
821	(MOVLstoreconstidx4 [ValAndOff(x).add(c)] {sym} ptr idx mem)
822
823(MOV(B|W|L)storeconstidx1 [x] {sym} ptr (ADDLconst [c] idx) mem) ->
824	(MOV(B|W|L)storeconstidx1 [ValAndOff(x).add(c)] {sym} ptr idx mem)
825(MOVWstoreconstidx2 [x] {sym} ptr (ADDLconst [c] idx) mem) ->
826	(MOVWstoreconstidx2 [ValAndOff(x).add(2*c)] {sym} ptr idx mem)
827(MOVLstoreconstidx4 [x] {sym} ptr (ADDLconst [c] idx) mem) ->
828	(MOVLstoreconstidx4 [ValAndOff(x).add(4*c)] {sym} ptr idx mem)
829
830// fold LEALs together
831(LEAL [off1] {sym1} (LEAL [off2] {sym2} x)) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
832      (LEAL [off1+off2] {mergeSym(sym1,sym2)} x)
833
834// LEAL into LEAL1
835(LEAL1 [off1] {sym1} (LEAL [off2] {sym2} x) y) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) && x.Op != OpSB ->
836       (LEAL1 [off1+off2] {mergeSym(sym1,sym2)} x y)
837
838// LEAL1 into LEAL
839(LEAL [off1] {sym1} (LEAL1 [off2] {sym2} x y)) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
840       (LEAL1 [off1+off2] {mergeSym(sym1,sym2)} x y)
841
842// LEAL into LEAL[248]
843(LEAL2 [off1] {sym1} (LEAL [off2] {sym2} x) y) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) && x.Op != OpSB ->
844       (LEAL2 [off1+off2] {mergeSym(sym1,sym2)} x y)
845(LEAL4 [off1] {sym1} (LEAL [off2] {sym2} x) y) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) && x.Op != OpSB ->
846       (LEAL4 [off1+off2] {mergeSym(sym1,sym2)} x y)
847(LEAL8 [off1] {sym1} (LEAL [off2] {sym2} x) y) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) && x.Op != OpSB ->
848       (LEAL8 [off1+off2] {mergeSym(sym1,sym2)} x y)
849
850// LEAL[248] into LEAL
851(LEAL [off1] {sym1} (LEAL2 [off2] {sym2} x y)) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
852      (LEAL2 [off1+off2] {mergeSym(sym1,sym2)} x y)
853(LEAL [off1] {sym1} (LEAL4 [off2] {sym2} x y)) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
854      (LEAL4 [off1+off2] {mergeSym(sym1,sym2)} x y)
855(LEAL [off1] {sym1} (LEAL8 [off2] {sym2} x y)) && is32Bit(off1+off2) && canMergeSym(sym1, sym2) ->
856      (LEAL8 [off1+off2] {mergeSym(sym1,sym2)} x y)
857
858// Absorb InvertFlags into branches.
859(LT (InvertFlags cmp) yes no) -> (GT cmp yes no)
860(GT (InvertFlags cmp) yes no) -> (LT cmp yes no)
861(LE (InvertFlags cmp) yes no) -> (GE cmp yes no)
862(GE (InvertFlags cmp) yes no) -> (LE cmp yes no)
863(ULT (InvertFlags cmp) yes no) -> (UGT cmp yes no)
864(UGT (InvertFlags cmp) yes no) -> (ULT cmp yes no)
865(ULE (InvertFlags cmp) yes no) -> (UGE cmp yes no)
866(UGE (InvertFlags cmp) yes no) -> (ULE cmp yes no)
867(EQ (InvertFlags cmp) yes no) -> (EQ cmp yes no)
868(NE (InvertFlags cmp) yes no) -> (NE cmp yes no)
869
870// Constant comparisons.
871(CMPLconst (MOVLconst [x]) [y]) && int32(x)==int32(y) -> (FlagEQ)
872(CMPLconst (MOVLconst [x]) [y]) && int32(x)<int32(y) && uint32(x)<uint32(y) -> (FlagLT_ULT)
873(CMPLconst (MOVLconst [x]) [y]) && int32(x)<int32(y) && uint32(x)>uint32(y) -> (FlagLT_UGT)
874(CMPLconst (MOVLconst [x]) [y]) && int32(x)>int32(y) && uint32(x)<uint32(y) -> (FlagGT_ULT)
875(CMPLconst (MOVLconst [x]) [y]) && int32(x)>int32(y) && uint32(x)>uint32(y) -> (FlagGT_UGT)
876(CMPWconst (MOVLconst [x]) [y]) && int16(x)==int16(y) -> (FlagEQ)
877(CMPWconst (MOVLconst [x]) [y]) && int16(x)<int16(y) && uint16(x)<uint16(y) -> (FlagLT_ULT)
878(CMPWconst (MOVLconst [x]) [y]) && int16(x)<int16(y) && uint16(x)>uint16(y) -> (FlagLT_UGT)
879(CMPWconst (MOVLconst [x]) [y]) && int16(x)>int16(y) && uint16(x)<uint16(y) -> (FlagGT_ULT)
880(CMPWconst (MOVLconst [x]) [y]) && int16(x)>int16(y) && uint16(x)>uint16(y) -> (FlagGT_UGT)
881(CMPBconst (MOVLconst [x]) [y]) && int8(x)==int8(y) -> (FlagEQ)
882(CMPBconst (MOVLconst [x]) [y]) && int8(x)<int8(y) && uint8(x)<uint8(y) -> (FlagLT_ULT)
883(CMPBconst (MOVLconst [x]) [y]) && int8(x)<int8(y) && uint8(x)>uint8(y) -> (FlagLT_UGT)
884(CMPBconst (MOVLconst [x]) [y]) && int8(x)>int8(y) && uint8(x)<uint8(y) -> (FlagGT_ULT)
885(CMPBconst (MOVLconst [x]) [y]) && int8(x)>int8(y) && uint8(x)>uint8(y) -> (FlagGT_UGT)
886
887// Other known comparisons.
888(CMPLconst (SHRLconst _ [c]) [n]) && 0 <= n && 0 < c && c <= 32 && (1<<uint64(32-c)) <= uint64(n) -> (FlagLT_ULT)
889(CMPLconst (ANDLconst _ [m]) [n]) && 0 <= int32(m) && int32(m) < int32(n) -> (FlagLT_ULT)
890(CMPWconst (ANDLconst _ [m]) [n]) && 0 <= int16(m) && int16(m) < int16(n) -> (FlagLT_ULT)
891(CMPBconst (ANDLconst _ [m]) [n]) && 0 <= int8(m) && int8(m) < int8(n) -> (FlagLT_ULT)
892// TODO: DIVxU also.
893
894// Absorb flag constants into SBB ops.
895(SBBLcarrymask (FlagEQ)) -> (MOVLconst [0])
896(SBBLcarrymask (FlagLT_ULT)) -> (MOVLconst [-1])
897(SBBLcarrymask (FlagLT_UGT)) -> (MOVLconst [0])
898(SBBLcarrymask (FlagGT_ULT)) -> (MOVLconst [-1])
899(SBBLcarrymask (FlagGT_UGT)) -> (MOVLconst [0])
900
901// Absorb flag constants into branches.
902(EQ (FlagEQ) yes no) -> (First yes no)
903(EQ (FlagLT_ULT) yes no) -> (First no yes)
904(EQ (FlagLT_UGT) yes no) -> (First no yes)
905(EQ (FlagGT_ULT) yes no) -> (First no yes)
906(EQ (FlagGT_UGT) yes no) -> (First no yes)
907
908(NE (FlagEQ) yes no) -> (First no yes)
909(NE (FlagLT_ULT) yes no) -> (First yes no)
910(NE (FlagLT_UGT) yes no) -> (First yes no)
911(NE (FlagGT_ULT) yes no) -> (First yes no)
912(NE (FlagGT_UGT) yes no) -> (First yes no)
913
914(LT (FlagEQ) yes no) -> (First no yes)
915(LT (FlagLT_ULT) yes no) -> (First yes no)
916(LT (FlagLT_UGT) yes no) -> (First yes no)
917(LT (FlagGT_ULT) yes no) -> (First no yes)
918(LT (FlagGT_UGT) yes no) -> (First no yes)
919
920(LE (FlagEQ) yes no) -> (First yes no)
921(LE (FlagLT_ULT) yes no) -> (First yes no)
922(LE (FlagLT_UGT) yes no) -> (First yes no)
923(LE (FlagGT_ULT) yes no) -> (First no yes)
924(LE (FlagGT_UGT) yes no) -> (First no yes)
925
926(GT (FlagEQ) yes no) -> (First no yes)
927(GT (FlagLT_ULT) yes no) -> (First no yes)
928(GT (FlagLT_UGT) yes no) -> (First no yes)
929(GT (FlagGT_ULT) yes no) -> (First yes no)
930(GT (FlagGT_UGT) yes no) -> (First yes no)
931
932(GE (FlagEQ) yes no) -> (First yes no)
933(GE (FlagLT_ULT) yes no) -> (First no yes)
934(GE (FlagLT_UGT) yes no) -> (First no yes)
935(GE (FlagGT_ULT) yes no) -> (First yes no)
936(GE (FlagGT_UGT) yes no) -> (First yes no)
937
938(ULT (FlagEQ) yes no) -> (First no yes)
939(ULT (FlagLT_ULT) yes no) -> (First yes no)
940(ULT (FlagLT_UGT) yes no) -> (First no yes)
941(ULT (FlagGT_ULT) yes no) -> (First yes no)
942(ULT (FlagGT_UGT) yes no) -> (First no yes)
943
944(ULE (FlagEQ) yes no) -> (First yes no)
945(ULE (FlagLT_ULT) yes no) -> (First yes no)
946(ULE (FlagLT_UGT) yes no) -> (First no yes)
947(ULE (FlagGT_ULT) yes no) -> (First yes no)
948(ULE (FlagGT_UGT) yes no) -> (First no yes)
949
950(UGT (FlagEQ) yes no) -> (First no yes)
951(UGT (FlagLT_ULT) yes no) -> (First no yes)
952(UGT (FlagLT_UGT) yes no) -> (First yes no)
953(UGT (FlagGT_ULT) yes no) -> (First no yes)
954(UGT (FlagGT_UGT) yes no) -> (First yes no)
955
956(UGE (FlagEQ) yes no) -> (First yes no)
957(UGE (FlagLT_ULT) yes no) -> (First no yes)
958(UGE (FlagLT_UGT) yes no) -> (First yes no)
959(UGE (FlagGT_ULT) yes no) -> (First no yes)
960(UGE (FlagGT_UGT) yes no) -> (First yes no)
961
962// Absorb flag constants into SETxx ops.
963(SETEQ (FlagEQ)) -> (MOVLconst [1])
964(SETEQ (FlagLT_ULT)) -> (MOVLconst [0])
965(SETEQ (FlagLT_UGT)) -> (MOVLconst [0])
966(SETEQ (FlagGT_ULT)) -> (MOVLconst [0])
967(SETEQ (FlagGT_UGT)) -> (MOVLconst [0])
968
969(SETNE (FlagEQ)) -> (MOVLconst [0])
970(SETNE (FlagLT_ULT)) -> (MOVLconst [1])
971(SETNE (FlagLT_UGT)) -> (MOVLconst [1])
972(SETNE (FlagGT_ULT)) -> (MOVLconst [1])
973(SETNE (FlagGT_UGT)) -> (MOVLconst [1])
974
975(SETL (FlagEQ)) -> (MOVLconst [0])
976(SETL (FlagLT_ULT)) -> (MOVLconst [1])
977(SETL (FlagLT_UGT)) -> (MOVLconst [1])
978(SETL (FlagGT_ULT)) -> (MOVLconst [0])
979(SETL (FlagGT_UGT)) -> (MOVLconst [0])
980
981(SETLE (FlagEQ)) -> (MOVLconst [1])
982(SETLE (FlagLT_ULT)) -> (MOVLconst [1])
983(SETLE (FlagLT_UGT)) -> (MOVLconst [1])
984(SETLE (FlagGT_ULT)) -> (MOVLconst [0])
985(SETLE (FlagGT_UGT)) -> (MOVLconst [0])
986
987(SETG (FlagEQ)) -> (MOVLconst [0])
988(SETG (FlagLT_ULT)) -> (MOVLconst [0])
989(SETG (FlagLT_UGT)) -> (MOVLconst [0])
990(SETG (FlagGT_ULT)) -> (MOVLconst [1])
991(SETG (FlagGT_UGT)) -> (MOVLconst [1])
992
993(SETGE (FlagEQ)) -> (MOVLconst [1])
994(SETGE (FlagLT_ULT)) -> (MOVLconst [0])
995(SETGE (FlagLT_UGT)) -> (MOVLconst [0])
996(SETGE (FlagGT_ULT)) -> (MOVLconst [1])
997(SETGE (FlagGT_UGT)) -> (MOVLconst [1])
998
999(SETB (FlagEQ)) -> (MOVLconst [0])
1000(SETB (FlagLT_ULT)) -> (MOVLconst [1])
1001(SETB (FlagLT_UGT)) -> (MOVLconst [0])
1002(SETB (FlagGT_ULT)) -> (MOVLconst [1])
1003(SETB (FlagGT_UGT)) -> (MOVLconst [0])
1004
1005(SETBE (FlagEQ)) -> (MOVLconst [1])
1006(SETBE (FlagLT_ULT)) -> (MOVLconst [1])
1007(SETBE (FlagLT_UGT)) -> (MOVLconst [0])
1008(SETBE (FlagGT_ULT)) -> (MOVLconst [1])
1009(SETBE (FlagGT_UGT)) -> (MOVLconst [0])
1010
1011(SETA (FlagEQ)) -> (MOVLconst [0])
1012(SETA (FlagLT_ULT)) -> (MOVLconst [0])
1013(SETA (FlagLT_UGT)) -> (MOVLconst [1])
1014(SETA (FlagGT_ULT)) -> (MOVLconst [0])
1015(SETA (FlagGT_UGT)) -> (MOVLconst [1])
1016
1017(SETAE (FlagEQ)) -> (MOVLconst [1])
1018(SETAE (FlagLT_ULT)) -> (MOVLconst [0])
1019(SETAE (FlagLT_UGT)) -> (MOVLconst [1])
1020(SETAE (FlagGT_ULT)) -> (MOVLconst [0])
1021(SETAE (FlagGT_UGT)) -> (MOVLconst [1])
1022
1023// Remove redundant *const ops
1024(ADDLconst [c] x) && int32(c)==0 -> x
1025(SUBLconst [c] x) && int32(c) == 0 -> x
1026(ANDLconst [c] _) && int32(c)==0  -> (MOVLconst [0])
1027(ANDLconst [c] x) && int32(c)==-1 -> x
1028(ORLconst [c] x) && int32(c)==0   -> x
1029(ORLconst [c] _) && int32(c)==-1  -> (MOVLconst [-1])
1030(XORLconst [c] x) && int32(c)==0   -> x
1031// TODO: since we got rid of the W/B versions, we might miss
1032// things like (ANDLconst [0x100] x) which were formerly
1033// (ANDBconst [0] x).  Probably doesn't happen very often.
1034// If we cared, we might do:
1035//  (ANDLconst <t> [c] x) && t.Size()==1 && int8(x)==0 -> (MOVLconst [0])
1036
1037// Convert constant subtracts to constant adds
1038(SUBLconst [c] x) -> (ADDLconst [int64(int32(-c))] x)
1039
1040// generic constant folding
1041// TODO: more of this
1042(ADDLconst [c] (MOVLconst [d])) -> (MOVLconst [int64(int32(c+d))])
1043(ADDLconst [c] (ADDLconst [d] x)) -> (ADDLconst [int64(int32(c+d))] x)
1044(SARLconst [c] (MOVLconst [d])) -> (MOVLconst [d>>uint64(c)])
1045(SARWconst [c] (MOVLconst [d])) -> (MOVLconst [d>>uint64(c)])
1046(SARBconst [c] (MOVLconst [d])) -> (MOVLconst [d>>uint64(c)])
1047(NEGL (MOVLconst [c])) -> (MOVLconst [int64(int32(-c))])
1048(MULLconst [c] (MOVLconst [d])) -> (MOVLconst [int64(int32(c*d))])
1049(ANDLconst [c] (MOVLconst [d])) -> (MOVLconst [c&d])
1050(ORLconst [c] (MOVLconst [d])) -> (MOVLconst [c|d])
1051(XORLconst [c] (MOVLconst [d])) -> (MOVLconst [c^d])
1052(NOTL (MOVLconst [c])) -> (MOVLconst [^c])
1053
1054// generic simplifications
1055// TODO: more of this
1056(ADDL x (NEGL y)) -> (SUBL x y)
1057(SUBL x x) -> (MOVLconst [0])
1058(ANDL x x) -> x
1059(ORL x x) -> x
1060(XORL x x) -> (MOVLconst [0])
1061
1062// checking AND against 0.
1063(CMP(L|W|B)const l:(ANDL x y) [0]) && l.Uses==1 -> (TEST(L|W|B) x y)
1064(CMPLconst l:(ANDLconst [c] x) [0]) && l.Uses==1 -> (TESTLconst [c] x)
1065(CMPWconst l:(ANDLconst [c] x) [0]) && l.Uses==1 -> (TESTWconst [int64(int16(c))] x)
1066(CMPBconst l:(ANDLconst [c] x) [0]) && l.Uses==1 -> (TESTBconst [int64(int8(c))] x)
1067
1068// TEST %reg,%reg is shorter than CMP
1069(CMP(L|W|B)const x [0]) -> (TEST(L|W|B) x x)
1070
1071// Combining byte loads into larger (unaligned) loads.
1072// There are many ways these combinations could occur.  This is
1073// designed to match the way encoding/binary.LittleEndian does it.
1074(ORL                  x0:(MOVBload [i0] {s} p mem)
1075    s0:(SHLLconst [8] x1:(MOVBload [i1] {s} p mem)))
1076  && i1 == i0+1
1077  && x0.Uses == 1
1078  && x1.Uses == 1
1079  && s0.Uses == 1
1080  && mergePoint(b,x0,x1) != nil
1081  && clobber(x0)
1082  && clobber(x1)
1083  && clobber(s0)
1084  -> @mergePoint(b,x0,x1) (MOVWload [i0] {s} p mem)
1085
1086(ORL o0:(ORL
1087                       x0:(MOVWload [i0] {s} p mem)
1088    s0:(SHLLconst [16] x1:(MOVBload [i2] {s} p mem)))
1089    s1:(SHLLconst [24] x2:(MOVBload [i3] {s} p mem)))
1090  && i2 == i0+2
1091  && i3 == i0+3
1092  && x0.Uses == 1
1093  && x1.Uses == 1
1094  && x2.Uses == 1
1095  && s0.Uses == 1
1096  && s1.Uses == 1
1097  && o0.Uses == 1
1098  && mergePoint(b,x0,x1,x2) != nil
1099  && clobber(x0)
1100  && clobber(x1)
1101  && clobber(x2)
1102  && clobber(s0)
1103  && clobber(s1)
1104  && clobber(o0)
1105  -> @mergePoint(b,x0,x1,x2) (MOVLload [i0] {s} p mem)
1106
1107(ORL                  x0:(MOVBloadidx1 [i0] {s} p idx mem)
1108    s0:(SHLLconst [8] x1:(MOVBloadidx1 [i1] {s} p idx mem)))
1109  && i1==i0+1
1110  && x0.Uses == 1
1111  && x1.Uses == 1
1112  && s0.Uses == 1
1113  && mergePoint(b,x0,x1) != nil
1114  && clobber(x0)
1115  && clobber(x1)
1116  && clobber(s0)
1117  -> @mergePoint(b,x0,x1) (MOVWloadidx1 <v.Type> [i0] {s} p idx mem)
1118
1119(ORL o0:(ORL
1120                       x0:(MOVWloadidx1 [i0] {s} p idx mem)
1121    s0:(SHLLconst [16] x1:(MOVBloadidx1 [i2] {s} p idx mem)))
1122    s1:(SHLLconst [24] x2:(MOVBloadidx1 [i3] {s} p idx mem)))
1123  && i2 == i0+2
1124  && i3 == i0+3
1125  && x0.Uses == 1
1126  && x1.Uses == 1
1127  && x2.Uses == 1
1128  && s0.Uses == 1
1129  && s1.Uses == 1
1130  && o0.Uses == 1
1131  && mergePoint(b,x0,x1,x2) != nil
1132  && clobber(x0)
1133  && clobber(x1)
1134  && clobber(x2)
1135  && clobber(s0)
1136  && clobber(s1)
1137  && clobber(o0)
1138  -> @mergePoint(b,x0,x1,x2) (MOVLloadidx1 <v.Type> [i0] {s} p idx mem)
1139
1140// Combine constant stores into larger (unaligned) stores.
1141(MOVBstoreconst [c] {s} p x:(MOVBstoreconst [a] {s} p mem))
1142  && x.Uses == 1
1143  && ValAndOff(a).Off() + 1 == ValAndOff(c).Off()
1144  && clobber(x)
1145  -> (MOVWstoreconst [makeValAndOff(ValAndOff(a).Val()&0xff | ValAndOff(c).Val()<<8, ValAndOff(a).Off())] {s} p mem)
1146(MOVBstoreconst [a] {s} p x:(MOVBstoreconst [c] {s} p mem))
1147  && x.Uses == 1
1148  && ValAndOff(a).Off() + 1 == ValAndOff(c).Off()
1149  && clobber(x)
1150  -> (MOVWstoreconst [makeValAndOff(ValAndOff(a).Val()&0xff | ValAndOff(c).Val()<<8, ValAndOff(a).Off())] {s} p mem)
1151(MOVWstoreconst [c] {s} p x:(MOVWstoreconst [a] {s} p mem))
1152  && x.Uses == 1
1153  && ValAndOff(a).Off() + 2 == ValAndOff(c).Off()
1154  && clobber(x)
1155  -> (MOVLstoreconst [makeValAndOff(ValAndOff(a).Val()&0xffff | ValAndOff(c).Val()<<16, ValAndOff(a).Off())] {s} p mem)
1156(MOVWstoreconst [a] {s} p x:(MOVWstoreconst [c] {s} p mem))
1157  && x.Uses == 1
1158  && ValAndOff(a).Off() + 2 == ValAndOff(c).Off()
1159  && clobber(x)
1160  -> (MOVLstoreconst [makeValAndOff(ValAndOff(a).Val()&0xffff | ValAndOff(c).Val()<<16, ValAndOff(a).Off())] {s} p mem)
1161
1162(MOVBstoreconstidx1 [c] {s} p i x:(MOVBstoreconstidx1 [a] {s} p i mem))
1163  && x.Uses == 1
1164  && ValAndOff(a).Off() + 1 == ValAndOff(c).Off()
1165  && clobber(x)
1166  -> (MOVWstoreconstidx1 [makeValAndOff(ValAndOff(a).Val()&0xff | ValAndOff(c).Val()<<8, ValAndOff(a).Off())] {s} p i mem)
1167(MOVWstoreconstidx1 [c] {s} p i x:(MOVWstoreconstidx1 [a] {s} p i mem))
1168  && x.Uses == 1
1169  && ValAndOff(a).Off() + 2 == ValAndOff(c).Off()
1170  && clobber(x)
1171  -> (MOVLstoreconstidx1 [makeValAndOff(ValAndOff(a).Val()&0xffff | ValAndOff(c).Val()<<16, ValAndOff(a).Off())] {s} p i mem)
1172
1173(MOVWstoreconstidx2 [c] {s} p i x:(MOVWstoreconstidx2 [a] {s} p i mem))
1174  && x.Uses == 1
1175  && ValAndOff(a).Off() + 2 == ValAndOff(c).Off()
1176  && clobber(x)
1177  -> (MOVLstoreconstidx1 [makeValAndOff(ValAndOff(a).Val()&0xffff | ValAndOff(c).Val()<<16, ValAndOff(a).Off())] {s} p (SHLLconst <i.Type> [1] i) mem)
1178
1179// Combine stores into larger (unaligned) stores.
1180(MOVBstore [i] {s} p (SHR(W|L)const [8] w) x:(MOVBstore [i-1] {s} p w mem))
1181  && x.Uses == 1
1182  && clobber(x)
1183  -> (MOVWstore [i-1] {s} p w mem)
1184(MOVBstore [i] {s} p w x:(MOVBstore {s} [i+1] p (SHR(W|L)const [8] w) mem))
1185  && x.Uses == 1
1186  && clobber(x)
1187  -> (MOVWstore [i] {s} p w mem)
1188(MOVBstore [i] {s} p (SHRLconst [j] w) x:(MOVBstore [i-1] {s} p w0:(SHRLconst [j-8] w) mem))
1189  && x.Uses == 1
1190  && clobber(x)
1191  -> (MOVWstore [i-1] {s} p w0 mem)
1192(MOVWstore [i] {s} p (SHRLconst [16] w) x:(MOVWstore [i-2] {s} p w mem))
1193  && x.Uses == 1
1194  && clobber(x)
1195  -> (MOVLstore [i-2] {s} p w mem)
1196(MOVWstore [i] {s} p (SHRLconst [j] w) x:(MOVWstore [i-2] {s} p w0:(SHRLconst [j-16] w) mem))
1197  && x.Uses == 1
1198  && clobber(x)
1199  -> (MOVLstore [i-2] {s} p w0 mem)
1200
1201(MOVBstoreidx1 [i] {s} p idx (SHR(L|W)const [8] w) x:(MOVBstoreidx1 [i-1] {s} p idx w mem))
1202  && x.Uses == 1
1203  && clobber(x)
1204  -> (MOVWstoreidx1 [i-1] {s} p idx w mem)
1205(MOVBstoreidx1 [i] {s} p idx w x:(MOVBstoreidx1 [i+1] {s} p idx (SHR(L|W)const [8] w) mem))
1206  && x.Uses == 1
1207  && clobber(x)
1208  -> (MOVWstoreidx1 [i] {s} p idx w mem)
1209(MOVBstoreidx1 [i] {s} p idx (SHRLconst [j] w) x:(MOVBstoreidx1 [i-1] {s} p idx w0:(SHRLconst [j-8] w) mem))
1210  && x.Uses == 1
1211  && clobber(x)
1212  -> (MOVWstoreidx1 [i-1] {s} p idx w0 mem)
1213(MOVWstoreidx1 [i] {s} p idx (SHRLconst [16] w) x:(MOVWstoreidx1 [i-2] {s} p idx w mem))
1214  && x.Uses == 1
1215  && clobber(x)
1216  -> (MOVLstoreidx1 [i-2] {s} p idx w mem)
1217(MOVWstoreidx1 [i] {s} p idx (SHRLconst [j] w) x:(MOVWstoreidx1 [i-2] {s} p idx w0:(SHRLconst [j-16] w) mem))
1218  && x.Uses == 1
1219  && clobber(x)
1220  -> (MOVLstoreidx1 [i-2] {s} p idx w0 mem)
1221
1222(MOVWstoreidx2 [i] {s} p idx (SHRLconst [16] w) x:(MOVWstoreidx2 [i-2] {s} p idx w mem))
1223  && x.Uses == 1
1224  && clobber(x)
1225  -> (MOVLstoreidx1 [i-2] {s} p (SHLLconst <idx.Type> [1] idx) w mem)
1226(MOVWstoreidx2 [i] {s} p idx (SHRLconst [j] w) x:(MOVWstoreidx2 [i-2] {s} p idx w0:(SHRLconst [j-16] w) mem))
1227  && x.Uses == 1
1228  && clobber(x)
1229  -> (MOVLstoreidx1 [i-2] {s} p (SHLLconst <idx.Type> [1] idx) w0 mem)
1230
1231// For PIC, break floating-point constant loading into two instructions so we have
1232// a register to use for holding the address of the constant pool entry.
1233(MOVSSconst [c]) && config.ctxt.Flag_shared -> (MOVSSconst2 (MOVSSconst1 [c]))
1234(MOVSDconst [c]) && config.ctxt.Flag_shared -> (MOVSDconst2 (MOVSDconst1 [c]))
1235
1236(CMP(L|W|B) l:(MOV(L|W|B)load {sym} [off] ptr mem) x) && canMergeLoad(v, l) && clobber(l) -> (CMP(L|W|B)load {sym} [off] ptr x mem)
1237(CMP(L|W|B) x l:(MOV(L|W|B)load {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) -> (InvertFlags (CMP(L|W|B)load {sym} [off] ptr x mem))
1238
1239(CMP(L|W|B)const l:(MOV(L|W|B)load {sym} [off] ptr mem) [c])
1240	&& l.Uses == 1
1241	&& validValAndOff(c, off)
1242	&& clobber(l) ->
1243  @l.Block (CMP(L|W|B)constload {sym} [makeValAndOff(c,off)] ptr mem)
1244
1245(CMPLload {sym} [off] ptr (MOVLconst [c]) mem) && validValAndOff(int64(int32(c)),off) -> (CMPLconstload {sym} [makeValAndOff(int64(int32(c)),off)] ptr mem)
1246(CMPWload {sym} [off] ptr (MOVLconst [c]) mem) && validValAndOff(int64(int16(c)),off) -> (CMPWconstload {sym} [makeValAndOff(int64(int16(c)),off)] ptr mem)
1247(CMPBload {sym} [off] ptr (MOVLconst [c]) mem) && validValAndOff(int64(int8(c)),off) -> (CMPBconstload {sym} [makeValAndOff(int64(int8(c)),off)] ptr mem)
1248
1249(MOVBload [off] {sym} (SB) _) && symIsRO(sym) -> (MOVLconst [int64(read8(sym, off))])
1250(MOVWload [off] {sym} (SB) _) && symIsRO(sym) -> (MOVLconst [int64(read16(sym, off, config.BigEndian))])
1251(MOVLload [off] {sym} (SB) _) && symIsRO(sym) -> (MOVLconst [int64(int32(read32(sym, off, config.BigEndian)))])
1252