1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp %s -o - | FileCheck %s 3 4; Check some LSR loop postinc 5 6; fma loop with a destination that is the same as one of the sources 7define void @fma(float* noalias nocapture readonly %A, float* noalias nocapture readonly %B, float* noalias nocapture %C, i32 %n) { 8; CHECK-LABEL: fma: 9; CHECK: @ %bb.0: @ %entry 10; CHECK-NEXT: .save {r4, r5, r6, lr} 11; CHECK-NEXT: push {r4, r5, r6, lr} 12; CHECK-NEXT: cmp r3, #1 13; CHECK-NEXT: blt .LBB0_8 14; CHECK-NEXT: @ %bb.1: @ %for.body.preheader 15; CHECK-NEXT: cmp r3, #3 16; CHECK-NEXT: bhi .LBB0_3 17; CHECK-NEXT: @ %bb.2: 18; CHECK-NEXT: mov.w r12, #0 19; CHECK-NEXT: b .LBB0_6 20; CHECK-NEXT: .LBB0_3: @ %vector.ph 21; CHECK-NEXT: bic r12, r3, #3 22; CHECK-NEXT: movs r5, #1 23; CHECK-NEXT: sub.w r6, r12, #4 24; CHECK-NEXT: mov r4, r0 25; CHECK-NEXT: add.w lr, r5, r6, lsr #2 26; CHECK-NEXT: mov r5, r1 27; CHECK-NEXT: mov r6, r2 28; CHECK-NEXT: dls lr, lr 29; CHECK-NEXT: .LBB0_4: @ %vector.body 30; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 31; CHECK-NEXT: vldrw.u32 q0, [r4], #16 32; CHECK-NEXT: vldrw.u32 q1, [r5], #16 33; CHECK-NEXT: vldrw.u32 q2, [r6] 34; CHECK-NEXT: vfma.f32 q2, q1, q0 35; CHECK-NEXT: vstrb.8 q2, [r6], #16 36; CHECK-NEXT: le lr, .LBB0_4 37; CHECK-NEXT: @ %bb.5: @ %middle.block 38; CHECK-NEXT: cmp r12, r3 39; CHECK-NEXT: it eq 40; CHECK-NEXT: popeq {r4, r5, r6, pc} 41; CHECK-NEXT: .LBB0_6: @ %for.body.preheader12 42; CHECK-NEXT: sub.w lr, r3, r12 43; CHECK-NEXT: add.w r0, r0, r12, lsl #2 44; CHECK-NEXT: add.w r1, r1, r12, lsl #2 45; CHECK-NEXT: add.w r2, r2, r12, lsl #2 46; CHECK-NEXT: dls lr, lr 47; CHECK-NEXT: .LBB0_7: @ %for.body 48; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 49; CHECK-NEXT: vldr s0, [r0] 50; CHECK-NEXT: adds r0, #4 51; CHECK-NEXT: vldr s2, [r1] 52; CHECK-NEXT: adds r1, #4 53; CHECK-NEXT: vldr s4, [r2] 54; CHECK-NEXT: vfma.f32 s4, s2, s0 55; CHECK-NEXT: vstr s4, [r2] 56; CHECK-NEXT: adds r2, #4 57; CHECK-NEXT: le lr, .LBB0_7 58; CHECK-NEXT: .LBB0_8: @ %for.cond.cleanup 59; CHECK-NEXT: pop {r4, r5, r6, pc} 60entry: 61 %cmp8 = icmp sgt i32 %n, 0 62 br i1 %cmp8, label %for.body.preheader, label %for.cond.cleanup 63 64for.body.preheader: ; preds = %entry 65 %min.iters.check = icmp ult i32 %n, 4 66 br i1 %min.iters.check, label %for.body.preheader12, label %vector.ph 67 68for.body.preheader12: ; preds = %middle.block, %for.body.preheader 69 %i.09.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] 70 br label %for.body 71 72vector.ph: ; preds = %for.body.preheader 73 %n.vec = and i32 %n, -4 74 br label %vector.body 75 76vector.body: ; preds = %vector.body, %vector.ph 77 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 78 %0 = getelementptr inbounds float, float* %A, i32 %index 79 %1 = bitcast float* %0 to <4 x float>* 80 %wide.load = load <4 x float>, <4 x float>* %1, align 4 81 %2 = getelementptr inbounds float, float* %B, i32 %index 82 %3 = bitcast float* %2 to <4 x float>* 83 %wide.load10 = load <4 x float>, <4 x float>* %3, align 4 84 %4 = fmul fast <4 x float> %wide.load10, %wide.load 85 %5 = getelementptr inbounds float, float* %C, i32 %index 86 %6 = bitcast float* %5 to <4 x float>* 87 %wide.load11 = load <4 x float>, <4 x float>* %6, align 4 88 %7 = fadd fast <4 x float> %wide.load11, %4 89 %8 = bitcast float* %5 to <4 x float>* 90 store <4 x float> %7, <4 x float>* %8, align 4 91 %index.next = add i32 %index, 4 92 %9 = icmp eq i32 %index.next, %n.vec 93 br i1 %9, label %middle.block, label %vector.body 94 95middle.block: ; preds = %vector.body 96 %cmp.n = icmp eq i32 %n.vec, %n 97 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader12 98 99for.cond.cleanup: ; preds = %for.body, %middle.block, %entry 100 ret void 101 102for.body: ; preds = %for.body.preheader12, %for.body 103 %i.09 = phi i32 [ %inc, %for.body ], [ %i.09.ph, %for.body.preheader12 ] 104 %arrayidx = getelementptr inbounds float, float* %A, i32 %i.09 105 %10 = load float, float* %arrayidx, align 4 106 %arrayidx1 = getelementptr inbounds float, float* %B, i32 %i.09 107 %11 = load float, float* %arrayidx1, align 4 108 %mul = fmul fast float %11, %10 109 %arrayidx2 = getelementptr inbounds float, float* %C, i32 %i.09 110 %12 = load float, float* %arrayidx2, align 4 111 %add = fadd fast float %12, %mul 112 store float %add, float* %arrayidx2, align 4 113 %inc = add nuw nsw i32 %i.09, 1 114 %exitcond = icmp eq i32 %inc, %n 115 br i1 %exitcond, label %for.cond.cleanup, label %for.body 116} 117 118 119; Same as above but tail predicated 120; FIXME: The postinc here is put on the load, not the store. An extra mov is needed in the loop because of it. 121define void @fma_tailpred(float* noalias nocapture readonly %A, float* noalias nocapture readonly %B, float* noalias nocapture %C, i32 %n) { 122; CHECK-LABEL: fma_tailpred: 123; CHECK: @ %bb.0: @ %entry 124; CHECK-NEXT: .save {r4, lr} 125; CHECK-NEXT: push {r4, lr} 126; CHECK-NEXT: .vsave {d8, d9} 127; CHECK-NEXT: vpush {d8, d9} 128; CHECK-NEXT: cmp r3, #1 129; CHECK-NEXT: blt .LBB1_3 130; CHECK-NEXT: @ %bb.1: @ %vector.ph 131; CHECK-NEXT: add.w r12, r3, #3 132; CHECK-NEXT: adr r4, .LCPI1_0 133; CHECK-NEXT: bic r12, r12, #3 134; CHECK-NEXT: mov.w lr, #1 135; CHECK-NEXT: sub.w r12, r12, #4 136; CHECK-NEXT: vldrw.u32 q0, [r4] 137; CHECK-NEXT: add.w lr, lr, r12, lsr #2 138; CHECK-NEXT: sub.w r12, r3, #1 139; CHECK-NEXT: movs r3, #0 140; CHECK-NEXT: vdup.32 q1, r12 141; CHECK-NEXT: dls lr, lr 142; CHECK-NEXT: .LBB1_2: @ %vector.body 143; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 144; CHECK-NEXT: vdup.32 q2, r3 145; CHECK-NEXT: adds r3, #4 146; CHECK-NEXT: vorr q2, q2, q0 147; CHECK-NEXT: vpttt.u32 cs, q1, q2 148; CHECK-NEXT: vldrwt.u32 q2, [r0], #16 149; CHECK-NEXT: vldrwt.u32 q3, [r1], #16 150; CHECK-NEXT: vldrwt.u32 q4, [r2] 151; CHECK-NEXT: vfma.f32 q4, q3, q2 152; CHECK-NEXT: vpst 153; CHECK-NEXT: vstrwt.32 q4, [r2], #16 154; CHECK-NEXT: le lr, .LBB1_2 155; CHECK-NEXT: .LBB1_3: @ %for.cond.cleanup 156; CHECK-NEXT: vpop {d8, d9} 157; CHECK-NEXT: pop {r4, pc} 158; CHECK-NEXT: .p2align 4 159; CHECK-NEXT: @ %bb.4: 160; CHECK-NEXT: .LCPI1_0: 161; CHECK-NEXT: .long 0 @ 0x0 162; CHECK-NEXT: .long 1 @ 0x1 163; CHECK-NEXT: .long 2 @ 0x2 164; CHECK-NEXT: .long 3 @ 0x3 165entry: 166 %cmp8 = icmp sgt i32 %n, 0 167 br i1 %cmp8, label %vector.ph, label %for.cond.cleanup 168 169vector.ph: ; preds = %entry 170 %n.rnd.up = add i32 %n, 3 171 %n.vec = and i32 %n.rnd.up, -4 172 %trip.count.minus.1 = add i32 %n, -1 173 %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 174 %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer 175 br label %vector.body 176 177vector.body: ; preds = %vector.body, %vector.ph 178 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 179 %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 180 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer 181 %induction = or <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3> 182 %0 = getelementptr inbounds float, float* %A, i32 %index 183 %1 = icmp ule <4 x i32> %induction, %broadcast.splat11 184 %2 = bitcast float* %0 to <4 x float>* 185 %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %2, i32 4, <4 x i1> %1, <4 x float> undef) 186 %3 = getelementptr inbounds float, float* %B, i32 %index 187 %4 = bitcast float* %3 to <4 x float>* 188 %wide.masked.load12 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %4, i32 4, <4 x i1> %1, <4 x float> undef) 189 %5 = fmul fast <4 x float> %wide.masked.load12, %wide.masked.load 190 %6 = getelementptr inbounds float, float* %C, i32 %index 191 %7 = bitcast float* %6 to <4 x float>* 192 %wide.masked.load13 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %7, i32 4, <4 x i1> %1, <4 x float> undef) 193 %8 = fadd fast <4 x float> %wide.masked.load13, %5 194 %9 = bitcast float* %6 to <4 x float>* 195 call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %8, <4 x float>* %9, i32 4, <4 x i1> %1) 196 %index.next = add i32 %index, 4 197 %10 = icmp eq i32 %index.next, %n.vec 198 br i1 %10, label %for.cond.cleanup, label %vector.body 199 200for.cond.cleanup: ; preds = %vector.body, %entry 201 ret void 202} 203 204 205; Multiple loads of the loop with a common base 206define i8* @test(i8* nocapture readonly %input_row, i8* nocapture readonly %input_col, i16 zeroext %output_ch, i16 zeroext %num_cols, i32 %col_offset, i16 signext %activation_min, i16 zeroext %row_len, i32* nocapture readonly %bias, i8* returned %out) { 207; CHECK-LABEL: test: 208; CHECK: @ %bb.0: @ %entry 209; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} 210; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} 211; CHECK-NEXT: .pad #20 212; CHECK-NEXT: sub sp, #20 213; CHECK-NEXT: cmp r3, #4 214; CHECK-NEXT: strd r0, r1, [sp, #12] @ 8-byte Folded Spill 215; CHECK-NEXT: bne .LBB2_8 216; CHECK-NEXT: @ %bb.1: @ %for.cond.preheader 217; CHECK-NEXT: cmp r2, #0 218; CHECK-NEXT: beq .LBB2_8 219; CHECK-NEXT: @ %bb.2: @ %for.body.lr.ph 220; CHECK-NEXT: ldr r3, [sp, #64] 221; CHECK-NEXT: mov.w r11, #0 222; CHECK-NEXT: ldr r1, [sp, #16] @ 4-byte Reload 223; CHECK-NEXT: ldr.w r9, [sp, #56] 224; CHECK-NEXT: add.w r0, r1, r3, lsl #1 225; CHECK-NEXT: str r0, [sp, #8] @ 4-byte Spill 226; CHECK-NEXT: adds r0, r1, r3 227; CHECK-NEXT: str r0, [sp, #4] @ 4-byte Spill 228; CHECK-NEXT: add.w r0, r3, r3, lsl #1 229; CHECK-NEXT: add r0, r1 230; CHECK-NEXT: str r0, [sp] @ 4-byte Spill 231; CHECK-NEXT: adds r0, r3, #7 232; CHECK-NEXT: lsrs r0, r0, #3 233; CHECK-NEXT: b .LBB2_5 234; CHECK-NEXT: .LBB2_3: @ in Loop: Header=BB2_5 Depth=1 235; CHECK-NEXT: mov r8, r12 236; CHECK-NEXT: mov r10, r12 237; CHECK-NEXT: mov r6, r12 238; CHECK-NEXT: .LBB2_4: @ %for.cond.cleanup23 239; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1 240; CHECK-NEXT: ldr r3, [sp, #72] 241; CHECK-NEXT: add.w r1, r10, r8 242; CHECK-NEXT: add r1, r6 243; CHECK-NEXT: add r1, r12 244; CHECK-NEXT: strb.w r1, [r3, r11] 245; CHECK-NEXT: add.w r11, r11, #1 246; CHECK-NEXT: cmp r11, r2 247; CHECK-NEXT: beq .LBB2_8 248; CHECK-NEXT: .LBB2_5: @ %for.body 249; CHECK-NEXT: @ =>This Loop Header: Depth=1 250; CHECK-NEXT: @ Child Loop BB2_7 Depth 2 251; CHECK-NEXT: ldr r1, [sp, #68] 252; CHECK-NEXT: subs.w lr, r0, r0 253; CHECK-NEXT: ldr.w r12, [r1, r11, lsl #2] 254; CHECK-NEXT: ble .LBB2_3 255; CHECK-NEXT: @ %bb.6: @ %for.body24.preheader 256; CHECK-NEXT: @ in Loop: Header=BB2_5 Depth=1 257; CHECK-NEXT: ldr r3, [sp, #64] 258; CHECK-NEXT: mov r6, r12 259; CHECK-NEXT: ldr r1, [sp, #12] @ 4-byte Reload 260; CHECK-NEXT: dls lr, lr 261; CHECK-NEXT: ldr r5, [sp, #8] @ 4-byte Reload 262; CHECK-NEXT: mov r10, r12 263; CHECK-NEXT: mla r7, r11, r3, r1 264; CHECK-NEXT: ldr r1, [sp, #16] @ 4-byte Reload 265; CHECK-NEXT: ldrd r4, r3, [sp] @ 8-byte Folded Reload 266; CHECK-NEXT: mov r8, r12 267; CHECK-NEXT: .LBB2_7: @ %for.body24 268; CHECK-NEXT: @ Parent Loop BB2_5 Depth=1 269; CHECK-NEXT: @ => This Inner Loop Header: Depth=2 270; CHECK-NEXT: vldrb.s16 q0, [r4], #8 271; CHECK-NEXT: vadd.i16 q1, q0, r9 272; CHECK-NEXT: vldrb.s16 q0, [r7], #8 273; CHECK-NEXT: vmlava.s16 r12, q0, q1 274; CHECK-NEXT: vldrb.s16 q1, [r5], #8 275; CHECK-NEXT: vadd.i16 q1, q1, r9 276; CHECK-NEXT: vmlava.s16 r6, q0, q1 277; CHECK-NEXT: vldrb.s16 q1, [r3], #8 278; CHECK-NEXT: vadd.i16 q1, q1, r9 279; CHECK-NEXT: vmlava.s16 r10, q0, q1 280; CHECK-NEXT: vldrb.s16 q1, [r1], #8 281; CHECK-NEXT: vadd.i16 q1, q1, r9 282; CHECK-NEXT: vmlava.s16 r8, q0, q1 283; CHECK-NEXT: le lr, .LBB2_7 284; CHECK-NEXT: b .LBB2_4 285; CHECK-NEXT: .LBB2_8: @ %if.end 286; CHECK-NEXT: ldr r0, [sp, #72] 287; CHECK-NEXT: add sp, #20 288; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} 289entry: 290 %cmp = icmp eq i16 %num_cols, 4 291 br i1 %cmp, label %for.cond.preheader, label %if.end 292 293for.cond.preheader: ; preds = %entry 294 %conv2 = zext i16 %output_ch to i32 295 %cmp3114 = icmp eq i16 %output_ch, 0 296 br i1 %cmp3114, label %if.end, label %for.body.lr.ph 297 298for.body.lr.ph: ; preds = %for.cond.preheader 299 %conv5 = zext i16 %row_len to i32 300 %add.ptr9 = getelementptr inbounds i8, i8* %input_col, i32 %conv5 301 %mul11 = shl nuw nsw i32 %conv5, 1 302 %add.ptr12 = getelementptr inbounds i8, i8* %input_col, i32 %mul11 303 %mul14 = mul nuw nsw i32 %conv5, 3 304 %add.ptr15 = getelementptr inbounds i8, i8* %input_col, i32 %mul14 305 %add = add nuw nsw i32 %conv5, 7 306 %div = lshr i32 %add, 3 307 %conv25 = trunc i32 %col_offset to i16 308 %.splatinsert = insertelement <8 x i16> undef, i16 %conv25, i32 0 309 %.splat = shufflevector <8 x i16> %.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer 310 br label %for.body 311 312for.body: ; preds = %for.cond.cleanup23, %for.body.lr.ph 313 %i_out_ch.0116 = phi i32 [ 0, %for.body.lr.ph ], [ %inc37, %for.cond.cleanup23 ] 314 %i_row_loop.0115 = phi i32 [ undef, %for.body.lr.ph ], [ %i_row_loop.1.lcssa, %for.cond.cleanup23 ] 315 %arrayidx = getelementptr inbounds i32, i32* %bias, i32 %i_out_ch.0116 316 %0 = load i32, i32* %arrayidx, align 4 317 %cmp2199 = icmp slt i32 %i_row_loop.0115, %div 318 br i1 %cmp2199, label %for.body24.preheader, label %for.cond.cleanup23 319 320for.body24.preheader: ; preds = %for.body 321 %mul = mul nuw nsw i32 %i_out_ch.0116, %conv5 322 %add.ptr = getelementptr inbounds i8, i8* %input_row, i32 %mul 323 br label %for.body24 324 325for.cond.cleanup23: ; preds = %for.body24, %for.body 326 %acc_0.0.lcssa = phi i32 [ %0, %for.body ], [ %20, %for.body24 ] 327 %acc_1.0.lcssa = phi i32 [ %0, %for.body ], [ %21, %for.body24 ] 328 %acc_2.0.lcssa = phi i32 [ %0, %for.body ], [ %22, %for.body24 ] 329 %acc_3.0.lcssa = phi i32 [ %0, %for.body ], [ %23, %for.body24 ] 330 %i_row_loop.1.lcssa = phi i32 [ %i_row_loop.0115, %for.body ], [ %div, %for.body24 ] 331 %add31 = add nsw i32 %acc_1.0.lcssa, %acc_0.0.lcssa 332 %add32 = add nsw i32 %add31, %acc_2.0.lcssa 333 %add33 = add nsw i32 %add32, %acc_3.0.lcssa 334 %conv34 = trunc i32 %add33 to i8 335 %arrayidx35 = getelementptr inbounds i8, i8* %out, i32 %i_out_ch.0116 336 store i8 %conv34, i8* %arrayidx35, align 1 337 %inc37 = add nuw nsw i32 %i_out_ch.0116, 1 338 %exitcond120 = icmp eq i32 %inc37, %conv2 339 br i1 %exitcond120, label %if.end, label %for.body 340 341for.body24: ; preds = %for.body24, %for.body24.preheader 342 %ip_r0.0109 = phi i8* [ %add.ptr26, %for.body24 ], [ %add.ptr, %for.body24.preheader ] 343 %ip_c0.0108 = phi i8* [ %add.ptr27, %for.body24 ], [ %input_col, %for.body24.preheader ] 344 %ip_c1.0107 = phi i8* [ %add.ptr28, %for.body24 ], [ %add.ptr9, %for.body24.preheader ] 345 %ip_c2.0106 = phi i8* [ %add.ptr29, %for.body24 ], [ %add.ptr12, %for.body24.preheader ] 346 %i_row_loop.1105 = phi i32 [ %inc, %for.body24 ], [ %i_row_loop.0115, %for.body24.preheader ] 347 %ip_c3.0104 = phi i8* [ %add.ptr30, %for.body24 ], [ %add.ptr15, %for.body24.preheader ] 348 %acc_3.0103 = phi i32 [ %23, %for.body24 ], [ %0, %for.body24.preheader ] 349 %acc_2.0102 = phi i32 [ %22, %for.body24 ], [ %0, %for.body24.preheader ] 350 %acc_1.0101 = phi i32 [ %21, %for.body24 ], [ %0, %for.body24.preheader ] 351 %acc_0.0100 = phi i32 [ %20, %for.body24 ], [ %0, %for.body24.preheader ] 352 %1 = bitcast i8* %ip_r0.0109 to <8 x i8>* 353 %2 = load <8 x i8>, <8 x i8>* %1, align 1 354 %3 = sext <8 x i8> %2 to <8 x i16> 355 %add.ptr26 = getelementptr inbounds i8, i8* %ip_r0.0109, i32 8 356 %4 = bitcast i8* %ip_c0.0108 to <8 x i8>* 357 %5 = load <8 x i8>, <8 x i8>* %4, align 1 358 %6 = sext <8 x i8> %5 to <8 x i16> 359 %add.ptr27 = getelementptr inbounds i8, i8* %ip_c0.0108, i32 8 360 %7 = add <8 x i16> %.splat, %6 361 %8 = bitcast i8* %ip_c1.0107 to <8 x i8>* 362 %9 = load <8 x i8>, <8 x i8>* %8, align 1 363 %10 = sext <8 x i8> %9 to <8 x i16> 364 %add.ptr28 = getelementptr inbounds i8, i8* %ip_c1.0107, i32 8 365 %11 = add <8 x i16> %.splat, %10 366 %12 = bitcast i8* %ip_c2.0106 to <8 x i8>* 367 %13 = load <8 x i8>, <8 x i8>* %12, align 1 368 %14 = sext <8 x i8> %13 to <8 x i16> 369 %add.ptr29 = getelementptr inbounds i8, i8* %ip_c2.0106, i32 8 370 %15 = add <8 x i16> %.splat, %14 371 %16 = bitcast i8* %ip_c3.0104 to <8 x i8>* 372 %17 = load <8 x i8>, <8 x i8>* %16, align 1 373 %18 = sext <8 x i8> %17 to <8 x i16> 374 %add.ptr30 = getelementptr inbounds i8, i8* %ip_c3.0104, i32 8 375 %19 = add <8 x i16> %.splat, %18 376 %20 = tail call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 0, i32 %acc_0.0100, <8 x i16> %3, <8 x i16> %7) 377 %21 = tail call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 0, i32 %acc_1.0101, <8 x i16> %3, <8 x i16> %11) 378 %22 = tail call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 0, i32 %acc_2.0102, <8 x i16> %3, <8 x i16> %15) 379 %23 = tail call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 0, i32 %acc_3.0103, <8 x i16> %3, <8 x i16> %19) 380 %inc = add nsw i32 %i_row_loop.1105, 1 381 %exitcond = icmp eq i32 %inc, %div 382 br i1 %exitcond, label %for.cond.cleanup23, label %for.body24 383 384if.end: ; preds = %for.cond.cleanup23, %for.cond.preheader, %entry 385 ret i8* %out 386} 387 388; Same as above with optsize 389define i8* @test_optsize(i8* nocapture readonly %input_row, i8* nocapture readonly %input_col, i16 zeroext %output_ch, i16 zeroext %num_cols, i32 %col_offset, i16 signext %activation_min, i16 zeroext %row_len, i32* nocapture readonly %bias, i8* returned %out) optsize { 390; CHECK-LABEL: test_optsize: 391; CHECK: @ %bb.0: @ %entry 392; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} 393; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} 394; CHECK-NEXT: .pad #20 395; CHECK-NEXT: sub sp, #20 396; CHECK-NEXT: cmp r3, #4 397; CHECK-NEXT: strd r0, r1, [sp, #12] @ 8-byte Folded Spill 398; CHECK-NEXT: bne .LBB3_8 399; CHECK-NEXT: @ %bb.1: @ %for.cond.preheader 400; CHECK-NEXT: cmp r2, #0 401; CHECK-NEXT: beq .LBB3_8 402; CHECK-NEXT: @ %bb.2: @ %for.body.lr.ph 403; CHECK-NEXT: ldr r3, [sp, #64] 404; CHECK-NEXT: mov.w r11, #0 405; CHECK-NEXT: ldr r1, [sp, #16] @ 4-byte Reload 406; CHECK-NEXT: ldr.w r9, [sp, #56] 407; CHECK-NEXT: add.w r0, r1, r3, lsl #1 408; CHECK-NEXT: str r0, [sp, #8] @ 4-byte Spill 409; CHECK-NEXT: adds r0, r1, r3 410; CHECK-NEXT: str r0, [sp, #4] @ 4-byte Spill 411; CHECK-NEXT: add.w r0, r3, r3, lsl #1 412; CHECK-NEXT: add r0, r1 413; CHECK-NEXT: str r0, [sp] @ 4-byte Spill 414; CHECK-NEXT: adds r0, r3, #7 415; CHECK-NEXT: lsrs r0, r0, #3 416; CHECK-NEXT: .LBB3_3: @ %for.body 417; CHECK-NEXT: @ =>This Loop Header: Depth=1 418; CHECK-NEXT: @ Child Loop BB3_5 Depth 2 419; CHECK-NEXT: ldr r1, [sp, #68] 420; CHECK-NEXT: subs.w lr, r0, r0 421; CHECK-NEXT: ldr.w r12, [r1, r11, lsl #2] 422; CHECK-NEXT: ble .LBB3_6 423; CHECK-NEXT: @ %bb.4: @ %for.body24.preheader 424; CHECK-NEXT: @ in Loop: Header=BB3_3 Depth=1 425; CHECK-NEXT: ldr r3, [sp, #64] 426; CHECK-NEXT: mov r6, r12 427; CHECK-NEXT: ldr r1, [sp, #12] @ 4-byte Reload 428; CHECK-NEXT: dls lr, lr 429; CHECK-NEXT: ldr r5, [sp, #8] @ 4-byte Reload 430; CHECK-NEXT: mov r10, r12 431; CHECK-NEXT: mla r7, r11, r3, r1 432; CHECK-NEXT: ldr r1, [sp, #16] @ 4-byte Reload 433; CHECK-NEXT: ldrd r4, r3, [sp] @ 8-byte Folded Reload 434; CHECK-NEXT: mov r8, r12 435; CHECK-NEXT: .LBB3_5: @ %for.body24 436; CHECK-NEXT: @ Parent Loop BB3_3 Depth=1 437; CHECK-NEXT: @ => This Inner Loop Header: Depth=2 438; CHECK-NEXT: vldrb.s16 q0, [r4], #8 439; CHECK-NEXT: vadd.i16 q1, q0, r9 440; CHECK-NEXT: vldrb.s16 q0, [r7], #8 441; CHECK-NEXT: vmlava.s16 r12, q0, q1 442; CHECK-NEXT: vldrb.s16 q1, [r5], #8 443; CHECK-NEXT: vadd.i16 q1, q1, r9 444; CHECK-NEXT: vmlava.s16 r6, q0, q1 445; CHECK-NEXT: vldrb.s16 q1, [r3], #8 446; CHECK-NEXT: vadd.i16 q1, q1, r9 447; CHECK-NEXT: vmlava.s16 r10, q0, q1 448; CHECK-NEXT: vldrb.s16 q1, [r1], #8 449; CHECK-NEXT: vadd.i16 q1, q1, r9 450; CHECK-NEXT: vmlava.s16 r8, q0, q1 451; CHECK-NEXT: le lr, .LBB3_5 452; CHECK-NEXT: b .LBB3_7 453; CHECK-NEXT: .LBB3_6: @ in Loop: Header=BB3_3 Depth=1 454; CHECK-NEXT: mov r8, r12 455; CHECK-NEXT: mov r10, r12 456; CHECK-NEXT: mov r6, r12 457; CHECK-NEXT: .LBB3_7: @ %for.cond.cleanup23 458; CHECK-NEXT: @ in Loop: Header=BB3_3 Depth=1 459; CHECK-NEXT: ldr r3, [sp, #72] 460; CHECK-NEXT: add.w r1, r10, r8 461; CHECK-NEXT: add r1, r6 462; CHECK-NEXT: add r1, r12 463; CHECK-NEXT: strb.w r1, [r3, r11] 464; CHECK-NEXT: add.w r11, r11, #1 465; CHECK-NEXT: cmp r11, r2 466; CHECK-NEXT: bne .LBB3_3 467; CHECK-NEXT: .LBB3_8: @ %if.end 468; CHECK-NEXT: ldr r0, [sp, #72] 469; CHECK-NEXT: add sp, #20 470; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} 471entry: 472 %cmp = icmp eq i16 %num_cols, 4 473 br i1 %cmp, label %for.cond.preheader, label %if.end 474 475for.cond.preheader: ; preds = %entry 476 %conv2 = zext i16 %output_ch to i32 477 %cmp3114 = icmp eq i16 %output_ch, 0 478 br i1 %cmp3114, label %if.end, label %for.body.lr.ph 479 480for.body.lr.ph: ; preds = %for.cond.preheader 481 %conv5 = zext i16 %row_len to i32 482 %add.ptr9 = getelementptr inbounds i8, i8* %input_col, i32 %conv5 483 %mul11 = shl nuw nsw i32 %conv5, 1 484 %add.ptr12 = getelementptr inbounds i8, i8* %input_col, i32 %mul11 485 %mul14 = mul nuw nsw i32 %conv5, 3 486 %add.ptr15 = getelementptr inbounds i8, i8* %input_col, i32 %mul14 487 %add = add nuw nsw i32 %conv5, 7 488 %div = lshr i32 %add, 3 489 %conv25 = trunc i32 %col_offset to i16 490 %.splatinsert = insertelement <8 x i16> undef, i16 %conv25, i32 0 491 %.splat = shufflevector <8 x i16> %.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer 492 br label %for.body 493 494for.body: ; preds = %for.cond.cleanup23, %for.body.lr.ph 495 %i_out_ch.0116 = phi i32 [ 0, %for.body.lr.ph ], [ %inc37, %for.cond.cleanup23 ] 496 %i_row_loop.0115 = phi i32 [ undef, %for.body.lr.ph ], [ %i_row_loop.1.lcssa, %for.cond.cleanup23 ] 497 %arrayidx = getelementptr inbounds i32, i32* %bias, i32 %i_out_ch.0116 498 %0 = load i32, i32* %arrayidx, align 4 499 %cmp2199 = icmp slt i32 %i_row_loop.0115, %div 500 br i1 %cmp2199, label %for.body24.preheader, label %for.cond.cleanup23 501 502for.body24.preheader: ; preds = %for.body 503 %mul = mul nuw nsw i32 %i_out_ch.0116, %conv5 504 %add.ptr = getelementptr inbounds i8, i8* %input_row, i32 %mul 505 br label %for.body24 506 507for.cond.cleanup23: ; preds = %for.body24, %for.body 508 %acc_0.0.lcssa = phi i32 [ %0, %for.body ], [ %20, %for.body24 ] 509 %acc_1.0.lcssa = phi i32 [ %0, %for.body ], [ %21, %for.body24 ] 510 %acc_2.0.lcssa = phi i32 [ %0, %for.body ], [ %22, %for.body24 ] 511 %acc_3.0.lcssa = phi i32 [ %0, %for.body ], [ %23, %for.body24 ] 512 %i_row_loop.1.lcssa = phi i32 [ %i_row_loop.0115, %for.body ], [ %div, %for.body24 ] 513 %add31 = add nsw i32 %acc_1.0.lcssa, %acc_0.0.lcssa 514 %add32 = add nsw i32 %add31, %acc_2.0.lcssa 515 %add33 = add nsw i32 %add32, %acc_3.0.lcssa 516 %conv34 = trunc i32 %add33 to i8 517 %arrayidx35 = getelementptr inbounds i8, i8* %out, i32 %i_out_ch.0116 518 store i8 %conv34, i8* %arrayidx35, align 1 519 %inc37 = add nuw nsw i32 %i_out_ch.0116, 1 520 %exitcond120 = icmp eq i32 %inc37, %conv2 521 br i1 %exitcond120, label %if.end, label %for.body 522 523for.body24: ; preds = %for.body24, %for.body24.preheader 524 %ip_r0.0109 = phi i8* [ %add.ptr26, %for.body24 ], [ %add.ptr, %for.body24.preheader ] 525 %ip_c0.0108 = phi i8* [ %add.ptr27, %for.body24 ], [ %input_col, %for.body24.preheader ] 526 %ip_c1.0107 = phi i8* [ %add.ptr28, %for.body24 ], [ %add.ptr9, %for.body24.preheader ] 527 %ip_c2.0106 = phi i8* [ %add.ptr29, %for.body24 ], [ %add.ptr12, %for.body24.preheader ] 528 %i_row_loop.1105 = phi i32 [ %inc, %for.body24 ], [ %i_row_loop.0115, %for.body24.preheader ] 529 %ip_c3.0104 = phi i8* [ %add.ptr30, %for.body24 ], [ %add.ptr15, %for.body24.preheader ] 530 %acc_3.0103 = phi i32 [ %23, %for.body24 ], [ %0, %for.body24.preheader ] 531 %acc_2.0102 = phi i32 [ %22, %for.body24 ], [ %0, %for.body24.preheader ] 532 %acc_1.0101 = phi i32 [ %21, %for.body24 ], [ %0, %for.body24.preheader ] 533 %acc_0.0100 = phi i32 [ %20, %for.body24 ], [ %0, %for.body24.preheader ] 534 %1 = bitcast i8* %ip_r0.0109 to <8 x i8>* 535 %2 = load <8 x i8>, <8 x i8>* %1, align 1 536 %3 = sext <8 x i8> %2 to <8 x i16> 537 %add.ptr26 = getelementptr inbounds i8, i8* %ip_r0.0109, i32 8 538 %4 = bitcast i8* %ip_c0.0108 to <8 x i8>* 539 %5 = load <8 x i8>, <8 x i8>* %4, align 1 540 %6 = sext <8 x i8> %5 to <8 x i16> 541 %add.ptr27 = getelementptr inbounds i8, i8* %ip_c0.0108, i32 8 542 %7 = add <8 x i16> %.splat, %6 543 %8 = bitcast i8* %ip_c1.0107 to <8 x i8>* 544 %9 = load <8 x i8>, <8 x i8>* %8, align 1 545 %10 = sext <8 x i8> %9 to <8 x i16> 546 %add.ptr28 = getelementptr inbounds i8, i8* %ip_c1.0107, i32 8 547 %11 = add <8 x i16> %.splat, %10 548 %12 = bitcast i8* %ip_c2.0106 to <8 x i8>* 549 %13 = load <8 x i8>, <8 x i8>* %12, align 1 550 %14 = sext <8 x i8> %13 to <8 x i16> 551 %add.ptr29 = getelementptr inbounds i8, i8* %ip_c2.0106, i32 8 552 %15 = add <8 x i16> %.splat, %14 553 %16 = bitcast i8* %ip_c3.0104 to <8 x i8>* 554 %17 = load <8 x i8>, <8 x i8>* %16, align 1 555 %18 = sext <8 x i8> %17 to <8 x i16> 556 %add.ptr30 = getelementptr inbounds i8, i8* %ip_c3.0104, i32 8 557 %19 = add <8 x i16> %.splat, %18 558 %20 = tail call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 0, i32 %acc_0.0100, <8 x i16> %3, <8 x i16> %7) 559 %21 = tail call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 0, i32 %acc_1.0101, <8 x i16> %3, <8 x i16> %11) 560 %22 = tail call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 0, i32 %acc_2.0102, <8 x i16> %3, <8 x i16> %15) 561 %23 = tail call i32 @llvm.arm.mve.vmldava.v8i16(i32 0, i32 0, i32 0, i32 %acc_3.0103, <8 x i16> %3, <8 x i16> %19) 562 %inc = add nsw i32 %i_row_loop.1105, 1 563 %exitcond = icmp eq i32 %inc, %div 564 br i1 %exitcond, label %for.cond.cleanup23, label %for.body24 565 566if.end: ; preds = %for.cond.cleanup23, %for.cond.preheader, %entry 567 ret i8* %out 568} 569 570 571; Similar but predicated 572define i32 @arm_nn_mat_mul_core_4x_s8(i32 %row_elements, i32 %offset, i8* %row_base, i8* %col_base, i32* nocapture readnone %sum_col, i32* nocapture %output) { 573; CHECK-LABEL: arm_nn_mat_mul_core_4x_s8: 574; CHECK: @ %bb.0: @ %entry 575; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r10, lr} 576; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r10, lr} 577; CHECK-NEXT: ldr.w r12, [sp, #32] 578; CHECK-NEXT: cmp r0, #1 579; CHECK-NEXT: blt .LBB4_3 580; CHECK-NEXT: @ %bb.1: @ %for.body.preheader 581; CHECK-NEXT: adds r5, r2, r1 582; CHECK-NEXT: add.w r7, r2, r1, lsl #1 583; CHECK-NEXT: add.w r1, r1, r1, lsl #1 584; CHECK-NEXT: mov.w r8, #0 585; CHECK-NEXT: add r1, r2 586; CHECK-NEXT: movs r6, #0 587; CHECK-NEXT: movs r4, #0 588; CHECK-NEXT: mov.w r10, #0 589; CHECK-NEXT: dlstp.8 lr, r0 590; CHECK-NEXT: .LBB4_2: @ %for.body 591; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 592; CHECK-NEXT: vldrb.u8 q0, [r3], #16 593; CHECK-NEXT: vldrb.u8 q1, [r1], #16 594; CHECK-NEXT: vmlava.s8 r10, q1, q0 595; CHECK-NEXT: vldrb.u8 q1, [r7], #16 596; CHECK-NEXT: vmlava.s8 r4, q1, q0 597; CHECK-NEXT: vldrb.u8 q1, [r5], #16 598; CHECK-NEXT: vmlava.s8 r6, q1, q0 599; CHECK-NEXT: vldrb.u8 q1, [r2], #16 600; CHECK-NEXT: vmlava.s8 r8, q1, q0 601; CHECK-NEXT: letp lr, .LBB4_2 602; CHECK-NEXT: b .LBB4_4 603; CHECK-NEXT: .LBB4_3: 604; CHECK-NEXT: mov.w r10, #0 605; CHECK-NEXT: movs r4, #0 606; CHECK-NEXT: movs r6, #0 607; CHECK-NEXT: mov.w r8, #0 608; CHECK-NEXT: .LBB4_4: @ %for.cond.cleanup 609; CHECK-NEXT: movs r0, #0 610; CHECK-NEXT: strd r8, r6, [r12] 611; CHECK-NEXT: strd r4, r10, [r12, #8] 612; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r10, pc} 613entry: 614 %add = add nsw i32 %row_elements, 15 615 %div = sdiv i32 %add, 16 616 %cmp84 = icmp sgt i32 %row_elements, 0 617 br i1 %cmp84, label %for.body.preheader, label %for.cond.cleanup 618 619for.body.preheader: ; preds = %entry 620 %mul2 = mul nsw i32 %offset, 3 621 %add.ptr3 = getelementptr inbounds i8, i8* %row_base, i32 %mul2 622 %mul = shl nsw i32 %offset, 1 623 %add.ptr1 = getelementptr inbounds i8, i8* %row_base, i32 %mul 624 %add.ptr = getelementptr inbounds i8, i8* %row_base, i32 %offset 625 %0 = icmp sgt i32 %div, 1 626 %smax = select i1 %0, i32 %div, i32 1 627 br label %for.body 628 629for.cond.cleanup: ; preds = %for.body, %entry 630 %acc_n.sroa.12.0.lcssa = phi i32 [ 0, %entry ], [ %15, %for.body ] 631 %acc_n.sroa.9.0.lcssa = phi i32 [ 0, %entry ], [ %12, %for.body ] 632 %acc_n.sroa.6.0.lcssa = phi i32 [ 0, %entry ], [ %9, %for.body ] 633 %acc_n.sroa.0.0.lcssa = phi i32 [ 0, %entry ], [ %6, %for.body ] 634 store i32 %acc_n.sroa.0.0.lcssa, i32* %output, align 4 635 %arrayidx19 = getelementptr inbounds i32, i32* %output, i32 1 636 store i32 %acc_n.sroa.6.0.lcssa, i32* %arrayidx19, align 4 637 %arrayidx21 = getelementptr inbounds i32, i32* %output, i32 2 638 store i32 %acc_n.sroa.9.0.lcssa, i32* %arrayidx21, align 4 639 %arrayidx23 = getelementptr inbounds i32, i32* %output, i32 3 640 store i32 %acc_n.sroa.12.0.lcssa, i32* %arrayidx23, align 4 641 ret i32 0 642 643for.body: ; preds = %for.body, %for.body.preheader 644 %col_base.addr.095 = phi i8* [ %add.ptr4, %for.body ], [ %col_base, %for.body.preheader ] 645 %acc_n.sroa.0.094 = phi i32 [ %6, %for.body ], [ 0, %for.body.preheader ] 646 %acc_n.sroa.6.093 = phi i32 [ %9, %for.body ], [ 0, %for.body.preheader ] 647 %acc_n.sroa.9.092 = phi i32 [ %12, %for.body ], [ 0, %for.body.preheader ] 648 %i.091 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ] 649 %row_elem.090 = phi i32 [ %sub, %for.body ], [ %row_elements, %for.body.preheader ] 650 %acc_n.sroa.12.089 = phi i32 [ %15, %for.body ], [ 0, %for.body.preheader ] 651 %ip_row_3.088 = phi i8* [ %add.ptr15, %for.body ], [ %add.ptr3, %for.body.preheader ] 652 %ip_row_2.087 = phi i8* [ %add.ptr14, %for.body ], [ %add.ptr1, %for.body.preheader ] 653 %ip_row_1.086 = phi i8* [ %add.ptr13, %for.body ], [ %add.ptr, %for.body.preheader ] 654 %ip_row_0.085 = phi i8* [ %add.ptr12, %for.body ], [ %row_base, %for.body.preheader ] 655 %1 = tail call <16 x i1> @llvm.arm.mve.vctp8(i32 %row_elem.090) 656 %sub = add nsw i32 %row_elem.090, -16 657 %2 = bitcast i8* %col_base.addr.095 to <16 x i8>* 658 %3 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %2, i32 1, <16 x i1> %1, <16 x i8> zeroinitializer) 659 %add.ptr4 = getelementptr inbounds i8, i8* %col_base.addr.095, i32 16 660 %4 = bitcast i8* %ip_row_0.085 to <16 x i8>* 661 %5 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %4, i32 1, <16 x i1> %1, <16 x i8> zeroinitializer) 662 %6 = tail call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 0, i32 0, i32 %acc_n.sroa.0.094, <16 x i8> %5, <16 x i8> %3, <16 x i1> %1) 663 %7 = bitcast i8* %ip_row_1.086 to <16 x i8>* 664 %8 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %7, i32 1, <16 x i1> %1, <16 x i8> zeroinitializer) 665 %9 = tail call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 0, i32 0, i32 %acc_n.sroa.6.093, <16 x i8> %8, <16 x i8> %3, <16 x i1> %1) 666 %10 = bitcast i8* %ip_row_2.087 to <16 x i8>* 667 %11 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %10, i32 1, <16 x i1> %1, <16 x i8> zeroinitializer) 668 %12 = tail call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 0, i32 0, i32 %acc_n.sroa.9.092, <16 x i8> %11, <16 x i8> %3, <16 x i1> %1) 669 %13 = bitcast i8* %ip_row_3.088 to <16 x i8>* 670 %14 = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %13, i32 1, <16 x i1> %1, <16 x i8> zeroinitializer) 671 %15 = tail call i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32 0, i32 0, i32 0, i32 %acc_n.sroa.12.089, <16 x i8> %14, <16 x i8> %3, <16 x i1> %1) 672 %add.ptr12 = getelementptr inbounds i8, i8* %ip_row_0.085, i32 16 673 %add.ptr13 = getelementptr inbounds i8, i8* %ip_row_1.086, i32 16 674 %add.ptr14 = getelementptr inbounds i8, i8* %ip_row_2.087, i32 16 675 %add.ptr15 = getelementptr inbounds i8, i8* %ip_row_3.088, i32 16 676 %inc = add nuw nsw i32 %i.091, 1 677 %exitcond = icmp eq i32 %inc, %smax 678 br i1 %exitcond, label %for.cond.cleanup, label %for.body 679} 680 681define i8* @signext(i8* %input_row, i8* %input_col, i16 zeroext %output_ch, i16 zeroext %num_cols, i32* nocapture readnone %output_shift, i32* nocapture readnone %output_mult, i32 %out_offset, i32 %col_offset, i32 %row_offset, i16 signext %activation_min, i16 signext %activation_max, i16 zeroext %row_len, i32* nocapture readonly %bias, i8* returned %out) { 682; CHECK-LABEL: signext: 683; CHECK: @ %bb.0: @ %entry 684; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} 685; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} 686; CHECK-NEXT: .pad #20 687; CHECK-NEXT: sub sp, #20 688; CHECK-NEXT: cmp r3, #4 689; CHECK-NEXT: strd r0, r1, [sp, #12] @ 8-byte Folded Spill 690; CHECK-NEXT: bne .LBB5_8 691; CHECK-NEXT: @ %bb.1: @ %for.cond.preheader 692; CHECK-NEXT: cmp r2, #0 693; CHECK-NEXT: beq .LBB5_8 694; CHECK-NEXT: @ %bb.2: @ %for.body.lr.ph 695; CHECK-NEXT: ldr r7, [sp, #84] 696; CHECK-NEXT: mov.w r11, #0 697; CHECK-NEXT: ldr r3, [sp, #16] @ 4-byte Reload 698; CHECK-NEXT: ldr r0, [sp, #68] 699; CHECK-NEXT: add.w r1, r3, r7, lsl #1 700; CHECK-NEXT: str r1, [sp, #8] @ 4-byte Spill 701; CHECK-NEXT: adds r1, r3, r7 702; CHECK-NEXT: str r1, [sp, #4] @ 4-byte Spill 703; CHECK-NEXT: add.w r1, r7, r7, lsl #1 704; CHECK-NEXT: vdup.16 q0, r0 705; CHECK-NEXT: adds r0, r3, r1 706; CHECK-NEXT: str r0, [sp] @ 4-byte Spill 707; CHECK-NEXT: adds r0, r7, #7 708; CHECK-NEXT: lsr.w r9, r0, #3 709; CHECK-NEXT: b .LBB5_5 710; CHECK-NEXT: .LBB5_3: @ in Loop: Header=BB5_5 Depth=1 711; CHECK-NEXT: mov r10, r12 712; CHECK-NEXT: mov r8, r12 713; CHECK-NEXT: mov r6, r12 714; CHECK-NEXT: .LBB5_4: @ %for.cond.cleanup23 715; CHECK-NEXT: @ in Loop: Header=BB5_5 Depth=1 716; CHECK-NEXT: ldr r1, [sp, #92] 717; CHECK-NEXT: add.w r0, r8, r10 718; CHECK-NEXT: add r0, r6 719; CHECK-NEXT: add r0, r12 720; CHECK-NEXT: strb.w r0, [r1, r11] 721; CHECK-NEXT: add.w r11, r11, #1 722; CHECK-NEXT: cmp r11, r2 723; CHECK-NEXT: beq .LBB5_8 724; CHECK-NEXT: .LBB5_5: @ %for.body 725; CHECK-NEXT: @ =>This Loop Header: Depth=1 726; CHECK-NEXT: @ Child Loop BB5_7 Depth 2 727; CHECK-NEXT: ldr r0, [sp, #88] 728; CHECK-NEXT: subs.w lr, r9, r9 729; CHECK-NEXT: ldr.w r12, [r0, r11, lsl #2] 730; CHECK-NEXT: ble .LBB5_3 731; CHECK-NEXT: @ %bb.6: @ %for.body24.preheader 732; CHECK-NEXT: @ in Loop: Header=BB5_5 Depth=1 733; CHECK-NEXT: ldr r3, [sp, #84] 734; CHECK-NEXT: mov r6, r12 735; CHECK-NEXT: ldr r0, [sp, #12] @ 4-byte Reload 736; CHECK-NEXT: dlstp.16 lr, r3 737; CHECK-NEXT: ldr r1, [sp, #16] @ 4-byte Reload 738; CHECK-NEXT: mov r8, r12 739; CHECK-NEXT: mla r5, r11, r3, r0 740; CHECK-NEXT: ldr r0, [sp, #8] @ 4-byte Reload 741; CHECK-NEXT: ldrd r4, r7, [sp] @ 8-byte Folded Reload 742; CHECK-NEXT: mov r10, r12 743; CHECK-NEXT: .LBB5_7: @ %for.body24 744; CHECK-NEXT: @ Parent Loop BB5_5 Depth=1 745; CHECK-NEXT: @ => This Inner Loop Header: Depth=2 746; CHECK-NEXT: vldrb.s16 q1, [r4], #8 747; CHECK-NEXT: vadd.i16 q2, q1, q0 748; CHECK-NEXT: vldrb.s16 q1, [r5], #8 749; CHECK-NEXT: vmlava.s16 r12, q1, q2 750; CHECK-NEXT: vldrb.s16 q2, [r0], #8 751; CHECK-NEXT: vadd.i16 q2, q2, q0 752; CHECK-NEXT: vmlava.s16 r6, q1, q2 753; CHECK-NEXT: vldrb.s16 q2, [r7], #8 754; CHECK-NEXT: vadd.i16 q2, q2, q0 755; CHECK-NEXT: vmlava.s16 r8, q1, q2 756; CHECK-NEXT: vldrb.s16 q2, [r1], #8 757; CHECK-NEXT: vadd.i16 q2, q2, q0 758; CHECK-NEXT: vmlava.s16 r10, q1, q2 759; CHECK-NEXT: letp lr, .LBB5_7 760; CHECK-NEXT: b .LBB5_4 761; CHECK-NEXT: .LBB5_8: @ %if.end 762; CHECK-NEXT: ldr r0, [sp, #92] 763; CHECK-NEXT: add sp, #20 764; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} 765entry: 766 %cmp = icmp eq i16 %num_cols, 4 767 br i1 %cmp, label %for.cond.preheader, label %if.end 768 769for.cond.preheader: ; preds = %entry 770 %conv2 = zext i16 %output_ch to i32 771 %cmp3127 = icmp eq i16 %output_ch, 0 772 br i1 %cmp3127, label %if.end, label %for.body.lr.ph 773 774for.body.lr.ph: ; preds = %for.cond.preheader 775 %conv5 = zext i16 %row_len to i32 776 %add.ptr9 = getelementptr inbounds i8, i8* %input_col, i32 %conv5 777 %mul11 = shl nuw nsw i32 %conv5, 1 778 %add.ptr12 = getelementptr inbounds i8, i8* %input_col, i32 %mul11 779 %mul14 = mul nuw nsw i32 %conv5, 3 780 %add.ptr15 = getelementptr inbounds i8, i8* %input_col, i32 %mul14 781 %add = add nuw nsw i32 %conv5, 7 782 %div = lshr i32 %add, 3 783 %conv25 = trunc i32 %col_offset to i16 784 %.splatinsert.i = insertelement <8 x i16> undef, i16 %conv25, i32 0 785 %.splat.i = shufflevector <8 x i16> %.splatinsert.i, <8 x i16> undef, <8 x i32> zeroinitializer 786 br label %for.body 787 788for.body: ; preds = %for.cond.cleanup23, %for.body.lr.ph 789 %i_out_ch.0129 = phi i32 [ 0, %for.body.lr.ph ], [ %inc37, %for.cond.cleanup23 ] 790 %i_row_loop.0128 = phi i32 [ undef, %for.body.lr.ph ], [ %i_row_loop.1.lcssa, %for.cond.cleanup23 ] 791 %arrayidx = getelementptr inbounds i32, i32* %bias, i32 %i_out_ch.0129 792 %0 = load i32, i32* %arrayidx, align 4 793 %cmp21111 = icmp slt i32 %i_row_loop.0128, %div 794 br i1 %cmp21111, label %for.body24.preheader, label %for.cond.cleanup23 795 796for.body24.preheader: ; preds = %for.body 797 %mul = mul nuw nsw i32 %i_out_ch.0129, %conv5 798 %add.ptr = getelementptr inbounds i8, i8* %input_row, i32 %mul 799 br label %for.body24 800 801for.cond.cleanup23: ; preds = %for.body24, %for.body 802 %acc_0.0.lcssa = phi i32 [ %0, %for.body ], [ %21, %for.body24 ] 803 %acc_1.0.lcssa = phi i32 [ %0, %for.body ], [ %22, %for.body24 ] 804 %acc_2.0.lcssa = phi i32 [ %0, %for.body ], [ %23, %for.body24 ] 805 %acc_3.0.lcssa = phi i32 [ %0, %for.body ], [ %24, %for.body24 ] 806 %i_row_loop.1.lcssa = phi i32 [ %i_row_loop.0128, %for.body ], [ %div, %for.body24 ] 807 %add31 = add nsw i32 %acc_1.0.lcssa, %acc_0.0.lcssa 808 %add32 = add nsw i32 %add31, %acc_2.0.lcssa 809 %add33 = add nsw i32 %add32, %acc_3.0.lcssa 810 %conv34 = trunc i32 %add33 to i8 811 %arrayidx35 = getelementptr inbounds i8, i8* %out, i32 %i_out_ch.0129 812 store i8 %conv34, i8* %arrayidx35, align 1 813 %inc37 = add nuw nsw i32 %i_out_ch.0129, 1 814 %exitcond133 = icmp eq i32 %inc37, %conv2 815 br i1 %exitcond133, label %if.end, label %for.body 816 817for.body24: ; preds = %for.body24, %for.body24.preheader 818 %row_len_tmp.0122 = phi i32 [ %sub, %for.body24 ], [ %conv5, %for.body24.preheader ] 819 %ip_r0.0121 = phi i8* [ %add.ptr26, %for.body24 ], [ %add.ptr, %for.body24.preheader ] 820 %ip_c0.0120 = phi i8* [ %add.ptr27, %for.body24 ], [ %input_col, %for.body24.preheader ] 821 %ip_c1.0119 = phi i8* [ %add.ptr28, %for.body24 ], [ %add.ptr9, %for.body24.preheader ] 822 %ip_c2.0118 = phi i8* [ %add.ptr29, %for.body24 ], [ %add.ptr12, %for.body24.preheader ] 823 %i_row_loop.1117 = phi i32 [ %inc, %for.body24 ], [ %i_row_loop.0128, %for.body24.preheader ] 824 %ip_c3.0116 = phi i8* [ %add.ptr30, %for.body24 ], [ %add.ptr15, %for.body24.preheader ] 825 %acc_3.0115 = phi i32 [ %24, %for.body24 ], [ %0, %for.body24.preheader ] 826 %acc_2.0114 = phi i32 [ %23, %for.body24 ], [ %0, %for.body24.preheader ] 827 %acc_1.0113 = phi i32 [ %22, %for.body24 ], [ %0, %for.body24.preheader ] 828 %acc_0.0112 = phi i32 [ %21, %for.body24 ], [ %0, %for.body24.preheader ] 829 %1 = tail call <8 x i1> @llvm.arm.mve.vctp16(i32 %row_len_tmp.0122) 830 %sub = add nsw i32 %row_len_tmp.0122, -8 831 %2 = bitcast i8* %ip_r0.0121 to <8 x i8>* 832 %3 = tail call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %2, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer) 833 %4 = sext <8 x i8> %3 to <8 x i16> 834 %add.ptr26 = getelementptr inbounds i8, i8* %ip_r0.0121, i32 8 835 %5 = bitcast i8* %ip_c0.0120 to <8 x i8>* 836 %6 = tail call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %5, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer) 837 %7 = sext <8 x i8> %6 to <8 x i16> 838 %add.ptr27 = getelementptr inbounds i8, i8* %ip_c0.0120, i32 8 839 %8 = tail call <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16> %7, <8 x i16> %.splat.i, <8 x i1> %1, <8 x i16> undef) 840 %9 = bitcast i8* %ip_c1.0119 to <8 x i8>* 841 %10 = tail call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %9, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer) 842 %11 = sext <8 x i8> %10 to <8 x i16> 843 %add.ptr28 = getelementptr inbounds i8, i8* %ip_c1.0119, i32 8 844 %12 = tail call <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16> %11, <8 x i16> %.splat.i, <8 x i1> %1, <8 x i16> undef) 845 %13 = bitcast i8* %ip_c2.0118 to <8 x i8>* 846 %14 = tail call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %13, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer) 847 %15 = sext <8 x i8> %14 to <8 x i16> 848 %add.ptr29 = getelementptr inbounds i8, i8* %ip_c2.0118, i32 8 849 %16 = tail call <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16> %15, <8 x i16> %.splat.i, <8 x i1> %1, <8 x i16> undef) 850 %17 = bitcast i8* %ip_c3.0116 to <8 x i8>* 851 %18 = tail call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %17, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer) 852 %19 = sext <8 x i8> %18 to <8 x i16> 853 %add.ptr30 = getelementptr inbounds i8, i8* %ip_c3.0116, i32 8 854 %20 = tail call <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16> %19, <8 x i16> %.splat.i, <8 x i1> %1, <8 x i16> undef) 855 %21 = tail call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 %acc_0.0112, <8 x i16> %4, <8 x i16> %8, <8 x i1> %1) 856 %22 = tail call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 %acc_1.0113, <8 x i16> %4, <8 x i16> %12, <8 x i1> %1) 857 %23 = tail call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 %acc_2.0114, <8 x i16> %4, <8 x i16> %16, <8 x i1> %1) 858 %24 = tail call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 %acc_3.0115, <8 x i16> %4, <8 x i16> %20, <8 x i1> %1) 859 %inc = add nsw i32 %i_row_loop.1117, 1 860 %exitcond = icmp eq i32 %inc, %div 861 br i1 %exitcond, label %for.cond.cleanup23, label %for.body24 862 863if.end: ; preds = %for.cond.cleanup23, %for.cond.preheader, %entry 864 ret i8* %out 865} 866 867define i8* @signext_optsize(i8* %input_row, i8* %input_col, i16 zeroext %output_ch, i16 zeroext %num_cols, i32* nocapture readnone %output_shift, i32* nocapture readnone %output_mult, i32 %out_offset, i32 %col_offset, i32 %row_offset, i16 signext %activation_min, i16 signext %activation_max, i16 zeroext %row_len, i32* nocapture readonly %bias, i8* returned %out) optsize { 868; CHECK-LABEL: signext_optsize: 869; CHECK: @ %bb.0: @ %entry 870; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} 871; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} 872; CHECK-NEXT: .pad #20 873; CHECK-NEXT: sub sp, #20 874; CHECK-NEXT: cmp r3, #4 875; CHECK-NEXT: strd r0, r1, [sp, #12] @ 8-byte Folded Spill 876; CHECK-NEXT: bne .LBB6_8 877; CHECK-NEXT: @ %bb.1: @ %for.cond.preheader 878; CHECK-NEXT: cmp r2, #0 879; CHECK-NEXT: beq .LBB6_8 880; CHECK-NEXT: @ %bb.2: @ %for.body.lr.ph 881; CHECK-NEXT: ldr r7, [sp, #84] 882; CHECK-NEXT: mov.w r11, #0 883; CHECK-NEXT: ldr r3, [sp, #16] @ 4-byte Reload 884; CHECK-NEXT: ldr r0, [sp, #68] 885; CHECK-NEXT: add.w r1, r3, r7, lsl #1 886; CHECK-NEXT: str r1, [sp, #8] @ 4-byte Spill 887; CHECK-NEXT: adds r1, r3, r7 888; CHECK-NEXT: str r1, [sp, #4] @ 4-byte Spill 889; CHECK-NEXT: add.w r1, r7, r7, lsl #1 890; CHECK-NEXT: vdup.16 q0, r0 891; CHECK-NEXT: adds r0, r3, r1 892; CHECK-NEXT: str r0, [sp] @ 4-byte Spill 893; CHECK-NEXT: adds r0, r7, #7 894; CHECK-NEXT: lsr.w r9, r0, #3 895; CHECK-NEXT: .LBB6_3: @ %for.body 896; CHECK-NEXT: @ =>This Loop Header: Depth=1 897; CHECK-NEXT: @ Child Loop BB6_5 Depth 2 898; CHECK-NEXT: ldr r0, [sp, #88] 899; CHECK-NEXT: subs.w lr, r9, r9 900; CHECK-NEXT: ldr.w r12, [r0, r11, lsl #2] 901; CHECK-NEXT: ble .LBB6_6 902; CHECK-NEXT: @ %bb.4: @ %for.body24.preheader 903; CHECK-NEXT: @ in Loop: Header=BB6_3 Depth=1 904; CHECK-NEXT: ldr r3, [sp, #84] 905; CHECK-NEXT: mov r6, r12 906; CHECK-NEXT: ldr r0, [sp, #12] @ 4-byte Reload 907; CHECK-NEXT: dlstp.16 lr, r3 908; CHECK-NEXT: ldr r1, [sp, #16] @ 4-byte Reload 909; CHECK-NEXT: mov r8, r12 910; CHECK-NEXT: mla r5, r11, r3, r0 911; CHECK-NEXT: ldr r0, [sp, #8] @ 4-byte Reload 912; CHECK-NEXT: ldrd r4, r7, [sp] @ 8-byte Folded Reload 913; CHECK-NEXT: mov r10, r12 914; CHECK-NEXT: .LBB6_5: @ %for.body24 915; CHECK-NEXT: @ Parent Loop BB6_3 Depth=1 916; CHECK-NEXT: @ => This Inner Loop Header: Depth=2 917; CHECK-NEXT: vldrb.s16 q1, [r4], #8 918; CHECK-NEXT: vadd.i16 q2, q1, q0 919; CHECK-NEXT: vldrb.s16 q1, [r5], #8 920; CHECK-NEXT: vmlava.s16 r12, q1, q2 921; CHECK-NEXT: vldrb.s16 q2, [r0], #8 922; CHECK-NEXT: vadd.i16 q2, q2, q0 923; CHECK-NEXT: vmlava.s16 r6, q1, q2 924; CHECK-NEXT: vldrb.s16 q2, [r7], #8 925; CHECK-NEXT: vadd.i16 q2, q2, q0 926; CHECK-NEXT: vmlava.s16 r8, q1, q2 927; CHECK-NEXT: vldrb.s16 q2, [r1], #8 928; CHECK-NEXT: vadd.i16 q2, q2, q0 929; CHECK-NEXT: vmlava.s16 r10, q1, q2 930; CHECK-NEXT: letp lr, .LBB6_5 931; CHECK-NEXT: b .LBB6_7 932; CHECK-NEXT: .LBB6_6: @ in Loop: Header=BB6_3 Depth=1 933; CHECK-NEXT: mov r10, r12 934; CHECK-NEXT: mov r8, r12 935; CHECK-NEXT: mov r6, r12 936; CHECK-NEXT: .LBB6_7: @ %for.cond.cleanup23 937; CHECK-NEXT: @ in Loop: Header=BB6_3 Depth=1 938; CHECK-NEXT: ldr r1, [sp, #92] 939; CHECK-NEXT: add.w r0, r8, r10 940; CHECK-NEXT: add r0, r6 941; CHECK-NEXT: add r0, r12 942; CHECK-NEXT: strb.w r0, [r1, r11] 943; CHECK-NEXT: add.w r11, r11, #1 944; CHECK-NEXT: cmp r11, r2 945; CHECK-NEXT: bne .LBB6_3 946; CHECK-NEXT: .LBB6_8: @ %if.end 947; CHECK-NEXT: ldr r0, [sp, #92] 948; CHECK-NEXT: add sp, #20 949; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} 950entry: 951 %cmp = icmp eq i16 %num_cols, 4 952 br i1 %cmp, label %for.cond.preheader, label %if.end 953 954for.cond.preheader: ; preds = %entry 955 %conv2 = zext i16 %output_ch to i32 956 %cmp3127 = icmp eq i16 %output_ch, 0 957 br i1 %cmp3127, label %if.end, label %for.body.lr.ph 958 959for.body.lr.ph: ; preds = %for.cond.preheader 960 %conv5 = zext i16 %row_len to i32 961 %add.ptr9 = getelementptr inbounds i8, i8* %input_col, i32 %conv5 962 %mul11 = shl nuw nsw i32 %conv5, 1 963 %add.ptr12 = getelementptr inbounds i8, i8* %input_col, i32 %mul11 964 %mul14 = mul nuw nsw i32 %conv5, 3 965 %add.ptr15 = getelementptr inbounds i8, i8* %input_col, i32 %mul14 966 %add = add nuw nsw i32 %conv5, 7 967 %div = lshr i32 %add, 3 968 %conv25 = trunc i32 %col_offset to i16 969 %.splatinsert.i = insertelement <8 x i16> undef, i16 %conv25, i32 0 970 %.splat.i = shufflevector <8 x i16> %.splatinsert.i, <8 x i16> undef, <8 x i32> zeroinitializer 971 br label %for.body 972 973for.body: ; preds = %for.cond.cleanup23, %for.body.lr.ph 974 %i_out_ch.0129 = phi i32 [ 0, %for.body.lr.ph ], [ %inc37, %for.cond.cleanup23 ] 975 %i_row_loop.0128 = phi i32 [ undef, %for.body.lr.ph ], [ %i_row_loop.1.lcssa, %for.cond.cleanup23 ] 976 %arrayidx = getelementptr inbounds i32, i32* %bias, i32 %i_out_ch.0129 977 %0 = load i32, i32* %arrayidx, align 4 978 %cmp21111 = icmp slt i32 %i_row_loop.0128, %div 979 br i1 %cmp21111, label %for.body24.preheader, label %for.cond.cleanup23 980 981for.body24.preheader: ; preds = %for.body 982 %mul = mul nuw nsw i32 %i_out_ch.0129, %conv5 983 %add.ptr = getelementptr inbounds i8, i8* %input_row, i32 %mul 984 br label %for.body24 985 986for.cond.cleanup23: ; preds = %for.body24, %for.body 987 %acc_0.0.lcssa = phi i32 [ %0, %for.body ], [ %21, %for.body24 ] 988 %acc_1.0.lcssa = phi i32 [ %0, %for.body ], [ %22, %for.body24 ] 989 %acc_2.0.lcssa = phi i32 [ %0, %for.body ], [ %23, %for.body24 ] 990 %acc_3.0.lcssa = phi i32 [ %0, %for.body ], [ %24, %for.body24 ] 991 %i_row_loop.1.lcssa = phi i32 [ %i_row_loop.0128, %for.body ], [ %div, %for.body24 ] 992 %add31 = add nsw i32 %acc_1.0.lcssa, %acc_0.0.lcssa 993 %add32 = add nsw i32 %add31, %acc_2.0.lcssa 994 %add33 = add nsw i32 %add32, %acc_3.0.lcssa 995 %conv34 = trunc i32 %add33 to i8 996 %arrayidx35 = getelementptr inbounds i8, i8* %out, i32 %i_out_ch.0129 997 store i8 %conv34, i8* %arrayidx35, align 1 998 %inc37 = add nuw nsw i32 %i_out_ch.0129, 1 999 %exitcond133 = icmp eq i32 %inc37, %conv2 1000 br i1 %exitcond133, label %if.end, label %for.body 1001 1002for.body24: ; preds = %for.body24, %for.body24.preheader 1003 %row_len_tmp.0122 = phi i32 [ %sub, %for.body24 ], [ %conv5, %for.body24.preheader ] 1004 %ip_r0.0121 = phi i8* [ %add.ptr26, %for.body24 ], [ %add.ptr, %for.body24.preheader ] 1005 %ip_c0.0120 = phi i8* [ %add.ptr27, %for.body24 ], [ %input_col, %for.body24.preheader ] 1006 %ip_c1.0119 = phi i8* [ %add.ptr28, %for.body24 ], [ %add.ptr9, %for.body24.preheader ] 1007 %ip_c2.0118 = phi i8* [ %add.ptr29, %for.body24 ], [ %add.ptr12, %for.body24.preheader ] 1008 %i_row_loop.1117 = phi i32 [ %inc, %for.body24 ], [ %i_row_loop.0128, %for.body24.preheader ] 1009 %ip_c3.0116 = phi i8* [ %add.ptr30, %for.body24 ], [ %add.ptr15, %for.body24.preheader ] 1010 %acc_3.0115 = phi i32 [ %24, %for.body24 ], [ %0, %for.body24.preheader ] 1011 %acc_2.0114 = phi i32 [ %23, %for.body24 ], [ %0, %for.body24.preheader ] 1012 %acc_1.0113 = phi i32 [ %22, %for.body24 ], [ %0, %for.body24.preheader ] 1013 %acc_0.0112 = phi i32 [ %21, %for.body24 ], [ %0, %for.body24.preheader ] 1014 %1 = tail call <8 x i1> @llvm.arm.mve.vctp16(i32 %row_len_tmp.0122) 1015 %sub = add nsw i32 %row_len_tmp.0122, -8 1016 %2 = bitcast i8* %ip_r0.0121 to <8 x i8>* 1017 %3 = tail call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %2, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer) 1018 %4 = sext <8 x i8> %3 to <8 x i16> 1019 %add.ptr26 = getelementptr inbounds i8, i8* %ip_r0.0121, i32 8 1020 %5 = bitcast i8* %ip_c0.0120 to <8 x i8>* 1021 %6 = tail call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %5, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer) 1022 %7 = sext <8 x i8> %6 to <8 x i16> 1023 %add.ptr27 = getelementptr inbounds i8, i8* %ip_c0.0120, i32 8 1024 %8 = tail call <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16> %7, <8 x i16> %.splat.i, <8 x i1> %1, <8 x i16> undef) 1025 %9 = bitcast i8* %ip_c1.0119 to <8 x i8>* 1026 %10 = tail call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %9, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer) 1027 %11 = sext <8 x i8> %10 to <8 x i16> 1028 %add.ptr28 = getelementptr inbounds i8, i8* %ip_c1.0119, i32 8 1029 %12 = tail call <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16> %11, <8 x i16> %.splat.i, <8 x i1> %1, <8 x i16> undef) 1030 %13 = bitcast i8* %ip_c2.0118 to <8 x i8>* 1031 %14 = tail call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %13, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer) 1032 %15 = sext <8 x i8> %14 to <8 x i16> 1033 %add.ptr29 = getelementptr inbounds i8, i8* %ip_c2.0118, i32 8 1034 %16 = tail call <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16> %15, <8 x i16> %.splat.i, <8 x i1> %1, <8 x i16> undef) 1035 %17 = bitcast i8* %ip_c3.0116 to <8 x i8>* 1036 %18 = tail call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* %17, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer) 1037 %19 = sext <8 x i8> %18 to <8 x i16> 1038 %add.ptr30 = getelementptr inbounds i8, i8* %ip_c3.0116, i32 8 1039 %20 = tail call <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16> %19, <8 x i16> %.splat.i, <8 x i1> %1, <8 x i16> undef) 1040 %21 = tail call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 %acc_0.0112, <8 x i16> %4, <8 x i16> %8, <8 x i1> %1) 1041 %22 = tail call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 %acc_1.0113, <8 x i16> %4, <8 x i16> %12, <8 x i1> %1) 1042 %23 = tail call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 %acc_2.0114, <8 x i16> %4, <8 x i16> %16, <8 x i1> %1) 1043 %24 = tail call i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32 0, i32 0, i32 0, i32 %acc_3.0115, <8 x i16> %4, <8 x i16> %20, <8 x i1> %1) 1044 %inc = add nsw i32 %i_row_loop.1117, 1 1045 %exitcond = icmp eq i32 %inc, %div 1046 br i1 %exitcond, label %for.cond.cleanup23, label %for.body24 1047 1048if.end: ; preds = %for.cond.cleanup23, %for.cond.preheader, %entry 1049 ret i8* %out 1050} 1051 1052%struct.arm_cfft_instance_f32 = type { i16, float*, i16*, i16, i32*, i32*, i32*, float*, float*, float* } 1053define arm_aapcs_vfpcc void @_Z37_arm_radix4_butterfly_inverse_f32_mvePK21arm_cfft_instance_f32Pfjf(%struct.arm_cfft_instance_f32* nocapture readonly %0, float* %1, i32 %2, float %3) { 1054; CHECK-LABEL: _Z37_arm_radix4_butterfly_inverse_f32_mvePK21arm_cfft_instance_f32Pfjf: 1055; CHECK: @ %bb.0: 1056; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} 1057; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} 1058; CHECK-NEXT: .pad #4 1059; CHECK-NEXT: sub sp, #4 1060; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15} 1061; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15} 1062; CHECK-NEXT: .pad #56 1063; CHECK-NEXT: sub sp, #56 1064; CHECK-NEXT: cmp r2, #8 1065; CHECK-NEXT: str r1, [sp, #20] @ 4-byte Spill 1066; CHECK-NEXT: vstr s0, [sp, #4] @ 4-byte Spill 1067; CHECK-NEXT: mov r1, r2 1068; CHECK-NEXT: str r2, [sp, #8] @ 4-byte Spill 1069; CHECK-NEXT: blo.w .LBB7_9 1070; CHECK-NEXT: @ %bb.1: 1071; CHECK-NEXT: ldr r2, [sp, #8] @ 4-byte Reload 1072; CHECK-NEXT: movs r3, #1 1073; CHECK-NEXT: mov.w r10, #0 1074; CHECK-NEXT: str r2, [sp, #16] @ 4-byte Spill 1075; CHECK-NEXT: lsrs r1, r2, #2 1076; CHECK-NEXT: b .LBB7_3 1077; CHECK-NEXT: .LBB7_2: @ in Loop: Header=BB7_3 Depth=1 1078; CHECK-NEXT: ldr r2, [sp, #12] @ 4-byte Reload 1079; CHECK-NEXT: add.w r10, r10, #1 1080; CHECK-NEXT: lsls r3, r3, #2 1081; CHECK-NEXT: cmp r2, #7 1082; CHECK-NEXT: asr.w r1, r2, #2 1083; CHECK-NEXT: ble .LBB7_9 1084; CHECK-NEXT: .LBB7_3: @ =>This Loop Header: Depth=1 1085; CHECK-NEXT: @ Child Loop BB7_6 Depth 2 1086; CHECK-NEXT: @ Child Loop BB7_7 Depth 3 1087; CHECK-NEXT: str r1, [sp, #12] @ 4-byte Spill 1088; CHECK-NEXT: cmp r3, #1 1089; CHECK-NEXT: ldr r1, [sp, #16] @ 4-byte Reload 1090; CHECK-NEXT: lsr.w r2, r1, #2 1091; CHECK-NEXT: str r2, [sp, #16] @ 4-byte Spill 1092; CHECK-NEXT: blt .LBB7_2 1093; CHECK-NEXT: @ %bb.4: @ in Loop: Header=BB7_3 Depth=1 1094; CHECK-NEXT: movs r2, #0 1095; CHECK-NEXT: cmp.w r2, r1, lsr #3 1096; CHECK-NEXT: beq .LBB7_2 1097; CHECK-NEXT: @ %bb.5: @ %.preheader 1098; CHECK-NEXT: @ in Loop: Header=BB7_3 Depth=1 1099; CHECK-NEXT: lsrs r2, r1, #3 1100; CHECK-NEXT: lsls r1, r1, #1 1101; CHECK-NEXT: str r2, [sp, #28] @ 4-byte Spill 1102; CHECK-NEXT: movs r5, #0 1103; CHECK-NEXT: ldr r2, [sp, #16] @ 4-byte Reload 1104; CHECK-NEXT: str r1, [sp, #24] @ 4-byte Spill 1105; CHECK-NEXT: str r3, [sp, #32] @ 4-byte Spill 1106; CHECK-NEXT: lsl.w r11, r2, #1 1107; CHECK-NEXT: .LBB7_6: @ Parent Loop BB7_3 Depth=1 1108; CHECK-NEXT: @ => This Loop Header: Depth=2 1109; CHECK-NEXT: @ Child Loop BB7_7 Depth 3 1110; CHECK-NEXT: add.w r12, r0, #16 1111; CHECK-NEXT: ldr r4, [sp, #24] @ 4-byte Reload 1112; CHECK-NEXT: ldr.w lr, [sp, #28] @ 4-byte Reload 1113; CHECK-NEXT: ldm.w r12, {r1, r2, r3, r12} 1114; CHECK-NEXT: muls r4, r5, r4 1115; CHECK-NEXT: ldr.w r2, [r2, r10, lsl #2] 1116; CHECK-NEXT: ldr.w r1, [r1, r10, lsl #2] 1117; CHECK-NEXT: ldrd r6, r7, [r0, #32] 1118; CHECK-NEXT: ldr.w r3, [r3, r10, lsl #2] 1119; CHECK-NEXT: dls lr, lr 1120; CHECK-NEXT: add.w r6, r6, r2, lsl #2 1121; CHECK-NEXT: add.w r12, r12, r1, lsl #2 1122; CHECK-NEXT: ldr r1, [sp, #20] @ 4-byte Reload 1123; CHECK-NEXT: add.w r2, r1, r4, lsl #2 1124; CHECK-NEXT: add.w r3, r7, r3, lsl #2 1125; CHECK-NEXT: add.w r1, r2, r11, lsl #2 1126; CHECK-NEXT: add.w r8, r1, r11, lsl #2 1127; CHECK-NEXT: add.w r9, r8, r11, lsl #2 1128; CHECK-NEXT: .LBB7_7: @ Parent Loop BB7_3 Depth=1 1129; CHECK-NEXT: @ Parent Loop BB7_6 Depth=2 1130; CHECK-NEXT: @ => This Inner Loop Header: Depth=3 1131; CHECK-NEXT: vldrw.u32 q3, [r9] 1132; CHECK-NEXT: vldrw.u32 q4, [r1] 1133; CHECK-NEXT: vldrw.u32 q6, [r8] 1134; CHECK-NEXT: vldrw.u32 q7, [r2] 1135; CHECK-NEXT: vsub.f32 q5, q4, q3 1136; CHECK-NEXT: vsub.f32 q0, q7, q6 1137; CHECK-NEXT: vcadd.f32 q1, q0, q5, #270 1138; CHECK-NEXT: vcadd.f32 q2, q0, q5, #90 1139; CHECK-NEXT: vadd.f32 q0, q4, q3 1140; CHECK-NEXT: vadd.f32 q3, q6, q7 1141; CHECK-NEXT: vsub.f32 q4, q3, q0 1142; CHECK-NEXT: vadd.f32 q0, q3, q0 1143; CHECK-NEXT: vstrb.8 q0, [r2], #16 1144; CHECK-NEXT: vldrw.u32 q0, [r6], #16 1145; CHECK-NEXT: vcmul.f32 q3, q0, q4, #0 1146; CHECK-NEXT: vcmla.f32 q3, q0, q4, #90 1147; CHECK-NEXT: vstrb.8 q3, [r1], #16 1148; CHECK-NEXT: vldrw.u32 q0, [r12], #16 1149; CHECK-NEXT: vcmul.f32 q3, q0, q2, #0 1150; CHECK-NEXT: vcmla.f32 q3, q0, q2, #90 1151; CHECK-NEXT: vstrb.8 q3, [r8], #16 1152; CHECK-NEXT: vldrw.u32 q0, [r3], #16 1153; CHECK-NEXT: vcmul.f32 q2, q0, q1, #0 1154; CHECK-NEXT: vcmla.f32 q2, q0, q1, #90 1155; CHECK-NEXT: vstrb.8 q2, [r9], #16 1156; CHECK-NEXT: le lr, .LBB7_7 1157; CHECK-NEXT: @ %bb.8: @ in Loop: Header=BB7_6 Depth=2 1158; CHECK-NEXT: ldr r3, [sp, #32] @ 4-byte Reload 1159; CHECK-NEXT: adds r5, #1 1160; CHECK-NEXT: cmp r5, r3 1161; CHECK-NEXT: bne .LBB7_6 1162; CHECK-NEXT: b .LBB7_2 1163; CHECK-NEXT: .LBB7_9: 1164; CHECK-NEXT: adr r0, .LCPI7_0 1165; CHECK-NEXT: vldrw.u32 q1, [r0] 1166; CHECK-NEXT: ldr r0, [sp, #20] @ 4-byte Reload 1167; CHECK-NEXT: vadd.i32 q1, q1, r0 1168; CHECK-NEXT: vldrw.u32 q2, [q1, #64]! 1169; CHECK-NEXT: ldr r0, [sp, #8] @ 4-byte Reload 1170; CHECK-NEXT: lsr.w lr, r0, #3 1171; CHECK-NEXT: wls lr, lr, .LBB7_12 1172; CHECK-NEXT: @ %bb.10: 1173; CHECK-NEXT: vldr s0, [sp, #4] @ 4-byte Reload 1174; CHECK-NEXT: vmov r0, s0 1175; CHECK-NEXT: vldrw.u32 q0, [q1, #16] 1176; CHECK-NEXT: .LBB7_11: @ =>This Inner Loop Header: Depth=1 1177; CHECK-NEXT: vldrw.u32 q3, [q1, #24] 1178; CHECK-NEXT: vldrw.u32 q4, [q1, #8] 1179; CHECK-NEXT: vsub.f32 q6, q2, q0 1180; CHECK-NEXT: vadd.f32 q0, q2, q0 1181; CHECK-NEXT: vsub.f32 q5, q4, q3 1182; CHECK-NEXT: vadd.f32 q3, q4, q3 1183; CHECK-NEXT: vcadd.f32 q7, q6, q5, #270 1184; CHECK-NEXT: vsub.f32 q2, q0, q3 1185; CHECK-NEXT: vmul.f32 q7, q7, r0 1186; CHECK-NEXT: vadd.f32 q3, q0, q3 1187; CHECK-NEXT: vstrw.32 q7, [sp, #32] @ 16-byte Spill 1188; CHECK-NEXT: vcadd.f32 q7, q6, q5, #90 1189; CHECK-NEXT: vmul.f32 q4, q2, r0 1190; CHECK-NEXT: vldrw.u32 q2, [q1, #64]! 1191; CHECK-NEXT: vmul.f32 q5, q7, r0 1192; CHECK-NEXT: vmul.f32 q3, q3, r0 1193; CHECK-NEXT: vldrw.u32 q0, [q1, #16] 1194; CHECK-NEXT: vstrw.32 q3, [q1, #-64] 1195; CHECK-NEXT: vstrw.32 q4, [q1, #-56] 1196; CHECK-NEXT: vstrw.32 q5, [q1, #-48] 1197; CHECK-NEXT: vldrw.u32 q3, [sp, #32] @ 16-byte Reload 1198; CHECK-NEXT: vstrw.32 q3, [q1, #-40] 1199; CHECK-NEXT: le lr, .LBB7_11 1200; CHECK-NEXT: .LBB7_12: 1201; CHECK-NEXT: add sp, #56 1202; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} 1203; CHECK-NEXT: add sp, #4 1204; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} 1205; CHECK-NEXT: .p2align 4 1206; CHECK-NEXT: @ %bb.13: 1207; CHECK-NEXT: .LCPI7_0: 1208; CHECK-NEXT: .long 4294967232 @ 0xffffffc0 1209; CHECK-NEXT: .long 4294967236 @ 0xffffffc4 1210; CHECK-NEXT: .long 4294967264 @ 0xffffffe0 1211; CHECK-NEXT: .long 4294967268 @ 0xffffffe4 1212 %5 = icmp ugt i32 %2, 7 1213 br i1 %5, label %6, label %26 1214 12156: ; preds = %4 1216 %7 = lshr i32 %2, 2 1217 %8 = getelementptr inbounds %struct.arm_cfft_instance_f32, %struct.arm_cfft_instance_f32* %0, i32 0, i32 7 1218 %9 = getelementptr inbounds %struct.arm_cfft_instance_f32, %struct.arm_cfft_instance_f32* %0, i32 0, i32 4 1219 %10 = getelementptr inbounds %struct.arm_cfft_instance_f32, %struct.arm_cfft_instance_f32* %0, i32 0, i32 8 1220 %11 = getelementptr inbounds %struct.arm_cfft_instance_f32, %struct.arm_cfft_instance_f32* %0, i32 0, i32 5 1221 %12 = getelementptr inbounds %struct.arm_cfft_instance_f32, %struct.arm_cfft_instance_f32* %0, i32 0, i32 9 1222 %13 = getelementptr inbounds %struct.arm_cfft_instance_f32, %struct.arm_cfft_instance_f32* %0, i32 0, i32 6 1223 br label %14 1224 122514: ; preds = %6, %40 1226 %15 = phi i32 [ %2, %6 ], [ %19, %40 ] 1227 %16 = phi i32 [ %7, %6 ], [ %43, %40 ] 1228 %17 = phi i32 [ 1, %6 ], [ %41, %40 ] 1229 %18 = phi i32 [ 0, %6 ], [ %42, %40 ] 1230 %19 = lshr i32 %15, 2 1231 %20 = icmp sgt i32 %17, 0 1232 br i1 %20, label %21, label %40 1233 123421: ; preds = %14 1235 %22 = shl i32 %15, 1 1236 %23 = shl nuw nsw i32 %19, 1 1237 %24 = lshr i32 %15, 3 1238 %25 = icmp eq i32 %24, 0 1239 br i1 %25, label %40, label %45 1240 124126: ; preds = %40, %4 1242 %27 = ptrtoint float* %1 to i32 1243 %28 = insertelement <4 x i32> undef, i32 %27, i32 0 1244 %29 = shufflevector <4 x i32> %28, <4 x i32> undef, <4 x i32> zeroinitializer 1245 %30 = add <4 x i32> %29, <i32 -64, i32 -60, i32 -32, i32 -28> 1246 %31 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32> %30, i32 64) 1247 %32 = extractvalue { <4 x float>, <4 x i32> } %31, 1 1248 %33 = lshr i32 %2, 3 1249 %34 = icmp eq i32 %33, 0 1250 br i1 %34, label %141, label %35 1251 125235: ; preds = %26 1253 %36 = tail call <4 x float> @llvm.arm.mve.vldr.gather.base.v4f32.v4i32(<4 x i32> %32, i32 16) 1254 %37 = extractvalue { <4 x float>, <4 x i32> } %31, 0 1255 %38 = insertelement <4 x float> undef, float %3, i32 0 1256 %39 = shufflevector <4 x float> %38, <4 x float> undef, <4 x i32> zeroinitializer 1257 br label %116 1258 125940: ; preds = %113, %21, %14 1260 %41 = shl i32 %17, 2 1261 %42 = add nuw nsw i32 %18, 1 1262 %43 = ashr i32 %16, 2 1263 %44 = icmp sgt i32 %16, 7 1264 br i1 %44, label %14, label %26 1265 126645: ; preds = %21, %113 1267 %46 = phi i32 [ %114, %113 ], [ 0, %21 ] 1268 %47 = load float*, float** %8, align 4 1269 %48 = load i32*, i32** %9, align 4 1270 %49 = getelementptr inbounds i32, i32* %48, i32 %18 1271 %50 = load i32, i32* %49, align 4 1272 %51 = getelementptr inbounds float, float* %47, i32 %50 1273 %52 = load float*, float** %10, align 4 1274 %53 = load i32*, i32** %11, align 4 1275 %54 = getelementptr inbounds i32, i32* %53, i32 %18 1276 %55 = load i32, i32* %54, align 4 1277 %56 = getelementptr inbounds float, float* %52, i32 %55 1278 %57 = load float*, float** %12, align 4 1279 %58 = load i32*, i32** %13, align 4 1280 %59 = getelementptr inbounds i32, i32* %58, i32 %18 1281 %60 = load i32, i32* %59, align 4 1282 %61 = getelementptr inbounds float, float* %57, i32 %60 1283 %62 = mul i32 %22, %46 1284 %63 = getelementptr inbounds float, float* %1, i32 %62 1285 %64 = getelementptr inbounds float, float* %63, i32 %23 1286 %65 = getelementptr inbounds float, float* %64, i32 %23 1287 %66 = getelementptr inbounds float, float* %65, i32 %23 1288 br label %67 1289 129067: ; preds = %45, %67 1291 %68 = phi float* [ %63, %45 ], [ %89, %67 ] 1292 %69 = phi float* [ %65, %45 ], [ %103, %67 ] 1293 %70 = phi float* [ %66, %45 ], [ %110, %67 ] 1294 %71 = phi float* [ %64, %45 ], [ %96, %67 ] 1295 %72 = phi float* [ %61, %45 ], [ %107, %67 ] 1296 %73 = phi float* [ %56, %45 ], [ %93, %67 ] 1297 %74 = phi float* [ %51, %45 ], [ %100, %67 ] 1298 %75 = phi i32 [ %24, %45 ], [ %111, %67 ] 1299 %76 = bitcast float* %69 to <4 x float>* 1300 %77 = bitcast float* %68 to <4 x float>* 1301 %78 = load <4 x float>, <4 x float>* %76, align 4 1302 %79 = load <4 x float>, <4 x float>* %77, align 4 1303 %80 = bitcast float* %71 to <4 x float>* 1304 %81 = load <4 x float>, <4 x float>* %80, align 4 1305 %82 = bitcast float* %70 to <4 x float>* 1306 %83 = load <4 x float>, <4 x float>* %82, align 4 1307 %84 = fadd <4 x float> %78, %79 1308 %85 = fsub <4 x float> %79, %78 1309 %86 = fadd <4 x float> %81, %83 1310 %87 = fsub <4 x float> %81, %83 1311 %88 = fadd <4 x float> %84, %86 1312 store <4 x float> %88, <4 x float>* %77, align 4 1313 %89 = getelementptr inbounds float, float* %68, i32 4 1314 %90 = fsub <4 x float> %84, %86 1315 %91 = bitcast float* %73 to <4 x float>* 1316 %92 = load <4 x float>, <4 x float>* %91, align 4 1317 %93 = getelementptr inbounds float, float* %73, i32 4 1318 %94 = tail call <4 x float> @llvm.arm.mve.vcmulq.v4f32(i32 0, <4 x float> %92, <4 x float> %90) 1319 %95 = tail call <4 x float> @llvm.arm.mve.vcmlaq.v4f32(i32 1, <4 x float> %94, <4 x float> %92, <4 x float> %90) 1320 store <4 x float> %95, <4 x float>* %80, align 4 1321 %96 = getelementptr inbounds float, float* %71, i32 4 1322 %97 = tail call <4 x float> @llvm.arm.mve.vcaddq.v4f32(i32 1, i32 0, <4 x float> %85, <4 x float> %87) 1323 %98 = bitcast float* %74 to <4 x float>* 1324 %99 = load <4 x float>, <4 x float>* %98, align 4 1325 %100 = getelementptr inbounds float, float* %74, i32 4 1326 %101 = tail call <4 x float> @llvm.arm.mve.vcmulq.v4f32(i32 0, <4 x float> %99, <4 x float> %97) 1327 %102 = tail call <4 x float> @llvm.arm.mve.vcmlaq.v4f32(i32 1, <4 x float> %101, <4 x float> %99, <4 x float> %97) 1328 store <4 x float> %102, <4 x float>* %76, align 4 1329 %103 = getelementptr inbounds float, float* %69, i32 4 1330 %104 = tail call <4 x float> @llvm.arm.mve.vcaddq.v4f32(i32 1, i32 1, <4 x float> %85, <4 x float> %87) 1331 %105 = bitcast float* %72 to <4 x float>* 1332 %106 = load <4 x float>, <4 x float>* %105, align 4 1333 %107 = getelementptr inbounds float, float* %72, i32 4 1334 %108 = tail call <4 x float> @llvm.arm.mve.vcmulq.v4f32(i32 0, <4 x float> %106, <4 x float> %104) 1335 %109 = tail call <4 x float> @llvm.arm.mve.vcmlaq.v4f32(i32 1, <4 x float> %108, <4 x float> %106, <4 x float> %104) 1336 store <4 x float> %109, <4 x float>* %82, align 4 1337 %110 = getelementptr inbounds float, float* %70, i32 4 1338 %111 = add nsw i32 %75, -1 1339 %112 = icmp eq i32 %111, 0 1340 br i1 %112, label %113, label %67 1341 1342113: ; preds = %67 1343 %114 = add nuw nsw i32 %46, 1 1344 %115 = icmp eq i32 %114, %17 1345 br i1 %115, label %40, label %45 1346 1347116: ; preds = %35, %116 1348 %117 = phi <4 x i32> [ %32, %35 ], [ %128, %116 ] 1349 %118 = phi i32 [ %33, %35 ], [ %139, %116 ] 1350 %119 = phi <4 x float> [ %36, %35 ], [ %130, %116 ] 1351 %120 = phi <4 x float> [ %37, %35 ], [ %129, %116 ] 1352 %121 = fadd <4 x float> %120, %119 1353 %122 = fsub <4 x float> %120, %119 1354 %123 = tail call <4 x float> @llvm.arm.mve.vldr.gather.base.v4f32.v4i32(<4 x i32> %117, i32 8) 1355 %124 = tail call <4 x float> @llvm.arm.mve.vldr.gather.base.v4f32.v4i32(<4 x i32> %117, i32 24) 1356 %125 = fadd <4 x float> %123, %124 1357 %126 = fsub <4 x float> %123, %124 1358 %127 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32> %117, i32 64) 1359 %128 = extractvalue { <4 x float>, <4 x i32> } %127, 1 1360 %129 = extractvalue { <4 x float>, <4 x i32> } %127, 0 1361 %130 = tail call <4 x float> @llvm.arm.mve.vldr.gather.base.v4f32.v4i32(<4 x i32> %128, i32 16) 1362 %131 = fadd <4 x float> %121, %125 1363 %132 = fmul <4 x float> %39, %131 1364 tail call void @llvm.arm.mve.vstr.scatter.base.v4i32.v4f32(<4 x i32> %128, i32 -64, <4 x float> %132) 1365 %133 = fsub <4 x float> %121, %125 1366 %134 = fmul <4 x float> %39, %133 1367 tail call void @llvm.arm.mve.vstr.scatter.base.v4i32.v4f32(<4 x i32> %128, i32 -56, <4 x float> %134) 1368 %135 = tail call <4 x float> @llvm.arm.mve.vcaddq.v4f32(i32 1, i32 0, <4 x float> %122, <4 x float> %126) 1369 %136 = fmul <4 x float> %39, %135 1370 tail call void @llvm.arm.mve.vstr.scatter.base.v4i32.v4f32(<4 x i32> %128, i32 -48, <4 x float> %136) 1371 %137 = tail call <4 x float> @llvm.arm.mve.vcaddq.v4f32(i32 1, i32 1, <4 x float> %122, <4 x float> %126) 1372 %138 = fmul <4 x float> %39, %137 1373 tail call void @llvm.arm.mve.vstr.scatter.base.v4i32.v4f32(<4 x i32> %128, i32 -40, <4 x float> %138) 1374 %139 = add nsw i32 %118, -1 1375 %140 = icmp eq i32 %139, 0 1376 br i1 %140, label %141, label %116 1377 1378141: ; preds = %116, %26 1379 ret void 1380} 1381 1382declare <16 x i1> @llvm.arm.mve.vctp8(i32) 1383declare <8 x i1> @llvm.arm.mve.vctp16(i32) 1384declare i32 @llvm.arm.mve.pred.v2i.v16i1(<16 x i1>) 1385declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32 immarg, <4 x i1>, <4 x float>) 1386declare <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>*, i32, <8 x i1>, <8 x i8>) 1387declare <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>*, i32 immarg, <16 x i1>, <16 x i8>) 1388declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32 immarg, <4 x i1>) 1389declare i32 @llvm.experimental.vector.reduce.add.v16i8(<16 x i32> %ext4) 1390declare i32 @llvm.arm.mve.vmldava.v8i16(i32, i32, i32, i32, <8 x i16>, <8 x i16>) 1391declare i32 @llvm.arm.mve.vmldava.predicated.v16i8.v16i1(i32, i32, i32, i32, <16 x i8>, <16 x i8>, <16 x i1>) 1392declare i32 @llvm.arm.mve.vmldava.predicated.v8i16.v8i1(i32, i32, i32, i32, <8 x i16>, <8 x i16>, <8 x i1>) 1393declare <8 x i16> @llvm.arm.mve.add.predicated.v8i16.v8i1(<8 x i16>, <8 x i16>, <8 x i1>, <8 x i16>) 1394 1395declare <4 x float> @llvm.arm.mve.vcmulq.v4f32(i32, <4 x float>, <4 x float>) 1396declare <4 x float> @llvm.arm.mve.vcmlaq.v4f32(i32, <4 x float>, <4 x float>, <4 x float>) 1397declare <4 x float> @llvm.arm.mve.vcaddq.v4f32(i32, i32, <4 x float>, <4 x float>) 1398declare { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32>, i32) 1399declare <4 x float> @llvm.arm.mve.vldr.gather.base.v4f32.v4i32(<4 x i32>, i32) 1400declare void @llvm.arm.mve.vstr.scatter.base.v4i32.v4f32(<4 x i32>, i32, <4 x float>) 1401