1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck  %s
3; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
4
5define void @transpose_multiply(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr, <9 x double>* %C.Ptr) {
6; CHECK-LABEL: @transpose_multiply(
7; CHECK-NEXT:  entry:
8; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x double>* [[A_PTR:%.*]] to double*
9; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
10; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST]], align 8
11; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 3
12; CHECK-NEXT:    [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
13; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST1]], align 8
14; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr double, double* [[TMP0]], i64 6
15; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <3 x double>*
16; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST4]], align 8
17; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <9 x double>* [[B_PTR:%.*]] to double*
18; CHECK-NEXT:    [[VEC_CAST6:%.*]] = bitcast double* [[TMP1]] to <3 x double>*
19; CHECK-NEXT:    [[COL_LOAD7:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST6]], align 8
20; CHECK-NEXT:    [[VEC_GEP8:%.*]] = getelementptr double, double* [[TMP1]], i64 3
21; CHECK-NEXT:    [[VEC_CAST9:%.*]] = bitcast double* [[VEC_GEP8]] to <3 x double>*
22; CHECK-NEXT:    [[COL_LOAD10:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST9]], align 8
23; CHECK-NEXT:    [[VEC_GEP11:%.*]] = getelementptr double, double* [[TMP1]], i64 6
24; CHECK-NEXT:    [[VEC_CAST12:%.*]] = bitcast double* [[VEC_GEP11]] to <3 x double>*
25; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST12]], align 8
26; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
27; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x double> undef, double [[TMP2]], i64 0
28; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 0
29; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 1
30; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 0
31; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <3 x double> [[TMP5]], double [[TMP6]], i64 2
32; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 1
33; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x double> undef, double [[TMP8]], i64 0
34; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 1
35; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 1
36; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 1
37; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <3 x double> [[TMP11]], double [[TMP12]], i64 2
38; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 2
39; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <3 x double> undef, double [[TMP14]], i64 0
40; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 2
41; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 1
42; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 2
43; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <3 x double> [[TMP17]], double [[TMP18]], i64 2
44; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
45; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
46; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP20]], i32 0
47; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
48; CHECK-NEXT:    [[TMP21:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
49; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
50; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
51; CHECK-NEXT:    [[SPLAT_SPLATINSERT15:%.*]] = insertelement <1 x double> poison, double [[TMP22]], i32 0
52; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT15]], <1 x double> poison, <1 x i32> zeroinitializer
53; CHECK-NEXT:    [[TMP23:%.*]] = fmul <1 x double> [[BLOCK14]], [[SPLAT_SPLAT16]]
54; CHECK-NEXT:    [[TMP24:%.*]] = fadd <1 x double> [[TMP21]], [[TMP23]]
55; CHECK-NEXT:    [[BLOCK17:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
56; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
57; CHECK-NEXT:    [[SPLAT_SPLATINSERT18:%.*]] = insertelement <1 x double> poison, double [[TMP25]], i32 0
58; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT18]], <1 x double> poison, <1 x i32> zeroinitializer
59; CHECK-NEXT:    [[TMP26:%.*]] = fmul <1 x double> [[BLOCK17]], [[SPLAT_SPLAT19]]
60; CHECK-NEXT:    [[TMP27:%.*]] = fadd <1 x double> [[TMP24]], [[TMP26]]
61; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <1 x double> [[TMP27]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
62; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP28]], <3 x i32> <i32 3, i32 1, i32 2>
63; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
64; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
65; CHECK-NEXT:    [[SPLAT_SPLATINSERT21:%.*]] = insertelement <1 x double> poison, double [[TMP30]], i32 0
66; CHECK-NEXT:    [[SPLAT_SPLAT22:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT21]], <1 x double> poison, <1 x i32> zeroinitializer
67; CHECK-NEXT:    [[TMP31:%.*]] = fmul <1 x double> [[BLOCK20]], [[SPLAT_SPLAT22]]
68; CHECK-NEXT:    [[BLOCK23:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
69; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
70; CHECK-NEXT:    [[SPLAT_SPLATINSERT24:%.*]] = insertelement <1 x double> poison, double [[TMP32]], i32 0
71; CHECK-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT24]], <1 x double> poison, <1 x i32> zeroinitializer
72; CHECK-NEXT:    [[TMP33:%.*]] = fmul <1 x double> [[BLOCK23]], [[SPLAT_SPLAT25]]
73; CHECK-NEXT:    [[TMP34:%.*]] = fadd <1 x double> [[TMP31]], [[TMP33]]
74; CHECK-NEXT:    [[BLOCK26:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
75; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
76; CHECK-NEXT:    [[SPLAT_SPLATINSERT27:%.*]] = insertelement <1 x double> poison, double [[TMP35]], i32 0
77; CHECK-NEXT:    [[SPLAT_SPLAT28:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT27]], <1 x double> poison, <1 x i32> zeroinitializer
78; CHECK-NEXT:    [[TMP36:%.*]] = fmul <1 x double> [[BLOCK26]], [[SPLAT_SPLAT28]]
79; CHECK-NEXT:    [[TMP37:%.*]] = fadd <1 x double> [[TMP34]], [[TMP36]]
80; CHECK-NEXT:    [[TMP38:%.*]] = shufflevector <1 x double> [[TMP37]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
81; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> [[TMP38]], <3 x i32> <i32 0, i32 3, i32 2>
82; CHECK-NEXT:    [[BLOCK29:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
83; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
84; CHECK-NEXT:    [[SPLAT_SPLATINSERT30:%.*]] = insertelement <1 x double> poison, double [[TMP40]], i32 0
85; CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT30]], <1 x double> poison, <1 x i32> zeroinitializer
86; CHECK-NEXT:    [[TMP41:%.*]] = fmul <1 x double> [[BLOCK29]], [[SPLAT_SPLAT31]]
87; CHECK-NEXT:    [[BLOCK32:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
88; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
89; CHECK-NEXT:    [[SPLAT_SPLATINSERT33:%.*]] = insertelement <1 x double> poison, double [[TMP42]], i32 0
90; CHECK-NEXT:    [[SPLAT_SPLAT34:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT33]], <1 x double> poison, <1 x i32> zeroinitializer
91; CHECK-NEXT:    [[TMP43:%.*]] = fmul <1 x double> [[BLOCK32]], [[SPLAT_SPLAT34]]
92; CHECK-NEXT:    [[TMP44:%.*]] = fadd <1 x double> [[TMP41]], [[TMP43]]
93; CHECK-NEXT:    [[BLOCK35:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
94; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
95; CHECK-NEXT:    [[SPLAT_SPLATINSERT36:%.*]] = insertelement <1 x double> poison, double [[TMP45]], i32 0
96; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT36]], <1 x double> poison, <1 x i32> zeroinitializer
97; CHECK-NEXT:    [[TMP46:%.*]] = fmul <1 x double> [[BLOCK35]], [[SPLAT_SPLAT37]]
98; CHECK-NEXT:    [[TMP47:%.*]] = fadd <1 x double> [[TMP44]], [[TMP46]]
99; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <1 x double> [[TMP47]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
100; CHECK-NEXT:    [[TMP49:%.*]] = shufflevector <3 x double> [[TMP39]], <3 x double> [[TMP48]], <3 x i32> <i32 0, i32 1, i32 3>
101; CHECK-NEXT:    [[BLOCK38:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
102; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
103; CHECK-NEXT:    [[SPLAT_SPLATINSERT39:%.*]] = insertelement <1 x double> poison, double [[TMP50]], i32 0
104; CHECK-NEXT:    [[SPLAT_SPLAT40:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT39]], <1 x double> poison, <1 x i32> zeroinitializer
105; CHECK-NEXT:    [[TMP51:%.*]] = fmul <1 x double> [[BLOCK38]], [[SPLAT_SPLAT40]]
106; CHECK-NEXT:    [[BLOCK41:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
107; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
108; CHECK-NEXT:    [[SPLAT_SPLATINSERT42:%.*]] = insertelement <1 x double> poison, double [[TMP52]], i32 0
109; CHECK-NEXT:    [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT42]], <1 x double> poison, <1 x i32> zeroinitializer
110; CHECK-NEXT:    [[TMP53:%.*]] = fmul <1 x double> [[BLOCK41]], [[SPLAT_SPLAT43]]
111; CHECK-NEXT:    [[TMP54:%.*]] = fadd <1 x double> [[TMP51]], [[TMP53]]
112; CHECK-NEXT:    [[BLOCK44:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
113; CHECK-NEXT:    [[TMP55:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
114; CHECK-NEXT:    [[SPLAT_SPLATINSERT45:%.*]] = insertelement <1 x double> poison, double [[TMP55]], i32 0
115; CHECK-NEXT:    [[SPLAT_SPLAT46:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT45]], <1 x double> poison, <1 x i32> zeroinitializer
116; CHECK-NEXT:    [[TMP56:%.*]] = fmul <1 x double> [[BLOCK44]], [[SPLAT_SPLAT46]]
117; CHECK-NEXT:    [[TMP57:%.*]] = fadd <1 x double> [[TMP54]], [[TMP56]]
118; CHECK-NEXT:    [[TMP58:%.*]] = shufflevector <1 x double> [[TMP57]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
119; CHECK-NEXT:    [[TMP59:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP58]], <3 x i32> <i32 3, i32 1, i32 2>
120; CHECK-NEXT:    [[BLOCK47:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
121; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
122; CHECK-NEXT:    [[SPLAT_SPLATINSERT48:%.*]] = insertelement <1 x double> poison, double [[TMP60]], i32 0
123; CHECK-NEXT:    [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT48]], <1 x double> poison, <1 x i32> zeroinitializer
124; CHECK-NEXT:    [[TMP61:%.*]] = fmul <1 x double> [[BLOCK47]], [[SPLAT_SPLAT49]]
125; CHECK-NEXT:    [[BLOCK50:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
126; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
127; CHECK-NEXT:    [[SPLAT_SPLATINSERT51:%.*]] = insertelement <1 x double> poison, double [[TMP62]], i32 0
128; CHECK-NEXT:    [[SPLAT_SPLAT52:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT51]], <1 x double> poison, <1 x i32> zeroinitializer
129; CHECK-NEXT:    [[TMP63:%.*]] = fmul <1 x double> [[BLOCK50]], [[SPLAT_SPLAT52]]
130; CHECK-NEXT:    [[TMP64:%.*]] = fadd <1 x double> [[TMP61]], [[TMP63]]
131; CHECK-NEXT:    [[BLOCK53:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
132; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
133; CHECK-NEXT:    [[SPLAT_SPLATINSERT54:%.*]] = insertelement <1 x double> poison, double [[TMP65]], i32 0
134; CHECK-NEXT:    [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT54]], <1 x double> poison, <1 x i32> zeroinitializer
135; CHECK-NEXT:    [[TMP66:%.*]] = fmul <1 x double> [[BLOCK53]], [[SPLAT_SPLAT55]]
136; CHECK-NEXT:    [[TMP67:%.*]] = fadd <1 x double> [[TMP64]], [[TMP66]]
137; CHECK-NEXT:    [[TMP68:%.*]] = shufflevector <1 x double> [[TMP67]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
138; CHECK-NEXT:    [[TMP69:%.*]] = shufflevector <3 x double> [[TMP59]], <3 x double> [[TMP68]], <3 x i32> <i32 0, i32 3, i32 2>
139; CHECK-NEXT:    [[BLOCK56:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
140; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
141; CHECK-NEXT:    [[SPLAT_SPLATINSERT57:%.*]] = insertelement <1 x double> poison, double [[TMP70]], i32 0
142; CHECK-NEXT:    [[SPLAT_SPLAT58:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT57]], <1 x double> poison, <1 x i32> zeroinitializer
143; CHECK-NEXT:    [[TMP71:%.*]] = fmul <1 x double> [[BLOCK56]], [[SPLAT_SPLAT58]]
144; CHECK-NEXT:    [[BLOCK59:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
145; CHECK-NEXT:    [[TMP72:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
146; CHECK-NEXT:    [[SPLAT_SPLATINSERT60:%.*]] = insertelement <1 x double> poison, double [[TMP72]], i32 0
147; CHECK-NEXT:    [[SPLAT_SPLAT61:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT60]], <1 x double> poison, <1 x i32> zeroinitializer
148; CHECK-NEXT:    [[TMP73:%.*]] = fmul <1 x double> [[BLOCK59]], [[SPLAT_SPLAT61]]
149; CHECK-NEXT:    [[TMP74:%.*]] = fadd <1 x double> [[TMP71]], [[TMP73]]
150; CHECK-NEXT:    [[BLOCK62:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
151; CHECK-NEXT:    [[TMP75:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
152; CHECK-NEXT:    [[SPLAT_SPLATINSERT63:%.*]] = insertelement <1 x double> poison, double [[TMP75]], i32 0
153; CHECK-NEXT:    [[SPLAT_SPLAT64:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT63]], <1 x double> poison, <1 x i32> zeroinitializer
154; CHECK-NEXT:    [[TMP76:%.*]] = fmul <1 x double> [[BLOCK62]], [[SPLAT_SPLAT64]]
155; CHECK-NEXT:    [[TMP77:%.*]] = fadd <1 x double> [[TMP74]], [[TMP76]]
156; CHECK-NEXT:    [[TMP78:%.*]] = shufflevector <1 x double> [[TMP77]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
157; CHECK-NEXT:    [[TMP79:%.*]] = shufflevector <3 x double> [[TMP69]], <3 x double> [[TMP78]], <3 x i32> <i32 0, i32 1, i32 3>
158; CHECK-NEXT:    [[BLOCK65:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
159; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
160; CHECK-NEXT:    [[SPLAT_SPLATINSERT66:%.*]] = insertelement <1 x double> poison, double [[TMP80]], i32 0
161; CHECK-NEXT:    [[SPLAT_SPLAT67:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT66]], <1 x double> poison, <1 x i32> zeroinitializer
162; CHECK-NEXT:    [[TMP81:%.*]] = fmul <1 x double> [[BLOCK65]], [[SPLAT_SPLAT67]]
163; CHECK-NEXT:    [[BLOCK68:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
164; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
165; CHECK-NEXT:    [[SPLAT_SPLATINSERT69:%.*]] = insertelement <1 x double> poison, double [[TMP82]], i32 0
166; CHECK-NEXT:    [[SPLAT_SPLAT70:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT69]], <1 x double> poison, <1 x i32> zeroinitializer
167; CHECK-NEXT:    [[TMP83:%.*]] = fmul <1 x double> [[BLOCK68]], [[SPLAT_SPLAT70]]
168; CHECK-NEXT:    [[TMP84:%.*]] = fadd <1 x double> [[TMP81]], [[TMP83]]
169; CHECK-NEXT:    [[BLOCK71:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
170; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
171; CHECK-NEXT:    [[SPLAT_SPLATINSERT72:%.*]] = insertelement <1 x double> poison, double [[TMP85]], i32 0
172; CHECK-NEXT:    [[SPLAT_SPLAT73:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT72]], <1 x double> poison, <1 x i32> zeroinitializer
173; CHECK-NEXT:    [[TMP86:%.*]] = fmul <1 x double> [[BLOCK71]], [[SPLAT_SPLAT73]]
174; CHECK-NEXT:    [[TMP87:%.*]] = fadd <1 x double> [[TMP84]], [[TMP86]]
175; CHECK-NEXT:    [[TMP88:%.*]] = shufflevector <1 x double> [[TMP87]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
176; CHECK-NEXT:    [[TMP89:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP88]], <3 x i32> <i32 3, i32 1, i32 2>
177; CHECK-NEXT:    [[BLOCK74:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
178; CHECK-NEXT:    [[TMP90:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
179; CHECK-NEXT:    [[SPLAT_SPLATINSERT75:%.*]] = insertelement <1 x double> poison, double [[TMP90]], i32 0
180; CHECK-NEXT:    [[SPLAT_SPLAT76:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT75]], <1 x double> poison, <1 x i32> zeroinitializer
181; CHECK-NEXT:    [[TMP91:%.*]] = fmul <1 x double> [[BLOCK74]], [[SPLAT_SPLAT76]]
182; CHECK-NEXT:    [[BLOCK77:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
183; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
184; CHECK-NEXT:    [[SPLAT_SPLATINSERT78:%.*]] = insertelement <1 x double> poison, double [[TMP92]], i32 0
185; CHECK-NEXT:    [[SPLAT_SPLAT79:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT78]], <1 x double> poison, <1 x i32> zeroinitializer
186; CHECK-NEXT:    [[TMP93:%.*]] = fmul <1 x double> [[BLOCK77]], [[SPLAT_SPLAT79]]
187; CHECK-NEXT:    [[TMP94:%.*]] = fadd <1 x double> [[TMP91]], [[TMP93]]
188; CHECK-NEXT:    [[BLOCK80:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
189; CHECK-NEXT:    [[TMP95:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
190; CHECK-NEXT:    [[SPLAT_SPLATINSERT81:%.*]] = insertelement <1 x double> poison, double [[TMP95]], i32 0
191; CHECK-NEXT:    [[SPLAT_SPLAT82:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT81]], <1 x double> poison, <1 x i32> zeroinitializer
192; CHECK-NEXT:    [[TMP96:%.*]] = fmul <1 x double> [[BLOCK80]], [[SPLAT_SPLAT82]]
193; CHECK-NEXT:    [[TMP97:%.*]] = fadd <1 x double> [[TMP94]], [[TMP96]]
194; CHECK-NEXT:    [[TMP98:%.*]] = shufflevector <1 x double> [[TMP97]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
195; CHECK-NEXT:    [[TMP99:%.*]] = shufflevector <3 x double> [[TMP89]], <3 x double> [[TMP98]], <3 x i32> <i32 0, i32 3, i32 2>
196; CHECK-NEXT:    [[BLOCK83:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
197; CHECK-NEXT:    [[TMP100:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
198; CHECK-NEXT:    [[SPLAT_SPLATINSERT84:%.*]] = insertelement <1 x double> poison, double [[TMP100]], i32 0
199; CHECK-NEXT:    [[SPLAT_SPLAT85:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT84]], <1 x double> poison, <1 x i32> zeroinitializer
200; CHECK-NEXT:    [[TMP101:%.*]] = fmul <1 x double> [[BLOCK83]], [[SPLAT_SPLAT85]]
201; CHECK-NEXT:    [[BLOCK86:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
202; CHECK-NEXT:    [[TMP102:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
203; CHECK-NEXT:    [[SPLAT_SPLATINSERT87:%.*]] = insertelement <1 x double> poison, double [[TMP102]], i32 0
204; CHECK-NEXT:    [[SPLAT_SPLAT88:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT87]], <1 x double> poison, <1 x i32> zeroinitializer
205; CHECK-NEXT:    [[TMP103:%.*]] = fmul <1 x double> [[BLOCK86]], [[SPLAT_SPLAT88]]
206; CHECK-NEXT:    [[TMP104:%.*]] = fadd <1 x double> [[TMP101]], [[TMP103]]
207; CHECK-NEXT:    [[BLOCK89:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
208; CHECK-NEXT:    [[TMP105:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
209; CHECK-NEXT:    [[SPLAT_SPLATINSERT90:%.*]] = insertelement <1 x double> poison, double [[TMP105]], i32 0
210; CHECK-NEXT:    [[SPLAT_SPLAT91:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT90]], <1 x double> poison, <1 x i32> zeroinitializer
211; CHECK-NEXT:    [[TMP106:%.*]] = fmul <1 x double> [[BLOCK89]], [[SPLAT_SPLAT91]]
212; CHECK-NEXT:    [[TMP107:%.*]] = fadd <1 x double> [[TMP104]], [[TMP106]]
213; CHECK-NEXT:    [[TMP108:%.*]] = shufflevector <1 x double> [[TMP107]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
214; CHECK-NEXT:    [[TMP109:%.*]] = shufflevector <3 x double> [[TMP99]], <3 x double> [[TMP108]], <3 x i32> <i32 0, i32 1, i32 3>
215; CHECK-NEXT:    [[TMP110:%.*]] = bitcast <9 x double>* [[C_PTR:%.*]] to double*
216; CHECK-NEXT:    [[VEC_CAST92:%.*]] = bitcast double* [[TMP110]] to <3 x double>*
217; CHECK-NEXT:    store <3 x double> [[TMP49]], <3 x double>* [[VEC_CAST92]], align 8
218; CHECK-NEXT:    [[VEC_GEP93:%.*]] = getelementptr double, double* [[TMP110]], i64 3
219; CHECK-NEXT:    [[VEC_CAST94:%.*]] = bitcast double* [[VEC_GEP93]] to <3 x double>*
220; CHECK-NEXT:    store <3 x double> [[TMP79]], <3 x double>* [[VEC_CAST94]], align 8
221; CHECK-NEXT:    [[VEC_GEP95:%.*]] = getelementptr double, double* [[TMP110]], i64 6
222; CHECK-NEXT:    [[VEC_CAST96:%.*]] = bitcast double* [[VEC_GEP95]] to <3 x double>*
223; CHECK-NEXT:    store <3 x double> [[TMP109]], <3 x double>* [[VEC_CAST96]], align 8
224; CHECK-NEXT:    ret void
225;
226
227; Load columns of input matrixes %A and %B.
228
229
230; Transpose %A.
231
232
233; Lower multiply(transpose(%A), %B)
234
235
236; Store result columns.
237
238
239entry:
240  %a = load <9 x double>, <9 x double>* %A.Ptr, align 8
241  %b = load <9 x double>, <9 x double>* %B.Ptr, align 8
242  %a.trans  = call <9 x double> @llvm.matrix.transpose(<9 x double> %a, i32 3, i32 3)
243  %c = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %a.trans, <9 x double> %b, i32 3, i32 3, i32 3)
244  store <9 x double> %c, <9 x double>* %C.Ptr, align 8
245  ret void
246}
247
248declare <9 x double> @llvm.matrix.transpose(<9 x double>, i32, i32)
249declare <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double>, <9 x double>, i32, i32, i32)
250
251define void @transpose_multiply_add(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr, <9 x double>* %C.Ptr) {
252; CHECK-LABEL: @transpose_multiply_add(
253; CHECK-NEXT:  entry:
254; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <9 x double>* [[A_PTR:%.*]] to double*
255; CHECK-NEXT:    [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
256; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST]], align 8
257; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 3
258; CHECK-NEXT:    [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
259; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST1]], align 8
260; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr double, double* [[TMP0]], i64 6
261; CHECK-NEXT:    [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <3 x double>*
262; CHECK-NEXT:    [[COL_LOAD5:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST4]], align 8
263; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <9 x double>* [[B_PTR:%.*]] to double*
264; CHECK-NEXT:    [[VEC_CAST6:%.*]] = bitcast double* [[TMP1]] to <3 x double>*
265; CHECK-NEXT:    [[COL_LOAD7:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST6]], align 8
266; CHECK-NEXT:    [[VEC_GEP8:%.*]] = getelementptr double, double* [[TMP1]], i64 3
267; CHECK-NEXT:    [[VEC_CAST9:%.*]] = bitcast double* [[VEC_GEP8]] to <3 x double>*
268; CHECK-NEXT:    [[COL_LOAD10:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST9]], align 8
269; CHECK-NEXT:    [[VEC_GEP11:%.*]] = getelementptr double, double* [[TMP1]], i64 6
270; CHECK-NEXT:    [[VEC_CAST12:%.*]] = bitcast double* [[VEC_GEP11]] to <3 x double>*
271; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST12]], align 8
272; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
273; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x double> undef, double [[TMP2]], i64 0
274; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 0
275; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 1
276; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 0
277; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <3 x double> [[TMP5]], double [[TMP6]], i64 2
278; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 1
279; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x double> undef, double [[TMP8]], i64 0
280; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 1
281; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 1
282; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 1
283; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <3 x double> [[TMP11]], double [[TMP12]], i64 2
284; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 2
285; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <3 x double> undef, double [[TMP14]], i64 0
286; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 2
287; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 1
288; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 2
289; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <3 x double> [[TMP17]], double [[TMP18]], i64 2
290; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
291; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
292; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP20]], i32 0
293; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
294; CHECK-NEXT:    [[TMP21:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
295; CHECK-NEXT:    [[BLOCK14:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
296; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
297; CHECK-NEXT:    [[SPLAT_SPLATINSERT15:%.*]] = insertelement <1 x double> poison, double [[TMP22]], i32 0
298; CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT15]], <1 x double> poison, <1 x i32> zeroinitializer
299; CHECK-NEXT:    [[TMP23:%.*]] = fmul <1 x double> [[BLOCK14]], [[SPLAT_SPLAT16]]
300; CHECK-NEXT:    [[TMP24:%.*]] = fadd <1 x double> [[TMP21]], [[TMP23]]
301; CHECK-NEXT:    [[BLOCK17:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
302; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
303; CHECK-NEXT:    [[SPLAT_SPLATINSERT18:%.*]] = insertelement <1 x double> poison, double [[TMP25]], i32 0
304; CHECK-NEXT:    [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT18]], <1 x double> poison, <1 x i32> zeroinitializer
305; CHECK-NEXT:    [[TMP26:%.*]] = fmul <1 x double> [[BLOCK17]], [[SPLAT_SPLAT19]]
306; CHECK-NEXT:    [[TMP27:%.*]] = fadd <1 x double> [[TMP24]], [[TMP26]]
307; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <1 x double> [[TMP27]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
308; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP28]], <3 x i32> <i32 3, i32 1, i32 2>
309; CHECK-NEXT:    [[BLOCK20:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
310; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
311; CHECK-NEXT:    [[SPLAT_SPLATINSERT21:%.*]] = insertelement <1 x double> poison, double [[TMP30]], i32 0
312; CHECK-NEXT:    [[SPLAT_SPLAT22:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT21]], <1 x double> poison, <1 x i32> zeroinitializer
313; CHECK-NEXT:    [[TMP31:%.*]] = fmul <1 x double> [[BLOCK20]], [[SPLAT_SPLAT22]]
314; CHECK-NEXT:    [[BLOCK23:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
315; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
316; CHECK-NEXT:    [[SPLAT_SPLATINSERT24:%.*]] = insertelement <1 x double> poison, double [[TMP32]], i32 0
317; CHECK-NEXT:    [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT24]], <1 x double> poison, <1 x i32> zeroinitializer
318; CHECK-NEXT:    [[TMP33:%.*]] = fmul <1 x double> [[BLOCK23]], [[SPLAT_SPLAT25]]
319; CHECK-NEXT:    [[TMP34:%.*]] = fadd <1 x double> [[TMP31]], [[TMP33]]
320; CHECK-NEXT:    [[BLOCK26:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
321; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
322; CHECK-NEXT:    [[SPLAT_SPLATINSERT27:%.*]] = insertelement <1 x double> poison, double [[TMP35]], i32 0
323; CHECK-NEXT:    [[SPLAT_SPLAT28:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT27]], <1 x double> poison, <1 x i32> zeroinitializer
324; CHECK-NEXT:    [[TMP36:%.*]] = fmul <1 x double> [[BLOCK26]], [[SPLAT_SPLAT28]]
325; CHECK-NEXT:    [[TMP37:%.*]] = fadd <1 x double> [[TMP34]], [[TMP36]]
326; CHECK-NEXT:    [[TMP38:%.*]] = shufflevector <1 x double> [[TMP37]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
327; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> [[TMP38]], <3 x i32> <i32 0, i32 3, i32 2>
328; CHECK-NEXT:    [[BLOCK29:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
329; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
330; CHECK-NEXT:    [[SPLAT_SPLATINSERT30:%.*]] = insertelement <1 x double> poison, double [[TMP40]], i32 0
331; CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT30]], <1 x double> poison, <1 x i32> zeroinitializer
332; CHECK-NEXT:    [[TMP41:%.*]] = fmul <1 x double> [[BLOCK29]], [[SPLAT_SPLAT31]]
333; CHECK-NEXT:    [[BLOCK32:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
334; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
335; CHECK-NEXT:    [[SPLAT_SPLATINSERT33:%.*]] = insertelement <1 x double> poison, double [[TMP42]], i32 0
336; CHECK-NEXT:    [[SPLAT_SPLAT34:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT33]], <1 x double> poison, <1 x i32> zeroinitializer
337; CHECK-NEXT:    [[TMP43:%.*]] = fmul <1 x double> [[BLOCK32]], [[SPLAT_SPLAT34]]
338; CHECK-NEXT:    [[TMP44:%.*]] = fadd <1 x double> [[TMP41]], [[TMP43]]
339; CHECK-NEXT:    [[BLOCK35:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
340; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
341; CHECK-NEXT:    [[SPLAT_SPLATINSERT36:%.*]] = insertelement <1 x double> poison, double [[TMP45]], i32 0
342; CHECK-NEXT:    [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT36]], <1 x double> poison, <1 x i32> zeroinitializer
343; CHECK-NEXT:    [[TMP46:%.*]] = fmul <1 x double> [[BLOCK35]], [[SPLAT_SPLAT37]]
344; CHECK-NEXT:    [[TMP47:%.*]] = fadd <1 x double> [[TMP44]], [[TMP46]]
345; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <1 x double> [[TMP47]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
346; CHECK-NEXT:    [[TMP49:%.*]] = shufflevector <3 x double> [[TMP39]], <3 x double> [[TMP48]], <3 x i32> <i32 0, i32 1, i32 3>
347; CHECK-NEXT:    [[BLOCK38:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
348; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
349; CHECK-NEXT:    [[SPLAT_SPLATINSERT39:%.*]] = insertelement <1 x double> poison, double [[TMP50]], i32 0
350; CHECK-NEXT:    [[SPLAT_SPLAT40:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT39]], <1 x double> poison, <1 x i32> zeroinitializer
351; CHECK-NEXT:    [[TMP51:%.*]] = fmul <1 x double> [[BLOCK38]], [[SPLAT_SPLAT40]]
352; CHECK-NEXT:    [[BLOCK41:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
353; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
354; CHECK-NEXT:    [[SPLAT_SPLATINSERT42:%.*]] = insertelement <1 x double> poison, double [[TMP52]], i32 0
355; CHECK-NEXT:    [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT42]], <1 x double> poison, <1 x i32> zeroinitializer
356; CHECK-NEXT:    [[TMP53:%.*]] = fmul <1 x double> [[BLOCK41]], [[SPLAT_SPLAT43]]
357; CHECK-NEXT:    [[TMP54:%.*]] = fadd <1 x double> [[TMP51]], [[TMP53]]
358; CHECK-NEXT:    [[BLOCK44:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
359; CHECK-NEXT:    [[TMP55:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
360; CHECK-NEXT:    [[SPLAT_SPLATINSERT45:%.*]] = insertelement <1 x double> poison, double [[TMP55]], i32 0
361; CHECK-NEXT:    [[SPLAT_SPLAT46:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT45]], <1 x double> poison, <1 x i32> zeroinitializer
362; CHECK-NEXT:    [[TMP56:%.*]] = fmul <1 x double> [[BLOCK44]], [[SPLAT_SPLAT46]]
363; CHECK-NEXT:    [[TMP57:%.*]] = fadd <1 x double> [[TMP54]], [[TMP56]]
364; CHECK-NEXT:    [[TMP58:%.*]] = shufflevector <1 x double> [[TMP57]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
365; CHECK-NEXT:    [[TMP59:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP58]], <3 x i32> <i32 3, i32 1, i32 2>
366; CHECK-NEXT:    [[BLOCK47:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
367; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
368; CHECK-NEXT:    [[SPLAT_SPLATINSERT48:%.*]] = insertelement <1 x double> poison, double [[TMP60]], i32 0
369; CHECK-NEXT:    [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT48]], <1 x double> poison, <1 x i32> zeroinitializer
370; CHECK-NEXT:    [[TMP61:%.*]] = fmul <1 x double> [[BLOCK47]], [[SPLAT_SPLAT49]]
371; CHECK-NEXT:    [[BLOCK50:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
372; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
373; CHECK-NEXT:    [[SPLAT_SPLATINSERT51:%.*]] = insertelement <1 x double> poison, double [[TMP62]], i32 0
374; CHECK-NEXT:    [[SPLAT_SPLAT52:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT51]], <1 x double> poison, <1 x i32> zeroinitializer
375; CHECK-NEXT:    [[TMP63:%.*]] = fmul <1 x double> [[BLOCK50]], [[SPLAT_SPLAT52]]
376; CHECK-NEXT:    [[TMP64:%.*]] = fadd <1 x double> [[TMP61]], [[TMP63]]
377; CHECK-NEXT:    [[BLOCK53:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
378; CHECK-NEXT:    [[TMP65:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
379; CHECK-NEXT:    [[SPLAT_SPLATINSERT54:%.*]] = insertelement <1 x double> poison, double [[TMP65]], i32 0
380; CHECK-NEXT:    [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT54]], <1 x double> poison, <1 x i32> zeroinitializer
381; CHECK-NEXT:    [[TMP66:%.*]] = fmul <1 x double> [[BLOCK53]], [[SPLAT_SPLAT55]]
382; CHECK-NEXT:    [[TMP67:%.*]] = fadd <1 x double> [[TMP64]], [[TMP66]]
383; CHECK-NEXT:    [[TMP68:%.*]] = shufflevector <1 x double> [[TMP67]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
384; CHECK-NEXT:    [[TMP69:%.*]] = shufflevector <3 x double> [[TMP59]], <3 x double> [[TMP68]], <3 x i32> <i32 0, i32 3, i32 2>
385; CHECK-NEXT:    [[BLOCK56:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
386; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
387; CHECK-NEXT:    [[SPLAT_SPLATINSERT57:%.*]] = insertelement <1 x double> poison, double [[TMP70]], i32 0
388; CHECK-NEXT:    [[SPLAT_SPLAT58:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT57]], <1 x double> poison, <1 x i32> zeroinitializer
389; CHECK-NEXT:    [[TMP71:%.*]] = fmul <1 x double> [[BLOCK56]], [[SPLAT_SPLAT58]]
390; CHECK-NEXT:    [[BLOCK59:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
391; CHECK-NEXT:    [[TMP72:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
392; CHECK-NEXT:    [[SPLAT_SPLATINSERT60:%.*]] = insertelement <1 x double> poison, double [[TMP72]], i32 0
393; CHECK-NEXT:    [[SPLAT_SPLAT61:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT60]], <1 x double> poison, <1 x i32> zeroinitializer
394; CHECK-NEXT:    [[TMP73:%.*]] = fmul <1 x double> [[BLOCK59]], [[SPLAT_SPLAT61]]
395; CHECK-NEXT:    [[TMP74:%.*]] = fadd <1 x double> [[TMP71]], [[TMP73]]
396; CHECK-NEXT:    [[BLOCK62:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
397; CHECK-NEXT:    [[TMP75:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
398; CHECK-NEXT:    [[SPLAT_SPLATINSERT63:%.*]] = insertelement <1 x double> poison, double [[TMP75]], i32 0
399; CHECK-NEXT:    [[SPLAT_SPLAT64:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT63]], <1 x double> poison, <1 x i32> zeroinitializer
400; CHECK-NEXT:    [[TMP76:%.*]] = fmul <1 x double> [[BLOCK62]], [[SPLAT_SPLAT64]]
401; CHECK-NEXT:    [[TMP77:%.*]] = fadd <1 x double> [[TMP74]], [[TMP76]]
402; CHECK-NEXT:    [[TMP78:%.*]] = shufflevector <1 x double> [[TMP77]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
403; CHECK-NEXT:    [[TMP79:%.*]] = shufflevector <3 x double> [[TMP69]], <3 x double> [[TMP78]], <3 x i32> <i32 0, i32 1, i32 3>
404; CHECK-NEXT:    [[BLOCK65:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
405; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
406; CHECK-NEXT:    [[SPLAT_SPLATINSERT66:%.*]] = insertelement <1 x double> poison, double [[TMP80]], i32 0
407; CHECK-NEXT:    [[SPLAT_SPLAT67:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT66]], <1 x double> poison, <1 x i32> zeroinitializer
408; CHECK-NEXT:    [[TMP81:%.*]] = fmul <1 x double> [[BLOCK65]], [[SPLAT_SPLAT67]]
409; CHECK-NEXT:    [[BLOCK68:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
410; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
411; CHECK-NEXT:    [[SPLAT_SPLATINSERT69:%.*]] = insertelement <1 x double> poison, double [[TMP82]], i32 0
412; CHECK-NEXT:    [[SPLAT_SPLAT70:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT69]], <1 x double> poison, <1 x i32> zeroinitializer
413; CHECK-NEXT:    [[TMP83:%.*]] = fmul <1 x double> [[BLOCK68]], [[SPLAT_SPLAT70]]
414; CHECK-NEXT:    [[TMP84:%.*]] = fadd <1 x double> [[TMP81]], [[TMP83]]
415; CHECK-NEXT:    [[BLOCK71:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
416; CHECK-NEXT:    [[TMP85:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
417; CHECK-NEXT:    [[SPLAT_SPLATINSERT72:%.*]] = insertelement <1 x double> poison, double [[TMP85]], i32 0
418; CHECK-NEXT:    [[SPLAT_SPLAT73:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT72]], <1 x double> poison, <1 x i32> zeroinitializer
419; CHECK-NEXT:    [[TMP86:%.*]] = fmul <1 x double> [[BLOCK71]], [[SPLAT_SPLAT73]]
420; CHECK-NEXT:    [[TMP87:%.*]] = fadd <1 x double> [[TMP84]], [[TMP86]]
421; CHECK-NEXT:    [[TMP88:%.*]] = shufflevector <1 x double> [[TMP87]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
422; CHECK-NEXT:    [[TMP89:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP88]], <3 x i32> <i32 3, i32 1, i32 2>
423; CHECK-NEXT:    [[BLOCK74:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
424; CHECK-NEXT:    [[TMP90:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
425; CHECK-NEXT:    [[SPLAT_SPLATINSERT75:%.*]] = insertelement <1 x double> poison, double [[TMP90]], i32 0
426; CHECK-NEXT:    [[SPLAT_SPLAT76:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT75]], <1 x double> poison, <1 x i32> zeroinitializer
427; CHECK-NEXT:    [[TMP91:%.*]] = fmul <1 x double> [[BLOCK74]], [[SPLAT_SPLAT76]]
428; CHECK-NEXT:    [[BLOCK77:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
429; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
430; CHECK-NEXT:    [[SPLAT_SPLATINSERT78:%.*]] = insertelement <1 x double> poison, double [[TMP92]], i32 0
431; CHECK-NEXT:    [[SPLAT_SPLAT79:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT78]], <1 x double> poison, <1 x i32> zeroinitializer
432; CHECK-NEXT:    [[TMP93:%.*]] = fmul <1 x double> [[BLOCK77]], [[SPLAT_SPLAT79]]
433; CHECK-NEXT:    [[TMP94:%.*]] = fadd <1 x double> [[TMP91]], [[TMP93]]
434; CHECK-NEXT:    [[BLOCK80:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
435; CHECK-NEXT:    [[TMP95:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
436; CHECK-NEXT:    [[SPLAT_SPLATINSERT81:%.*]] = insertelement <1 x double> poison, double [[TMP95]], i32 0
437; CHECK-NEXT:    [[SPLAT_SPLAT82:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT81]], <1 x double> poison, <1 x i32> zeroinitializer
438; CHECK-NEXT:    [[TMP96:%.*]] = fmul <1 x double> [[BLOCK80]], [[SPLAT_SPLAT82]]
439; CHECK-NEXT:    [[TMP97:%.*]] = fadd <1 x double> [[TMP94]], [[TMP96]]
440; CHECK-NEXT:    [[TMP98:%.*]] = shufflevector <1 x double> [[TMP97]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
441; CHECK-NEXT:    [[TMP99:%.*]] = shufflevector <3 x double> [[TMP89]], <3 x double> [[TMP98]], <3 x i32> <i32 0, i32 3, i32 2>
442; CHECK-NEXT:    [[BLOCK83:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
443; CHECK-NEXT:    [[TMP100:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
444; CHECK-NEXT:    [[SPLAT_SPLATINSERT84:%.*]] = insertelement <1 x double> poison, double [[TMP100]], i32 0
445; CHECK-NEXT:    [[SPLAT_SPLAT85:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT84]], <1 x double> poison, <1 x i32> zeroinitializer
446; CHECK-NEXT:    [[TMP101:%.*]] = fmul <1 x double> [[BLOCK83]], [[SPLAT_SPLAT85]]
447; CHECK-NEXT:    [[BLOCK86:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
448; CHECK-NEXT:    [[TMP102:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
449; CHECK-NEXT:    [[SPLAT_SPLATINSERT87:%.*]] = insertelement <1 x double> poison, double [[TMP102]], i32 0
450; CHECK-NEXT:    [[SPLAT_SPLAT88:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT87]], <1 x double> poison, <1 x i32> zeroinitializer
451; CHECK-NEXT:    [[TMP103:%.*]] = fmul <1 x double> [[BLOCK86]], [[SPLAT_SPLAT88]]
452; CHECK-NEXT:    [[TMP104:%.*]] = fadd <1 x double> [[TMP101]], [[TMP103]]
453; CHECK-NEXT:    [[BLOCK89:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
454; CHECK-NEXT:    [[TMP105:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
455; CHECK-NEXT:    [[SPLAT_SPLATINSERT90:%.*]] = insertelement <1 x double> poison, double [[TMP105]], i32 0
456; CHECK-NEXT:    [[SPLAT_SPLAT91:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT90]], <1 x double> poison, <1 x i32> zeroinitializer
457; CHECK-NEXT:    [[TMP106:%.*]] = fmul <1 x double> [[BLOCK89]], [[SPLAT_SPLAT91]]
458; CHECK-NEXT:    [[TMP107:%.*]] = fadd <1 x double> [[TMP104]], [[TMP106]]
459; CHECK-NEXT:    [[TMP108:%.*]] = shufflevector <1 x double> [[TMP107]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
460; CHECK-NEXT:    [[TMP109:%.*]] = shufflevector <3 x double> [[TMP99]], <3 x double> [[TMP108]], <3 x i32> <i32 0, i32 1, i32 3>
461; CHECK-NEXT:    [[TMP110:%.*]] = bitcast <9 x double>* [[C_PTR:%.*]] to double*
462; CHECK-NEXT:    [[VEC_CAST92:%.*]] = bitcast double* [[TMP110]] to <3 x double>*
463; CHECK-NEXT:    [[COL_LOAD93:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST92]], align 8
464; CHECK-NEXT:    [[VEC_GEP94:%.*]] = getelementptr double, double* [[TMP110]], i64 3
465; CHECK-NEXT:    [[VEC_CAST95:%.*]] = bitcast double* [[VEC_GEP94]] to <3 x double>*
466; CHECK-NEXT:    [[COL_LOAD96:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST95]], align 8
467; CHECK-NEXT:    [[VEC_GEP97:%.*]] = getelementptr double, double* [[TMP110]], i64 6
468; CHECK-NEXT:    [[VEC_CAST98:%.*]] = bitcast double* [[VEC_GEP97]] to <3 x double>*
469; CHECK-NEXT:    [[COL_LOAD99:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST98]], align 8
470; CHECK-NEXT:    [[TMP111:%.*]] = fadd <3 x double> [[COL_LOAD93]], [[TMP49]]
471; CHECK-NEXT:    [[TMP112:%.*]] = fadd <3 x double> [[COL_LOAD96]], [[TMP79]]
472; CHECK-NEXT:    [[TMP113:%.*]] = fadd <3 x double> [[COL_LOAD99]], [[TMP109]]
473; CHECK-NEXT:    [[TMP114:%.*]] = bitcast <9 x double>* [[C_PTR]] to double*
474; CHECK-NEXT:    [[VEC_CAST100:%.*]] = bitcast double* [[TMP114]] to <3 x double>*
475; CHECK-NEXT:    store <3 x double> [[TMP111]], <3 x double>* [[VEC_CAST100]], align 8
476; CHECK-NEXT:    [[VEC_GEP101:%.*]] = getelementptr double, double* [[TMP114]], i64 3
477; CHECK-NEXT:    [[VEC_CAST102:%.*]] = bitcast double* [[VEC_GEP101]] to <3 x double>*
478; CHECK-NEXT:    store <3 x double> [[TMP112]], <3 x double>* [[VEC_CAST102]], align 8
479; CHECK-NEXT:    [[VEC_GEP103:%.*]] = getelementptr double, double* [[TMP114]], i64 6
480; CHECK-NEXT:    [[VEC_CAST104:%.*]] = bitcast double* [[VEC_GEP103]] to <3 x double>*
481; CHECK-NEXT:    store <3 x double> [[TMP113]], <3 x double>* [[VEC_CAST104]], align 8
482; CHECK-NEXT:    ret void
483;
484
485
486; Transpose %A.
487
488
489; Lower multiply(transpose(%A), %B)
490
491
492;  Embed result of multiply into flat vector.
493
494
495; Load %C.
496
497
498; Add column vectors.
499
500
501; Store result columns.
502
503
504entry:
505  %a = load <9 x double>, <9 x double>* %A.Ptr, align 8
506  %b = load <9 x double>, <9 x double>* %B.Ptr, align 8
507  %a.trans  = call <9 x double> @llvm.matrix.transpose(<9 x double> %a, i32 3, i32 3)
508  %mult = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %a.trans, <9 x double> %b, i32 3, i32 3, i32 3)
509  %c = load <9 x double>, <9 x double>* %C.Ptr, align 8
510  %res = fadd <9 x double> %c, %mult
511
512  store <9 x double> %res, <9 x double>* %C.Ptr, align 8
513  ret void
514}
515