1//===--- arm_neon.td - ARM NEON compiler interface ------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines the TableGen definitions from which the ARM NEON header 10// file will be generated. See ARM document DUI0348B. 11// 12//===----------------------------------------------------------------------===// 13 14include "arm_neon_incl.td" 15 16def OP_ADD : Op<(op "+", $p0, $p1)>; 17def OP_ADDL : Op<(op "+", (call "vmovl", $p0), (call "vmovl", $p1))>; 18def OP_ADDLHi : Op<(op "+", (call "vmovl_high", $p0), 19 (call "vmovl_high", $p1))>; 20def OP_ADDW : Op<(op "+", $p0, (call "vmovl", $p1))>; 21def OP_ADDWHi : Op<(op "+", $p0, (call "vmovl_high", $p1))>; 22def OP_SUB : Op<(op "-", $p0, $p1)>; 23def OP_SUBL : Op<(op "-", (call "vmovl", $p0), (call "vmovl", $p1))>; 24def OP_SUBLHi : Op<(op "-", (call "vmovl_high", $p0), 25 (call "vmovl_high", $p1))>; 26def OP_SUBW : Op<(op "-", $p0, (call "vmovl", $p1))>; 27def OP_SUBWHi : Op<(op "-", $p0, (call "vmovl_high", $p1))>; 28def OP_MUL : Op<(op "*", $p0, $p1)>; 29def OP_MLA : Op<(op "+", $p0, (op "*", $p1, $p2))>; 30def OP_MLAL : Op<(op "+", $p0, (call "vmull", $p1, $p2))>; 31def OP_MULLHi : Op<(call "vmull", (call "vget_high", $p0), 32 (call "vget_high", $p1))>; 33def OP_MULLHi_P64 : Op<(call "vmull", 34 (cast "poly64_t", (call "vget_high", $p0)), 35 (cast "poly64_t", (call "vget_high", $p1)))>; 36def OP_MULLHi_N : Op<(call "vmull_n", (call "vget_high", $p0), $p1)>; 37def OP_MLALHi : Op<(call "vmlal", $p0, (call "vget_high", $p1), 38 (call "vget_high", $p2))>; 39def OP_MLALHi_N : Op<(call "vmlal_n", $p0, (call "vget_high", $p1), $p2)>; 40def OP_MLS : Op<(op "-", $p0, (op "*", $p1, $p2))>; 41def OP_FMLS : Op<(call "vfma", $p0, (op "-", $p1), $p2)>; 42def OP_MLSL : Op<(op "-", $p0, (call "vmull", $p1, $p2))>; 43def OP_MLSLHi : Op<(call "vmlsl", $p0, (call "vget_high", $p1), 44 (call "vget_high", $p2))>; 45def OP_MLSLHi_N : Op<(call "vmlsl_n", $p0, (call "vget_high", $p1), $p2)>; 46def OP_MUL_N : Op<(op "*", $p0, (dup $p1))>; 47def OP_MULX_N : Op<(call "vmulx", $p0, (dup $p1))>; 48def OP_MLA_N : Op<(op "+", $p0, (op "*", $p1, (dup $p2)))>; 49def OP_MLS_N : Op<(op "-", $p0, (op "*", $p1, (dup $p2)))>; 50def OP_FMLA_N : Op<(call "vfma", $p0, $p1, (dup $p2))>; 51def OP_FMLS_N : Op<(call "vfma", $p0, (op "-", $p1), (dup $p2))>; 52def OP_MLAL_N : Op<(op "+", $p0, (call "vmull", $p1, (dup $p2)))>; 53def OP_MLSL_N : Op<(op "-", $p0, (call "vmull", $p1, (dup $p2)))>; 54def OP_MUL_LN : Op<(op "*", $p0, (call_mangled "splat_lane", $p1, $p2))>; 55def OP_MULX_LN : Op<(call "vmulx", $p0, (call_mangled "splat_lane", $p1, $p2))>; 56def OP_MULL_N : Op<(call "vmull", $p0, (dup $p1))>; 57def OP_MULL_LN : Op<(call "vmull", $p0, (call_mangled "splat_lane", $p1, $p2))>; 58def OP_MULLHi_LN: Op<(call "vmull", (call "vget_high", $p0), (call_mangled "splat_lane", $p1, $p2))>; 59def OP_MLA_LN : Op<(op "+", $p0, (op "*", $p1, (call_mangled "splat_lane", $p2, $p3)))>; 60def OP_MLS_LN : Op<(op "-", $p0, (op "*", $p1, (call_mangled "splat_lane", $p2, $p3)))>; 61def OP_MLAL_LN : Op<(op "+", $p0, (call "vmull", $p1, (call_mangled "splat_lane", $p2, $p3)))>; 62def OP_MLALHi_LN: Op<(op "+", $p0, (call "vmull", (call "vget_high", $p1), 63 (call_mangled "splat_lane", $p2, $p3)))>; 64def OP_MLSL_LN : Op<(op "-", $p0, (call "vmull", $p1, (call_mangled "splat_lane", $p2, $p3)))>; 65def OP_MLSLHi_LN : Op<(op "-", $p0, (call "vmull", (call "vget_high", $p1), 66 (call_mangled "splat_lane", $p2, $p3)))>; 67def OP_QDMULL_N : Op<(call "vqdmull", $p0, (dup $p1))>; 68def OP_QDMULL_LN : Op<(call "vqdmull", $p0, (call_mangled "splat_lane", $p1, $p2))>; 69def OP_QDMULLHi_LN : Op<(call "vqdmull", (call "vget_high", $p0), 70 (call_mangled "splat_lane", $p1, $p2))>; 71def OP_QDMLAL_N : Op<(call "vqdmlal", $p0, $p1, (dup $p2))>; 72def OP_QDMLAL_LN : Op<(call "vqdmlal", $p0, $p1, (call_mangled "splat_lane", $p2, $p3))>; 73def OP_QDMLALHi_LN : Op<(call "vqdmlal", $p0, (call "vget_high", $p1), 74 (call_mangled "splat_lane", $p2, $p3))>; 75def OP_QDMLSL_N : Op<(call "vqdmlsl", $p0, $p1, (dup $p2))>; 76def OP_QDMLSL_LN : Op<(call "vqdmlsl", $p0, $p1, (call_mangled "splat_lane", $p2, $p3))>; 77def OP_QDMLSLHi_LN : Op<(call "vqdmlsl", $p0, (call "vget_high", $p1), 78 (call_mangled "splat_lane", $p2, $p3))>; 79def OP_QDMULH_N : Op<(call "vqdmulh", $p0, (dup $p1))>; 80def OP_QDMULH_LN : Op<(call "vqdmulh", $p0, (call_mangled "splat_lane", $p1, $p2))>; 81def OP_QRDMULH_LN : Op<(call "vqrdmulh", $p0, (call_mangled "splat_lane", $p1, $p2))>; 82def OP_QRDMULH_N : Op<(call "vqrdmulh", $p0, (dup $p1))>; 83def OP_QRDMLAH : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, $p2))>; 84def OP_QRDMLSH : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, $p2))>; 85def OP_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, (call_mangled "splat_lane", $p2, $p3)))>; 86def OP_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, (call_mangled "splat_lane", $p2, $p3)))>; 87def OP_FMS_LN : Op<(call "vfma_lane", $p0, (op "-", $p1), $p2, $p3)>; 88def OP_FMS_LNQ : Op<(call "vfma_laneq", $p0, (op "-", $p1), $p2, $p3)>; 89def OP_TRN1 : Op<(shuffle $p0, $p1, (interleave (decimate mask0, 2), 90 (decimate mask1, 2)))>; 91def OP_ZIP1 : Op<(shuffle $p0, $p1, (lowhalf (interleave mask0, mask1)))>; 92def OP_UZP1 : Op<(shuffle $p0, $p1, (add (decimate mask0, 2), 93 (decimate mask1, 2)))>; 94def OP_TRN2 : Op<(shuffle $p0, $p1, (interleave 95 (decimate (rotl mask0, 1), 2), 96 (decimate (rotl mask1, 1), 2)))>; 97def OP_ZIP2 : Op<(shuffle $p0, $p1, (highhalf (interleave mask0, mask1)))>; 98def OP_UZP2 : Op<(shuffle $p0, $p1, (add (decimate (rotl mask0, 1), 2), 99 (decimate (rotl mask1, 1), 2)))>; 100def OP_EQ : Op<(cast "R", (op "==", $p0, $p1))>; 101def OP_GE : Op<(cast "R", (op ">=", $p0, $p1))>; 102def OP_LE : Op<(cast "R", (op "<=", $p0, $p1))>; 103def OP_GT : Op<(cast "R", (op ">", $p0, $p1))>; 104def OP_LT : Op<(cast "R", (op "<", $p0, $p1))>; 105def OP_NEG : Op<(op "-", $p0)>; 106def OP_NOT : Op<(op "~", $p0)>; 107def OP_AND : Op<(op "&", $p0, $p1)>; 108def OP_OR : Op<(op "|", $p0, $p1)>; 109def OP_XOR : Op<(op "^", $p0, $p1)>; 110def OP_ANDN : Op<(op "&", $p0, (op "~", $p1))>; 111def OP_ORN : Op<(op "|", $p0, (op "~", $p1))>; 112def OP_CAST : LOp<[(save_temp $promote, $p0), 113 (cast "R", $promote)]>; 114def OP_HI : Op<(shuffle $p0, $p0, (highhalf mask0))>; 115def OP_LO : Op<(shuffle $p0, $p0, (lowhalf mask0))>; 116def OP_CONC : Op<(shuffle $p0, $p1, (add mask0, mask1))>; 117def OP_DUP : Op<(dup $p0)>; 118def OP_DUP_LN : Op<(call_mangled "splat_lane", $p0, $p1)>; 119def OP_SEL : Op<(cast "R", (op "|", 120 (op "&", $p0, (cast $p0, $p1)), 121 (op "&", (op "~", $p0), (cast $p0, $p2))))>; 122def OP_REV16 : Op<(shuffle $p0, $p0, (rev 16, mask0))>; 123def OP_REV32 : Op<(shuffle $p0, $p0, (rev 32, mask0))>; 124def OP_REV64 : Op<(shuffle $p0, $p0, (rev 64, mask0))>; 125def OP_XTN : Op<(call "vcombine", $p0, (call "vmovn", $p1))>; 126def OP_SQXTUN : Op<(call "vcombine", (cast $p0, "U", $p0), 127 (call "vqmovun", $p1))>; 128def OP_QXTN : Op<(call "vcombine", $p0, (call "vqmovn", $p1))>; 129def OP_VCVT_NA_HI_F16 : Op<(call "vcombine", $p0, (call "vcvt_f16_f32", $p1))>; 130def OP_VCVT_NA_HI_F32 : Op<(call "vcombine", $p0, (call "vcvt_f32_f64", $p1))>; 131def OP_VCVT_EX_HI_F32 : Op<(call "vcvt_f32_f16", (call "vget_high", $p0))>; 132def OP_VCVT_EX_HI_F64 : Op<(call "vcvt_f64_f32", (call "vget_high", $p0))>; 133def OP_VCVTX_HI : Op<(call "vcombine", $p0, (call "vcvtx_f32", $p1))>; 134def OP_REINT : Op<(cast "R", $p0)>; 135def OP_ADDHNHi : Op<(call "vcombine", $p0, (call "vaddhn", $p1, $p2))>; 136def OP_RADDHNHi : Op<(call "vcombine", $p0, (call "vraddhn", $p1, $p2))>; 137def OP_SUBHNHi : Op<(call "vcombine", $p0, (call "vsubhn", $p1, $p2))>; 138def OP_RSUBHNHi : Op<(call "vcombine", $p0, (call "vrsubhn", $p1, $p2))>; 139def OP_ABDL : Op<(cast "R", (call "vmovl", (cast $p0, "U", 140 (call "vabd", $p0, $p1))))>; 141def OP_ABDLHi : Op<(call "vabdl", (call "vget_high", $p0), 142 (call "vget_high", $p1))>; 143def OP_ABA : Op<(op "+", $p0, (call "vabd", $p1, $p2))>; 144def OP_ABAL : Op<(op "+", $p0, (call "vabdl", $p1, $p2))>; 145def OP_ABALHi : Op<(call "vabal", $p0, (call "vget_high", $p1), 146 (call "vget_high", $p2))>; 147def OP_QDMULLHi : Op<(call "vqdmull", (call "vget_high", $p0), 148 (call "vget_high", $p1))>; 149def OP_QDMULLHi_N : Op<(call "vqdmull_n", (call "vget_high", $p0), $p1)>; 150def OP_QDMLALHi : Op<(call "vqdmlal", $p0, (call "vget_high", $p1), 151 (call "vget_high", $p2))>; 152def OP_QDMLALHi_N : Op<(call "vqdmlal_n", $p0, (call "vget_high", $p1), $p2)>; 153def OP_QDMLSLHi : Op<(call "vqdmlsl", $p0, (call "vget_high", $p1), 154 (call "vget_high", $p2))>; 155def OP_QDMLSLHi_N : Op<(call "vqdmlsl_n", $p0, (call "vget_high", $p1), $p2)>; 156def OP_DIV : Op<(op "/", $p0, $p1)>; 157def OP_LONG_HI : Op<(cast "R", (call (name_replace "_high_", "_"), 158 (call "vget_high", $p0), $p1))>; 159def OP_NARROW_HI : Op<(cast "R", (call "vcombine", 160 (cast "R", "H", $p0), 161 (cast "R", "H", 162 (call (name_replace "_high_", "_"), 163 $p1, $p2))))>; 164def OP_MOVL_HI : LOp<[(save_temp $a1, (call "vget_high", $p0)), 165 (cast "R", 166 (call "vshll_n", $a1, (literal "int32_t", "0")))]>; 167def OP_COPY_LN : Op<(call "vset_lane", (call "vget_lane", $p2, $p3), $p0, $p1)>; 168def OP_SCALAR_MUL_LN : Op<(op "*", $p0, (call "vget_lane", $p1, $p2))>; 169def OP_SCALAR_MULX_LN : Op<(call "vmulx", $p0, (call "vget_lane", $p1, $p2))>; 170def OP_SCALAR_VMULX_LN : LOp<[(save_temp $x, (call "vget_lane", $p0, 171 (literal "int32_t", "0"))), 172 (save_temp $y, (call "vget_lane", $p1, $p2)), 173 (save_temp $z, (call "vmulx", $x, $y)), 174 (call "vset_lane", $z, $p0, $p2)]>; 175def OP_SCALAR_VMULX_LNQ : LOp<[(save_temp $x, (call "vget_lane", $p0, 176 (literal "int32_t", "0"))), 177 (save_temp $y, (call "vget_lane", $p1, $p2)), 178 (save_temp $z, (call "vmulx", $x, $y)), 179 (call "vset_lane", $z, $p0, (literal "int32_t", 180 "0"))]>; 181class ScalarMulOp<string opname> : 182 Op<(call opname, $p0, (call "vget_lane", $p1, $p2))>; 183 184def OP_SCALAR_QDMULL_LN : ScalarMulOp<"vqdmull">; 185def OP_SCALAR_QDMULH_LN : ScalarMulOp<"vqdmulh">; 186def OP_SCALAR_QRDMULH_LN : ScalarMulOp<"vqrdmulh">; 187 188def OP_SCALAR_QRDMLAH_LN : Op<(call "vqadd", $p0, (call "vqrdmulh", $p1, 189 (call "vget_lane", $p2, $p3)))>; 190def OP_SCALAR_QRDMLSH_LN : Op<(call "vqsub", $p0, (call "vqrdmulh", $p1, 191 (call "vget_lane", $p2, $p3)))>; 192 193def OP_SCALAR_HALF_GET_LN : Op<(bitcast "float16_t", 194 (call "vget_lane", 195 (bitcast "int16x4_t", $p0), $p1))>; 196def OP_SCALAR_HALF_GET_LNQ : Op<(bitcast "float16_t", 197 (call "vget_lane", 198 (bitcast "int16x8_t", $p0), $p1))>; 199def OP_SCALAR_HALF_SET_LN : Op<(bitcast "float16x4_t", 200 (call "vset_lane", 201 (bitcast "int16_t", $p0), 202 (bitcast "int16x4_t", $p1), $p2))>; 203def OP_SCALAR_HALF_SET_LNQ : Op<(bitcast "float16x8_t", 204 (call "vset_lane", 205 (bitcast "int16_t", $p0), 206 (bitcast "int16x8_t", $p1), $p2))>; 207 208def OP_DOT_LN 209 : Op<(call "vdot", $p0, $p1, 210 (bitcast $p1, (call_mangled "splat_lane", (bitcast "32", $p2), $p3)))>; 211def OP_DOT_LNQ 212 : Op<(call "vdot", $p0, $p1, 213 (bitcast $p1, (call_mangled "splat_lane", (bitcast "32", $p2), $p3)))>; 214 215def OP_FMLAL_LN : Op<(call "vfmlal_low", $p0, $p1, 216 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>; 217def OP_FMLSL_LN : Op<(call "vfmlsl_low", $p0, $p1, 218 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>; 219def OP_FMLAL_LN_Hi : Op<(call "vfmlal_high", $p0, $p1, 220 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>; 221def OP_FMLSL_LN_Hi : Op<(call "vfmlsl_high", $p0, $p1, 222 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>; 223 224def OP_USDOT_LN 225 : Op<(call "vusdot", $p0, $p1, 226 (cast "8", "S", (call_mangled "splat_lane", (bitcast "int32x2_t", $p2), $p3)))>; 227def OP_USDOT_LNQ 228 : Op<(call "vusdot", $p0, $p1, 229 (cast "8", "S", (call_mangled "splat_lane", (bitcast "int32x4_t", $p2), $p3)))>; 230 231// sudot splats the second vector and then calls vusdot 232def OP_SUDOT_LN 233 : Op<(call "vusdot", $p0, 234 (cast "8", "U", (call_mangled "splat_lane", (bitcast "int32x2_t", $p2), $p3)), $p1)>; 235def OP_SUDOT_LNQ 236 : Op<(call "vusdot", $p0, 237 (cast "8", "U", (call_mangled "splat_lane", (bitcast "int32x4_t", $p2), $p3)), $p1)>; 238 239def OP_BFDOT_LN 240 : Op<(call "vbfdot", $p0, $p1, 241 (bitcast $p1, (call_mangled "splat_lane", (bitcast "float32x2_t", $p2), $p3)))>; 242 243def OP_BFDOT_LNQ 244 : Op<(call "vbfdot", $p0, $p1, 245 (bitcast $p1, (call_mangled "splat_lane", (bitcast "float32x4_t", $p2), $p3)))>; 246 247def OP_BFMLALB_LN 248 : Op<(call "vbfmlalb", $p0, $p1, 249 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>; 250 251def OP_BFMLALT_LN 252 : Op<(call "vbfmlalt", $p0, $p1, 253 (dup_typed $p1, (call "vget_lane", $p2, $p3)))>; 254 255def OP_VCVT_F32_BF16 256 : Op<(bitcast "R", 257 (call "vshll_n", (bitcast "int16x4_t", $p0), 258 (literal "int32_t", "16")))>; 259def OP_VCVT_F32_BF16_LO 260 : Op<(call "vcvt_f32_bf16", (call "vget_low", $p0))>; 261def OP_VCVT_F32_BF16_HI 262 : Op<(call "vcvt_f32_bf16", (call "vget_high", $p0))>; 263 264def OP_VCVT_BF16_F32_LO_A64 265 : Op<(call "__a64_vcvtq_low_bf16", $p0)>; 266def OP_VCVT_BF16_F32_A64 267 : Op<(call "vget_low", (call "__a64_vcvtq_low_bf16", $p0))>; 268 269def OP_VCVT_BF16_F32_A32 270 : Op<(call "__a32_vcvt_bf16", $p0)>; 271 272def OP_VCVT_BF16_F32_LO_A32 273 : Op<(call "vcombine", (cast "bfloat16x4_t", (literal "uint64_t", "0ULL")), 274 (call "__a32_vcvt_bf16", $p0))>; 275def OP_VCVT_BF16_F32_HI_A32 276 : Op<(call "vcombine", (call "__a32_vcvt_bf16", $p1), 277 (call "vget_low", $p0))>; 278 279def OP_CVT_F32_BF16 280 : Op<(bitcast "R", (op "<<", (bitcast "int32_t", $p0), 281 (literal "int32_t", "16")))>; 282 283//===----------------------------------------------------------------------===// 284// Auxiliary Instructions 285//===----------------------------------------------------------------------===// 286 287// Splat operation - performs a range-checked splat over a vector 288def SPLAT : WInst<"splat_lane", ".(!q)I", 289 "UcUsUicsilPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUlhdQhQdPlQPl">; 290def SPLATQ : WInst<"splat_laneq", ".(!Q)I", 291 "UcUsUicsilPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUlhdQhQdPlQPl"> { 292 let isLaneQ = 1; 293} 294let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in { 295 def SPLAT_BF : WInst<"splat_lane", ".(!q)I", "bQb">; 296 def SPLATQ_BF : WInst<"splat_laneq", ".(!Q)I", "bQb"> { 297 let isLaneQ = 1; 298 } 299} 300 301//===----------------------------------------------------------------------===// 302// Intrinsics 303//===----------------------------------------------------------------------===// 304 305//////////////////////////////////////////////////////////////////////////////// 306// E.3.1 Addition 307def VADD : IOpInst<"vadd", "...", 308 "csilfUcUsUiUlQcQsQiQlQfQUcQUsQUiQUl", OP_ADD>; 309def VADDL : SOpInst<"vaddl", "(>Q)..", "csiUcUsUi", OP_ADDL>; 310def VADDW : SOpInst<"vaddw", "(>Q)(>Q).", "csiUcUsUi", OP_ADDW>; 311def VHADD : SInst<"vhadd", "...", "csiUcUsUiQcQsQiQUcQUsQUi">; 312def VRHADD : SInst<"vrhadd", "...", "csiUcUsUiQcQsQiQUcQUsQUi">; 313def VQADD : SInst<"vqadd", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 314def VADDHN : IInst<"vaddhn", "<QQ", "silUsUiUl">; 315def VRADDHN : IInst<"vraddhn", "<QQ", "silUsUiUl">; 316 317//////////////////////////////////////////////////////////////////////////////// 318// E.3.2 Multiplication 319def VMUL : IOpInst<"vmul", "...", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MUL>; 320def VMULP : SInst<"vmul", "...", "PcQPc">; 321def VMLA : IOpInst<"vmla", "....", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MLA>; 322def VMLAL : SOpInst<"vmlal", "(>Q)(>Q)..", "csiUcUsUi", OP_MLAL>; 323def VMLS : IOpInst<"vmls", "....", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_MLS>; 324def VMLSL : SOpInst<"vmlsl", "(>Q)(>Q)..", "csiUcUsUi", OP_MLSL>; 325def VQDMULH : SInst<"vqdmulh", "...", "siQsQi">; 326def VQRDMULH : SInst<"vqrdmulh", "...", "siQsQi">; 327 328let ArchGuard = "defined(__ARM_FEATURE_QRDMX)" in { 329def VQRDMLAH : SOpInst<"vqrdmlah", "....", "siQsQi", OP_QRDMLAH>; 330def VQRDMLSH : SOpInst<"vqrdmlsh", "....", "siQsQi", OP_QRDMLSH>; 331} 332 333def VQDMLAL : SInst<"vqdmlal", "(>Q)(>Q)..", "si">; 334def VQDMLSL : SInst<"vqdmlsl", "(>Q)(>Q)..", "si">; 335def VMULL : SInst<"vmull", "(>Q)..", "csiUcUsUiPc">; 336def VQDMULL : SInst<"vqdmull", "(>Q)..", "si">; 337 338//////////////////////////////////////////////////////////////////////////////// 339// E.3.3 Subtraction 340def VSUB : IOpInst<"vsub", "...", 341 "csilfUcUsUiUlQcQsQiQlQfQUcQUsQUiQUl", OP_SUB>; 342def VSUBL : SOpInst<"vsubl", "(>Q)..", "csiUcUsUi", OP_SUBL>; 343def VSUBW : SOpInst<"vsubw", "(>Q)(>Q).", "csiUcUsUi", OP_SUBW>; 344def VQSUB : SInst<"vqsub", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 345def VHSUB : SInst<"vhsub", "...", "csiUcUsUiQcQsQiQUcQUsQUi">; 346def VSUBHN : IInst<"vsubhn", "<QQ", "silUsUiUl">; 347def VRSUBHN : IInst<"vrsubhn", "<QQ", "silUsUiUl">; 348 349//////////////////////////////////////////////////////////////////////////////// 350// E.3.4 Comparison 351def VCEQ : IOpInst<"vceq", "U..", "csifUcUsUiPcQcQsQiQfQUcQUsQUiQPc", OP_EQ>; 352def VCGE : SOpInst<"vcge", "U..", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_GE>; 353let InstName = "vcge" in 354def VCLE : SOpInst<"vcle", "U..", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_LE>; 355def VCGT : SOpInst<"vcgt", "U..", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_GT>; 356let InstName = "vcgt" in 357def VCLT : SOpInst<"vclt", "U..", "csifUcUsUiQcQsQiQfQUcQUsQUi", OP_LT>; 358let InstName = "vacge" in { 359def VCAGE : IInst<"vcage", "U..", "fQf">; 360def VCALE : IInst<"vcale", "U..", "fQf">; 361} 362let InstName = "vacgt" in { 363def VCAGT : IInst<"vcagt", "U..", "fQf">; 364def VCALT : IInst<"vcalt", "U..", "fQf">; 365} 366def VTST : WInst<"vtst", "U..", "csiUcUsUiPcPsQcQsQiQUcQUsQUiQPcQPs">; 367 368//////////////////////////////////////////////////////////////////////////////// 369// E.3.5 Absolute Difference 370def VABD : SInst<"vabd", "...", "csiUcUsUifQcQsQiQUcQUsQUiQf">; 371def VABDL : SOpInst<"vabdl", "(>Q)..", "csiUcUsUi", OP_ABDL>; 372def VABA : SOpInst<"vaba", "....", "csiUcUsUiQcQsQiQUcQUsQUi", OP_ABA>; 373def VABAL : SOpInst<"vabal", "(>Q)(>Q)..", "csiUcUsUi", OP_ABAL>; 374 375//////////////////////////////////////////////////////////////////////////////// 376// E.3.6 Max/Min 377def VMAX : SInst<"vmax", "...", "csiUcUsUifQcQsQiQUcQUsQUiQf">; 378def VMIN : SInst<"vmin", "...", "csiUcUsUifQcQsQiQUcQUsQUiQf">; 379 380//////////////////////////////////////////////////////////////////////////////// 381// E.3.7 Pairwise Addition 382def VPADD : IInst<"vpadd", "...", "csiUcUsUif">; 383def VPADDL : SInst<"vpaddl", ">.", "csiUcUsUiQcQsQiQUcQUsQUi">; 384def VPADAL : SInst<"vpadal", ">>.", "csiUcUsUiQcQsQiQUcQUsQUi">; 385 386//////////////////////////////////////////////////////////////////////////////// 387// E.3.8-9 Folding Max/Min 388def VPMAX : SInst<"vpmax", "...", "csiUcUsUif">; 389def VPMIN : SInst<"vpmin", "...", "csiUcUsUif">; 390 391//////////////////////////////////////////////////////////////////////////////// 392// E.3.10 Reciprocal/Sqrt 393def VRECPS : IInst<"vrecps", "...", "fQf">; 394def VRSQRTS : IInst<"vrsqrts", "...", "fQf">; 395 396//////////////////////////////////////////////////////////////////////////////// 397// E.3.11 Shifts by signed variable 398def VSHL : SInst<"vshl", "..S", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 399def VQSHL : SInst<"vqshl", "..S", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 400def VRSHL : SInst<"vrshl", "..S", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 401def VQRSHL : SInst<"vqrshl", "..S", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 402 403//////////////////////////////////////////////////////////////////////////////// 404// E.3.12 Shifts by constant 405let isShift = 1 in { 406def VSHR_N : SInst<"vshr_n", "..I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 407def VSHL_N : IInst<"vshl_n", "..I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 408def VRSHR_N : SInst<"vrshr_n", "..I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 409def VSRA_N : SInst<"vsra_n", "...I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 410def VRSRA_N : SInst<"vrsra_n", "...I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 411def VQSHL_N : SInst<"vqshl_n", "..I", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl">; 412def VQSHLU_N : SInst<"vqshlu_n", "U.I", "csilQcQsQiQl">; 413def VSHRN_N : IInst<"vshrn_n", "<QI", "silUsUiUl">; 414def VQSHRUN_N : SInst<"vqshrun_n", "(<U)QI", "sil">; 415def VQRSHRUN_N : SInst<"vqrshrun_n", "(<U)QI", "sil">; 416def VQSHRN_N : SInst<"vqshrn_n", "<QI", "silUsUiUl">; 417def VRSHRN_N : IInst<"vrshrn_n", "<QI", "silUsUiUl">; 418def VQRSHRN_N : SInst<"vqrshrn_n", "<QI", "silUsUiUl">; 419def VSHLL_N : SInst<"vshll_n", "(>Q).I", "csiUcUsUi">; 420 421//////////////////////////////////////////////////////////////////////////////// 422// E.3.13 Shifts with insert 423def VSRI_N : WInst<"vsri_n", "...I", 424 "csilUcUsUiUlPcPsQcQsQiQlQUcQUsQUiQUlQPcQPs">; 425def VSLI_N : WInst<"vsli_n", "...I", 426 "csilUcUsUiUlPcPsQcQsQiQlQUcQUsQUiQUlQPcQPs">; 427} 428 429//////////////////////////////////////////////////////////////////////////////// 430// E.3.14 Loads and stores of a single vector 431def VLD1 : WInst<"vld1", ".(c*!)", 432 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">; 433def VLD1_X2 : WInst<"vld1_x2", "2(c*!)", 434 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">; 435def VLD1_X3 : WInst<"vld1_x3", "3(c*!)", 436 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">; 437def VLD1_X4 : WInst<"vld1_x4", "4(c*!)", 438 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">; 439def VLD1_LANE : WInst<"vld1_lane", ".(c*!).I", 440 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">; 441def VLD1_DUP : WInst<"vld1_dup", ".(c*!)", 442 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">; 443def VST1 : WInst<"vst1", "v*(.!)", 444 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">; 445def VST1_X2 : WInst<"vst1_x2", "v*(2!)", 446 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">; 447def VST1_X3 : WInst<"vst1_x3", "v*(3!)", 448 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">; 449def VST1_X4 : WInst<"vst1_x4", "v*(4!)", 450 "cfilsUcUiUlUsQcQfQiQlQsQUcQUiQUlQUsPcPsQPcQPs">; 451def VST1_LANE : WInst<"vst1_lane", "v*(.!)I", 452 "QUcQUsQUiQUlQcQsQiQlQfQPcQPsUcUsUiUlcsilfPcPs">; 453let ArchGuard = "(__ARM_FP & 2)" in { 454def VLD1_F16 : WInst<"vld1", ".(c*!)", "hQh">; 455def VLD1_X2_F16 : WInst<"vld1_x2", "2(c*!)", "hQh">; 456def VLD1_X3_F16 : WInst<"vld1_x3", "3(c*!)", "hQh">; 457def VLD1_X4_F16 : WInst<"vld1_x4", "4(c*!)", "hQh">; 458def VLD1_LANE_F16 : WInst<"vld1_lane", ".(c*!).I", "hQh">; 459def VLD1_DUP_F16 : WInst<"vld1_dup", ".(c*!)", "hQh">; 460def VST1_F16 : WInst<"vst1", "v*(.!)", "hQh">; 461def VST1_X2_F16 : WInst<"vst1_x2", "v*(2!)", "hQh">; 462def VST1_X3_F16 : WInst<"vst1_x3", "v*(3!)", "hQh">; 463def VST1_X4_F16 : WInst<"vst1_x4", "v*(4!)", "hQh">; 464def VST1_LANE_F16 : WInst<"vst1_lane", "v*(.!)I", "hQh">; 465} 466 467//////////////////////////////////////////////////////////////////////////////// 468// E.3.15 Loads and stores of an N-element structure 469def VLD2 : WInst<"vld2", "2(c*!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">; 470def VLD3 : WInst<"vld3", "3(c*!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">; 471def VLD4 : WInst<"vld4", "4(c*!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">; 472def VLD2_DUP : WInst<"vld2_dup", "2(c*!)", 473 "UcUsUiUlcsilfPcPsQcQfQiQlQsQPcQPsQUcQUiQUlQUs">; 474def VLD3_DUP : WInst<"vld3_dup", "3(c*!)", 475 "UcUsUiUlcsilfPcPsQcQfQiQlQsQPcQPsQUcQUiQUlQUs">; 476def VLD4_DUP : WInst<"vld4_dup", "4(c*!)", 477 "UcUsUiUlcsilfPcPsQcQfQiQlQsQPcQPsQUcQUiQUlQUs">; 478def VLD2_LANE : WInst<"vld2_lane", "2(c*!)2I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">; 479def VLD3_LANE : WInst<"vld3_lane", "3(c*!)3I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">; 480def VLD4_LANE : WInst<"vld4_lane", "4(c*!)4I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">; 481def VST2 : WInst<"vst2", "v*(2!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">; 482def VST3 : WInst<"vst3", "v*(3!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">; 483def VST4 : WInst<"vst4", "v*(4!)", "QUcQUsQUiQcQsQiQfQPcQPsUcUsUiUlcsilfPcPs">; 484def VST2_LANE : WInst<"vst2_lane", "v*(2!)I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">; 485def VST3_LANE : WInst<"vst3_lane", "v*(3!)I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">; 486def VST4_LANE : WInst<"vst4_lane", "v*(4!)I", "QUsQUiQsQiQfQPsUcUsUicsifPcPs">; 487let ArchGuard = "(__ARM_FP & 2)" in { 488def VLD2_F16 : WInst<"vld2", "2(c*!)", "hQh">; 489def VLD3_F16 : WInst<"vld3", "3(c*!)", "hQh">; 490def VLD4_F16 : WInst<"vld4", "4(c*!)", "hQh">; 491def VLD2_DUP_F16 : WInst<"vld2_dup", "2(c*!)", "hQh">; 492def VLD3_DUP_F16 : WInst<"vld3_dup", "3(c*!)", "hQh">; 493def VLD4_DUP_F16 : WInst<"vld4_dup", "4(c*!)", "hQh">; 494def VLD2_LANE_F16 : WInst<"vld2_lane", "2(c*!)2I", "hQh">; 495def VLD3_LANE_F16 : WInst<"vld3_lane", "3(c*!)3I", "hQh">; 496def VLD4_LANE_F16 : WInst<"vld4_lane", "4(c*!)4I", "hQh">; 497def VST2_F16 : WInst<"vst2", "v*(2!)", "hQh">; 498def VST3_F16 : WInst<"vst3", "v*(3!)", "hQh">; 499def VST4_F16 : WInst<"vst4", "v*(4!)", "hQh">; 500def VST2_LANE_F16 : WInst<"vst2_lane", "v*(2!)I", "hQh">; 501def VST3_LANE_F16 : WInst<"vst3_lane", "v*(3!)I", "hQh">; 502def VST4_LANE_F16 : WInst<"vst4_lane", "v*(4!)I", "hQh">; 503} 504 505//////////////////////////////////////////////////////////////////////////////// 506// E.3.16 Extract lanes from a vector 507let InstName = "vmov" in 508def VGET_LANE : IInst<"vget_lane", "1.I", 509 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl">; 510 511//////////////////////////////////////////////////////////////////////////////// 512// E.3.17 Set lanes within a vector 513let InstName = "vmov" in 514def VSET_LANE : IInst<"vset_lane", ".1.I", 515 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl">; 516 517//////////////////////////////////////////////////////////////////////////////// 518// E.3.18 Initialize a vector from bit pattern 519def VCREATE : NoTestOpInst<"vcreate", ".(IU>)", "csihfUcUsUiUlPcPsl", OP_CAST> { 520 let BigEndianSafe = 1; 521} 522 523//////////////////////////////////////////////////////////////////////////////// 524// E.3.19 Set all lanes to same value 525let InstName = "vmov" in { 526def VDUP_N : WOpInst<"vdup_n", ".1", 527 "UcUsUicsiPcPshfQUcQUsQUiQcQsQiQPcQPsQhQflUlQlQUl", 528 OP_DUP>; 529def VMOV_N : WOpInst<"vmov_n", ".1", 530 "UcUsUicsiPcPshfQUcQUsQUiQcQsQiQPcQPsQhQflUlQlQUl", 531 OP_DUP>; 532} 533let InstName = "" in 534def VDUP_LANE: WOpInst<"vdup_lane", ".qI", 535 "UcUsUicsiPcPsfQUcQUsQUiQcQsQiQPcQPsQflUlQlQUl", 536 OP_DUP_LN>; 537 538//////////////////////////////////////////////////////////////////////////////// 539// E.3.20 Combining vectors 540def VCOMBINE : NoTestOpInst<"vcombine", "Q..", "csilhfUcUsUiUlPcPs", OP_CONC>; 541 542//////////////////////////////////////////////////////////////////////////////// 543// E.3.21 Splitting vectors 544// Note that the ARM NEON Reference 2.0 mistakenly document the vget_high_f16() 545// and vget_low_f16() intrinsics as AArch64-only. We (and GCC) support all 546// versions of these intrinsics in both AArch32 and AArch64 architectures. See 547// D45668 for more details. 548let InstName = "vmov" in { 549def VGET_HIGH : NoTestOpInst<"vget_high", ".Q", "csilhfUcUsUiUlPcPs", OP_HI>; 550def VGET_LOW : NoTestOpInst<"vget_low", ".Q", "csilhfUcUsUiUlPcPs", OP_LO>; 551} 552 553//////////////////////////////////////////////////////////////////////////////// 554// E.3.22 Converting vectors 555 556let ArchGuard = "(__ARM_FP & 2)" in { 557 def VCVT_F16_F32 : SInst<"vcvt_f16_f32", "(<q)(.!)", "Hf">; 558 def VCVT_F32_F16 : SInst<"vcvt_f32_f16", "(>Q)(.!)", "h">; 559} 560 561def VCVT_S32 : SInst<"vcvt_s32", "S.", "fQf">; 562def VCVT_U32 : SInst<"vcvt_u32", "U.", "fQf">; 563def VCVT_F32 : SInst<"vcvt_f32", "F(.!)", "iUiQiQUi">; 564let isVCVT_N = 1 in { 565def VCVT_N_S32 : SInst<"vcvt_n_s32", "S.I", "fQf">; 566def VCVT_N_U32 : SInst<"vcvt_n_u32", "U.I", "fQf">; 567def VCVT_N_F32 : SInst<"vcvt_n_f32", "F(.!)I", "iUiQiQUi">; 568} 569 570def VMOVN : IInst<"vmovn", "<Q", "silUsUiUl">; 571def VMOVL : SInst<"vmovl", "(>Q).", "csiUcUsUi">; 572def VQMOVN : SInst<"vqmovn", "<Q", "silUsUiUl">; 573def VQMOVUN : SInst<"vqmovun", "(<U)Q", "sil">; 574 575//////////////////////////////////////////////////////////////////////////////// 576// E.3.23-24 Table lookup, Extended table lookup 577let InstName = "vtbl" in { 578def VTBL1 : WInst<"vtbl1", "..p", "UccPc">; 579def VTBL2 : WInst<"vtbl2", ".2p", "UccPc">; 580def VTBL3 : WInst<"vtbl3", ".3p", "UccPc">; 581def VTBL4 : WInst<"vtbl4", ".4p", "UccPc">; 582} 583let InstName = "vtbx" in { 584def VTBX1 : WInst<"vtbx1", "...p", "UccPc">; 585def VTBX2 : WInst<"vtbx2", "..2p", "UccPc">; 586def VTBX3 : WInst<"vtbx3", "..3p", "UccPc">; 587def VTBX4 : WInst<"vtbx4", "..4p", "UccPc">; 588} 589 590//////////////////////////////////////////////////////////////////////////////// 591// E.3.25 Operations with a scalar value 592def VMLA_LANE : IOpInst<"vmla_lane", "...qI", 593 "siUsUifQsQiQUsQUiQf", OP_MLA_LN>; 594def VMLAL_LANE : SOpInst<"vmlal_lane", "(>Q)(>Q)..I", "siUsUi", OP_MLAL_LN>; 595def VQDMLAL_LANE : SOpInst<"vqdmlal_lane", "(>Q)(>Q)..I", "si", OP_QDMLAL_LN>; 596def VMLS_LANE : IOpInst<"vmls_lane", "...qI", 597 "siUsUifQsQiQUsQUiQf", OP_MLS_LN>; 598def VMLSL_LANE : SOpInst<"vmlsl_lane", "(>Q)(>Q)..I", "siUsUi", OP_MLSL_LN>; 599def VQDMLSL_LANE : SOpInst<"vqdmlsl_lane", "(>Q)(>Q)..I", "si", OP_QDMLSL_LN>; 600def VMUL_N : IOpInst<"vmul_n", "..1", "sifUsUiQsQiQfQUsQUi", OP_MUL_N>; 601def VMUL_LANE : IOpInst<"vmul_lane", "..qI", 602 "sifUsUiQsQiQfQUsQUi", OP_MUL_LN>; 603def VMULL_N : SOpInst<"vmull_n", "(>Q).1", "siUsUi", OP_MULL_N>; 604def VMULL_LANE : SOpInst<"vmull_lane", "(>Q)..I", "siUsUi", OP_MULL_LN>; 605def VQDMULL_N : SOpInst<"vqdmull_n", "(>Q).1", "si", OP_QDMULL_N>; 606def VQDMULL_LANE : SOpInst<"vqdmull_lane", "(>Q)..I", "si", OP_QDMULL_LN>; 607def VQDMULH_N : SOpInst<"vqdmulh_n", "..1", "siQsQi", OP_QDMULH_N>; 608def VQRDMULH_N : SOpInst<"vqrdmulh_n", "..1", "siQsQi", OP_QRDMULH_N>; 609 610let ArchGuard = "!defined(__aarch64__)" in { 611def VQDMULH_LANE : SOpInst<"vqdmulh_lane", "..qI", "siQsQi", OP_QDMULH_LN>; 612def VQRDMULH_LANE : SOpInst<"vqrdmulh_lane", "..qI", "siQsQi", OP_QRDMULH_LN>; 613} 614let ArchGuard = "defined(__aarch64__)" in { 615def A64_VQDMULH_LANE : SInst<"vqdmulh_lane", "..(!q)I", "siQsQi">; 616def A64_VQRDMULH_LANE : SInst<"vqrdmulh_lane", "..(!q)I", "siQsQi">; 617} 618 619let ArchGuard = "defined(__ARM_FEATURE_QRDMX)" in { 620def VQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "...qI", "siQsQi", OP_QRDMLAH_LN>; 621def VQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "...qI", "siQsQi", OP_QRDMLSH_LN>; 622} 623 624def VMLA_N : IOpInst<"vmla_n", "...1", "siUsUifQsQiQUsQUiQf", OP_MLA_N>; 625def VMLAL_N : SOpInst<"vmlal_n", "(>Q)(>Q).1", "siUsUi", OP_MLAL_N>; 626def VQDMLAL_N : SOpInst<"vqdmlal_n", "(>Q)(>Q).1", "si", OP_QDMLAL_N>; 627def VMLS_N : IOpInst<"vmls_n", "...1", "siUsUifQsQiQUsQUiQf", OP_MLS_N>; 628def VMLSL_N : SOpInst<"vmlsl_n", "(>Q)(>Q).1", "siUsUi", OP_MLSL_N>; 629def VQDMLSL_N : SOpInst<"vqdmlsl_n", "(>Q)(>Q).1", "si", OP_QDMLSL_N>; 630 631//////////////////////////////////////////////////////////////////////////////// 632// E.3.26 Vector Extract 633def VEXT : WInst<"vext", "...I", 634 "cUcPcsUsPsiUilUlfQcQUcQPcQsQUsQPsQiQUiQlQUlQf">; 635 636//////////////////////////////////////////////////////////////////////////////// 637// E.3.27 Reverse vector elements 638def VREV64 : WOpInst<"vrev64", "..", "csiUcUsUiPcPsfQcQsQiQUcQUsQUiQPcQPsQf", 639 OP_REV64>; 640def VREV32 : WOpInst<"vrev32", "..", "csUcUsPcPsQcQsQUcQUsQPcQPs", OP_REV32>; 641def VREV16 : WOpInst<"vrev16", "..", "cUcPcQcQUcQPc", OP_REV16>; 642 643//////////////////////////////////////////////////////////////////////////////// 644// E.3.28 Other single operand arithmetic 645def VABS : SInst<"vabs", "..", "csifQcQsQiQf">; 646def VQABS : SInst<"vqabs", "..", "csiQcQsQi">; 647def VNEG : SOpInst<"vneg", "..", "csifQcQsQiQf", OP_NEG>; 648def VQNEG : SInst<"vqneg", "..", "csiQcQsQi">; 649def VCLS : SInst<"vcls", "..", "csiQcQsQi">; 650def VCLZ : IInst<"vclz", "..", "csiUcUsUiQcQsQiQUcQUsQUi">; 651def VCNT : WInst<"vcnt", "..", "UccPcQUcQcQPc">; 652def VRECPE : SInst<"vrecpe", "..", "fUiQfQUi">; 653def VRSQRTE : SInst<"vrsqrte", "..", "fUiQfQUi">; 654 655//////////////////////////////////////////////////////////////////////////////// 656// E.3.29 Logical operations 657def VMVN : LOpInst<"vmvn", "..", "csiUcUsUiPcQcQsQiQUcQUsQUiQPc", OP_NOT>; 658def VAND : LOpInst<"vand", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_AND>; 659def VORR : LOpInst<"vorr", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_OR>; 660def VEOR : LOpInst<"veor", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_XOR>; 661def VBIC : LOpInst<"vbic", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_ANDN>; 662def VORN : LOpInst<"vorn", "...", "csilUcUsUiUlQcQsQiQlQUcQUsQUiQUl", OP_ORN>; 663let isHiddenLInst = 1 in 664def VBSL : SInst<"vbsl", ".U..", 665 "csilUcUsUiUlfPcPsQcQsQiQlQUcQUsQUiQUlQfQPcQPs">; 666 667//////////////////////////////////////////////////////////////////////////////// 668// E.3.30 Transposition operations 669def VTRN : WInst<"vtrn", "2..", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">; 670def VZIP : WInst<"vzip", "2..", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">; 671def VUZP : WInst<"vuzp", "2..", "csiUcUsUifPcPsQcQsQiQUcQUsQUiQfQPcQPs">; 672 673//////////////////////////////////////////////////////////////////////////////// 674 675class REINTERPRET_CROSS_SELF<string Types> : 676 NoTestOpInst<"vreinterpret", "..", Types, OP_REINT> { 677 let CartesianProductWith = Types; 678} 679 680multiclass REINTERPRET_CROSS_TYPES<string TypesA, string TypesB> { 681 def AXB: NoTestOpInst<"vreinterpret", "..", TypesA, OP_REINT> { 682 let CartesianProductWith = TypesB; 683 } 684 def BXA: NoTestOpInst<"vreinterpret", "..", TypesB, OP_REINT> { 685 let CartesianProductWith = TypesA; 686 } 687} 688 689// E.3.31 Vector reinterpret cast operations 690def VREINTERPRET : REINTERPRET_CROSS_SELF<"csilUcUsUiUlhfPcPsQcQsQiQlQUcQUsQUiQUlQhQfQPcQPs"> { 691 let ArchGuard = "!defined(__aarch64__)"; 692 let BigEndianSafe = 1; 693} 694 695//////////////////////////////////////////////////////////////////////////////// 696// Vector fused multiply-add operations 697 698let ArchGuard = "defined(__ARM_FEATURE_FMA)" in { 699 def VFMA : SInst<"vfma", "....", "fQf">; 700 def VFMS : SOpInst<"vfms", "....", "fQf", OP_FMLS>; 701 def FMLA_N_F32 : SOpInst<"vfma_n", "...1", "fQf", OP_FMLA_N>; 702} 703 704//////////////////////////////////////////////////////////////////////////////// 705// fp16 vector operations 706def SCALAR_HALF_GET_LANE : IOpInst<"vget_lane", "1.I", "h", OP_SCALAR_HALF_GET_LN>; 707def SCALAR_HALF_SET_LANE : IOpInst<"vset_lane", ".1.I", "h", OP_SCALAR_HALF_SET_LN>; 708def SCALAR_HALF_GET_LANEQ : IOpInst<"vget_lane", "1.I", "Qh", OP_SCALAR_HALF_GET_LNQ>; 709def SCALAR_HALF_SET_LANEQ : IOpInst<"vset_lane", ".1.I", "Qh", OP_SCALAR_HALF_SET_LNQ>; 710 711//////////////////////////////////////////////////////////////////////////////// 712// AArch64 Intrinsics 713 714let ArchGuard = "defined(__aarch64__)" in { 715 716//////////////////////////////////////////////////////////////////////////////// 717// Load/Store 718def LD1 : WInst<"vld1", ".(c*!)", "dQdPlQPl">; 719def LD2 : WInst<"vld2", "2(c*!)", "QUlQldQdPlQPl">; 720def LD3 : WInst<"vld3", "3(c*!)", "QUlQldQdPlQPl">; 721def LD4 : WInst<"vld4", "4(c*!)", "QUlQldQdPlQPl">; 722def ST1 : WInst<"vst1", "v*(.!)", "dQdPlQPl">; 723def ST2 : WInst<"vst2", "v*(2!)", "QUlQldQdPlQPl">; 724def ST3 : WInst<"vst3", "v*(3!)", "QUlQldQdPlQPl">; 725def ST4 : WInst<"vst4", "v*(4!)", "QUlQldQdPlQPl">; 726 727def LD1_X2 : WInst<"vld1_x2", "2(c*!)", 728 "dQdPlQPl">; 729def LD1_X3 : WInst<"vld1_x3", "3(c*!)", 730 "dQdPlQPl">; 731def LD1_X4 : WInst<"vld1_x4", "4(c*!)", 732 "dQdPlQPl">; 733 734def ST1_X2 : WInst<"vst1_x2", "v*(2!)", "dQdPlQPl">; 735def ST1_X3 : WInst<"vst1_x3", "v*(3!)", "dQdPlQPl">; 736def ST1_X4 : WInst<"vst1_x4", "v*(4!)", "dQdPlQPl">; 737 738def LD1_LANE : WInst<"vld1_lane", ".(c*!).I", "dQdPlQPl">; 739def LD2_LANE : WInst<"vld2_lane", "2(c*!)2I", "lUlQcQUcQPcQlQUldQdPlQPl">; 740def LD3_LANE : WInst<"vld3_lane", "3(c*!)3I", "lUlQcQUcQPcQlQUldQdPlQPl">; 741def LD4_LANE : WInst<"vld4_lane", "4(c*!)4I", "lUlQcQUcQPcQlQUldQdPlQPl">; 742def ST1_LANE : WInst<"vst1_lane", "v*(.!)I", "dQdPlQPl">; 743def ST2_LANE : WInst<"vst2_lane", "v*(2!)I", "lUlQcQUcQPcQlQUldQdPlQPl">; 744def ST3_LANE : WInst<"vst3_lane", "v*(3!)I", "lUlQcQUcQPcQlQUldQdPlQPl">; 745def ST4_LANE : WInst<"vst4_lane", "v*(4!)I", "lUlQcQUcQPcQlQUldQdPlQPl">; 746 747def LD1_DUP : WInst<"vld1_dup", ".(c*!)", "dQdPlQPl">; 748def LD2_DUP : WInst<"vld2_dup", "2(c*!)", "dQdPlQPl">; 749def LD3_DUP : WInst<"vld3_dup", "3(c*!)", "dQdPlQPl">; 750def LD4_DUP : WInst<"vld4_dup", "4(c*!)", "dQdPlQPl">; 751 752def VLDRQ : WInst<"vldrq", "1(c*!)", "Pk">; 753def VSTRQ : WInst<"vstrq", "v*(1!)", "Pk">; 754 755//////////////////////////////////////////////////////////////////////////////// 756// Addition 757def ADD : IOpInst<"vadd", "...", "dQd", OP_ADD>; 758 759//////////////////////////////////////////////////////////////////////////////// 760// Subtraction 761def SUB : IOpInst<"vsub", "...", "dQd", OP_SUB>; 762 763//////////////////////////////////////////////////////////////////////////////// 764// Multiplication 765def MUL : IOpInst<"vmul", "...", "dQd", OP_MUL>; 766def MLA : IOpInst<"vmla", "....", "dQd", OP_MLA>; 767def MLS : IOpInst<"vmls", "....", "dQd", OP_MLS>; 768 769//////////////////////////////////////////////////////////////////////////////// 770// Multiplication Extended 771def MULX : SInst<"vmulx", "...", "fdQfQd">; 772 773//////////////////////////////////////////////////////////////////////////////// 774// Division 775def FDIV : IOpInst<"vdiv", "...", "fdQfQd", OP_DIV>; 776 777//////////////////////////////////////////////////////////////////////////////// 778// Vector fused multiply-add operations 779def FMLA : SInst<"vfma", "....", "dQd">; 780def FMLS : SOpInst<"vfms", "....", "dQd", OP_FMLS>; 781 782//////////////////////////////////////////////////////////////////////////////// 783// MUL, MLA, MLS, FMA, FMS definitions with scalar argument 784def VMUL_N_A64 : IOpInst<"vmul_n", "..1", "Qd", OP_MUL_N>; 785 786def FMLA_N : SOpInst<"vfma_n", "...1", "dQd", OP_FMLA_N>; 787def FMLS_N : SOpInst<"vfms_n", "...1", "fdQfQd", OP_FMLS_N>; 788 789//////////////////////////////////////////////////////////////////////////////// 790// Logical operations 791def BSL : SInst<"vbsl", ".U..", "dPlQdQPl">; 792 793//////////////////////////////////////////////////////////////////////////////// 794// Absolute Difference 795def ABD : SInst<"vabd", "...", "dQd">; 796 797//////////////////////////////////////////////////////////////////////////////// 798// saturating absolute/negate 799def ABS : SInst<"vabs", "..", "dQdlQl">; 800def QABS : SInst<"vqabs", "..", "lQl">; 801def NEG : SOpInst<"vneg", "..", "dlQdQl", OP_NEG>; 802def QNEG : SInst<"vqneg", "..", "lQl">; 803 804//////////////////////////////////////////////////////////////////////////////// 805// Signed Saturating Accumulated of Unsigned Value 806def SUQADD : SInst<"vuqadd", "..U", "csilQcQsQiQl">; 807 808//////////////////////////////////////////////////////////////////////////////// 809// Unsigned Saturating Accumulated of Signed Value 810def USQADD : SInst<"vsqadd", "..S", "UcUsUiUlQUcQUsQUiQUl">; 811 812//////////////////////////////////////////////////////////////////////////////// 813// Reciprocal/Sqrt 814def FRECPS : IInst<"vrecps", "...", "dQd">; 815def FRSQRTS : IInst<"vrsqrts", "...", "dQd">; 816def FRECPE : SInst<"vrecpe", "..", "dQd">; 817def FRSQRTE : SInst<"vrsqrte", "..", "dQd">; 818def FSQRT : SInst<"vsqrt", "..", "fdQfQd">; 819 820//////////////////////////////////////////////////////////////////////////////// 821// bitwise reverse 822def RBIT : IInst<"vrbit", "..", "cUcPcQcQUcQPc">; 823 824//////////////////////////////////////////////////////////////////////////////// 825// Integer extract and narrow to high 826def XTN2 : SOpInst<"vmovn_high", "(<Q)<Q", "silUsUiUl", OP_XTN>; 827 828//////////////////////////////////////////////////////////////////////////////// 829// Signed integer saturating extract and unsigned narrow to high 830def SQXTUN2 : SOpInst<"vqmovun_high", "(<U)(<Uq).", "HsHiHl", OP_SQXTUN>; 831 832//////////////////////////////////////////////////////////////////////////////// 833// Integer saturating extract and narrow to high 834def QXTN2 : SOpInst<"vqmovn_high", "(<Q)<Q", "silUsUiUl", OP_QXTN>; 835 836//////////////////////////////////////////////////////////////////////////////// 837// Converting vectors 838 839def VCVT_F32_F64 : SInst<"vcvt_f32_f64", "(<q).", "Qd">; 840def VCVT_F64_F32 : SInst<"vcvt_f64_f32", "(>Q).", "f">; 841 842def VCVT_S64 : SInst<"vcvt_s64", "S.", "dQd">; 843def VCVT_U64 : SInst<"vcvt_u64", "U.", "dQd">; 844def VCVT_F64 : SInst<"vcvt_f64", "F(.!)", "lUlQlQUl">; 845 846def VCVT_HIGH_F16_F32 : SOpInst<"vcvt_high_f16", "<(<q!)Q", "Hf", OP_VCVT_NA_HI_F16>; 847def VCVT_HIGH_F32_F16 : SOpInst<"vcvt_high_f32", "(>Q)(Q!)", "h", OP_VCVT_EX_HI_F32>; 848def VCVT_HIGH_F32_F64 : SOpInst<"vcvt_high_f32", "(<Q)(F<!)Q", "d", OP_VCVT_NA_HI_F32>; 849def VCVT_HIGH_F64_F32 : SOpInst<"vcvt_high_f64", "(>Q)(Q!)", "f", OP_VCVT_EX_HI_F64>; 850 851def VCVTX_F32_F64 : SInst<"vcvtx_f32", "(F<)(Q!)", "d">; 852def VCVTX_HIGH_F32_F64 : SOpInst<"vcvtx_high_f32", "(<Q)(F<!)Q", "d", OP_VCVTX_HI>; 853 854//////////////////////////////////////////////////////////////////////////////// 855// Comparison 856def FCAGE : IInst<"vcage", "U..", "dQd">; 857def FCAGT : IInst<"vcagt", "U..", "dQd">; 858def FCALE : IInst<"vcale", "U..", "dQd">; 859def FCALT : IInst<"vcalt", "U..", "dQd">; 860def CMTST : WInst<"vtst", "U..", "lUlPlQlQUlQPl">; 861def CFMEQ : SOpInst<"vceq", "U..", "lUldQdQlQUlPlQPl", OP_EQ>; 862def CFMGE : SOpInst<"vcge", "U..", "lUldQdQlQUl", OP_GE>; 863def CFMLE : SOpInst<"vcle", "U..", "lUldQdQlQUl", OP_LE>; 864def CFMGT : SOpInst<"vcgt", "U..", "lUldQdQlQUl", OP_GT>; 865def CFMLT : SOpInst<"vclt", "U..", "lUldQdQlQUl", OP_LT>; 866 867def CMEQ : SInst<"vceqz", "U.", 868 "csilfUcUsUiUlPcPlQcQsQiQlQfQUcQUsQUiQUlQPcdQdQPl">; 869def CMGE : SInst<"vcgez", "U.", "csilfdQcQsQiQlQfQd">; 870def CMLE : SInst<"vclez", "U.", "csilfdQcQsQiQlQfQd">; 871def CMGT : SInst<"vcgtz", "U.", "csilfdQcQsQiQlQfQd">; 872def CMLT : SInst<"vcltz", "U.", "csilfdQcQsQiQlQfQd">; 873 874//////////////////////////////////////////////////////////////////////////////// 875// Max/Min Integer 876def MAX : SInst<"vmax", "...", "dQd">; 877def MIN : SInst<"vmin", "...", "dQd">; 878 879//////////////////////////////////////////////////////////////////////////////// 880// Pairwise Max/Min 881def MAXP : SInst<"vpmax", "...", "QcQsQiQUcQUsQUiQfQd">; 882def MINP : SInst<"vpmin", "...", "QcQsQiQUcQUsQUiQfQd">; 883 884//////////////////////////////////////////////////////////////////////////////// 885// Pairwise MaxNum/MinNum Floating Point 886def FMAXNMP : SInst<"vpmaxnm", "...", "fQfQd">; 887def FMINNMP : SInst<"vpminnm", "...", "fQfQd">; 888 889//////////////////////////////////////////////////////////////////////////////// 890// Pairwise Addition 891def ADDP : IInst<"vpadd", "...", "QcQsQiQlQUcQUsQUiQUlQfQd">; 892 893//////////////////////////////////////////////////////////////////////////////// 894// Shifts by constant 895let isShift = 1 in { 896// Left shift long high 897def SHLL_HIGH_N : SOpInst<"vshll_high_n", ">.I", "HcHsHiHUcHUsHUi", 898 OP_LONG_HI>; 899 900//////////////////////////////////////////////////////////////////////////////// 901def SRI_N : WInst<"vsri_n", "...I", "PlQPl">; 902def SLI_N : WInst<"vsli_n", "...I", "PlQPl">; 903 904// Right shift narrow high 905def SHRN_HIGH_N : IOpInst<"vshrn_high_n", "<(<q).I", 906 "HsHiHlHUsHUiHUl", OP_NARROW_HI>; 907def QSHRUN_HIGH_N : SOpInst<"vqshrun_high_n", "<(<q).I", 908 "HsHiHl", OP_NARROW_HI>; 909def RSHRN_HIGH_N : IOpInst<"vrshrn_high_n", "<(<q).I", 910 "HsHiHlHUsHUiHUl", OP_NARROW_HI>; 911def QRSHRUN_HIGH_N : SOpInst<"vqrshrun_high_n", "<(<q).I", 912 "HsHiHl", OP_NARROW_HI>; 913def QSHRN_HIGH_N : SOpInst<"vqshrn_high_n", "<(<q).I", 914 "HsHiHlHUsHUiHUl", OP_NARROW_HI>; 915def QRSHRN_HIGH_N : SOpInst<"vqrshrn_high_n", "<(<q).I", 916 "HsHiHlHUsHUiHUl", OP_NARROW_HI>; 917} 918 919//////////////////////////////////////////////////////////////////////////////// 920// Converting vectors 921def VMOVL_HIGH : SOpInst<"vmovl_high", ">.", "HcHsHiHUcHUsHUi", OP_MOVL_HI>; 922 923let isVCVT_N = 1 in { 924def CVTF_N_F64 : SInst<"vcvt_n_f64", "F(.!)I", "lUlQlQUl">; 925def FCVTZS_N_S64 : SInst<"vcvt_n_s64", "S.I", "dQd">; 926def FCVTZS_N_U64 : SInst<"vcvt_n_u64", "U.I", "dQd">; 927} 928 929//////////////////////////////////////////////////////////////////////////////// 930// 3VDiff class using high 64-bit in operands 931def VADDL_HIGH : SOpInst<"vaddl_high", "(>Q)QQ", "csiUcUsUi", OP_ADDLHi>; 932def VADDW_HIGH : SOpInst<"vaddw_high", "(>Q)(>Q)Q", "csiUcUsUi", OP_ADDWHi>; 933def VSUBL_HIGH : SOpInst<"vsubl_high", "(>Q)QQ", "csiUcUsUi", OP_SUBLHi>; 934def VSUBW_HIGH : SOpInst<"vsubw_high", "(>Q)(>Q)Q", "csiUcUsUi", OP_SUBWHi>; 935 936def VABDL_HIGH : SOpInst<"vabdl_high", "(>Q)QQ", "csiUcUsUi", OP_ABDLHi>; 937def VABAL_HIGH : SOpInst<"vabal_high", "(>Q)(>Q)QQ", "csiUcUsUi", OP_ABALHi>; 938 939def VMULL_HIGH : SOpInst<"vmull_high", "(>Q)QQ", "csiUcUsUiPc", OP_MULLHi>; 940def VMULL_HIGH_N : SOpInst<"vmull_high_n", "(>Q)Q1", "siUsUi", OP_MULLHi_N>; 941def VMLAL_HIGH : SOpInst<"vmlal_high", "(>Q)(>Q)QQ", "csiUcUsUi", OP_MLALHi>; 942def VMLAL_HIGH_N : SOpInst<"vmlal_high_n", "(>Q)(>Q)Q1", "siUsUi", OP_MLALHi_N>; 943def VMLSL_HIGH : SOpInst<"vmlsl_high", "(>Q)(>Q)QQ", "csiUcUsUi", OP_MLSLHi>; 944def VMLSL_HIGH_N : SOpInst<"vmlsl_high_n", "(>Q)(>Q)Q1", "siUsUi", OP_MLSLHi_N>; 945 946def VADDHN_HIGH : SOpInst<"vaddhn_high", "(<Q)<QQ", "silUsUiUl", OP_ADDHNHi>; 947def VRADDHN_HIGH : SOpInst<"vraddhn_high", "(<Q)<QQ", "silUsUiUl", OP_RADDHNHi>; 948def VSUBHN_HIGH : SOpInst<"vsubhn_high", "(<Q)<QQ", "silUsUiUl", OP_SUBHNHi>; 949def VRSUBHN_HIGH : SOpInst<"vrsubhn_high", "(<Q)<QQ", "silUsUiUl", OP_RSUBHNHi>; 950 951def VQDMULL_HIGH : SOpInst<"vqdmull_high", "(>Q)QQ", "si", OP_QDMULLHi>; 952def VQDMULL_HIGH_N : SOpInst<"vqdmull_high_n", "(>Q)Q1", "si", OP_QDMULLHi_N>; 953def VQDMLAL_HIGH : SOpInst<"vqdmlal_high", "(>Q)(>Q)QQ", "si", OP_QDMLALHi>; 954def VQDMLAL_HIGH_N : SOpInst<"vqdmlal_high_n", "(>Q)(>Q)Q1", "si", OP_QDMLALHi_N>; 955def VQDMLSL_HIGH : SOpInst<"vqdmlsl_high", "(>Q)(>Q)QQ", "si", OP_QDMLSLHi>; 956def VQDMLSL_HIGH_N : SOpInst<"vqdmlsl_high_n", "(>Q)(>Q)Q1", "si", OP_QDMLSLHi_N>; 957def VMULL_P64 : SInst<"vmull", "(1>)11", "Pl">; 958def VMULL_HIGH_P64 : SOpInst<"vmull_high", "(1>)..", "HPl", OP_MULLHi_P64>; 959 960 961//////////////////////////////////////////////////////////////////////////////// 962// Extract or insert element from vector 963def GET_LANE : IInst<"vget_lane", "1.I", "dQdPlQPl">; 964def SET_LANE : IInst<"vset_lane", ".1.I", "dQdPlQPl">; 965def COPY_LANE : IOpInst<"vcopy_lane", "..I.I", 966 "csilUcUsUiUlPcPsPlfd", OP_COPY_LN>; 967def COPYQ_LANE : IOpInst<"vcopy_lane", "..IqI", 968 "QcQsQiQlQUcQUsQUiQUlQPcQPsQfQdQPl", OP_COPY_LN>; 969def COPY_LANEQ : IOpInst<"vcopy_laneq", "..IQI", 970 "csilPcPsPlUcUsUiUlfd", OP_COPY_LN> { 971 let isLaneQ = 1; 972} 973def COPYQ_LANEQ : IOpInst<"vcopy_laneq", "..I.I", 974 "QcQsQiQlQUcQUsQUiQUlQPcQPsQfQdQPl", OP_COPY_LN> { 975 let isLaneQ = 1; 976} 977 978//////////////////////////////////////////////////////////////////////////////// 979// Set all lanes to same value 980def VDUP_LANE1: WOpInst<"vdup_lane", ".qI", "hdQhQdPlQPl", OP_DUP_LN>; 981def VDUP_LANE2: WOpInst<"vdup_laneq", ".QI", 982 "csilUcUsUiUlPcPshfdQcQsQiQlQPcQPsQUcQUsQUiQUlQhQfQdPlQPl", 983 OP_DUP_LN> { 984 let isLaneQ = 1; 985} 986def DUP_N : WOpInst<"vdup_n", ".1", "dQdPlQPl", OP_DUP>; 987def MOV_N : WOpInst<"vmov_n", ".1", "dQdPlQPl", OP_DUP>; 988 989//////////////////////////////////////////////////////////////////////////////// 990def COMBINE : NoTestOpInst<"vcombine", "Q..", "dPl", OP_CONC>; 991 992//////////////////////////////////////////////////////////////////////////////// 993//Initialize a vector from bit pattern 994def CREATE : NoTestOpInst<"vcreate", ".(IU>)", "dPl", OP_CAST> { 995 let BigEndianSafe = 1; 996} 997 998//////////////////////////////////////////////////////////////////////////////// 999 1000def VMLA_LANEQ : IOpInst<"vmla_laneq", "...QI", 1001 "siUsUifQsQiQUsQUiQf", OP_MLA_LN> { 1002 let isLaneQ = 1; 1003} 1004def VMLS_LANEQ : IOpInst<"vmls_laneq", "...QI", 1005 "siUsUifQsQiQUsQUiQf", OP_MLS_LN> { 1006 let isLaneQ = 1; 1007} 1008 1009def VFMA_LANE : IInst<"vfma_lane", "...qI", "fdQfQd">; 1010def VFMA_LANEQ : IInst<"vfma_laneq", "...QI", "fdQfQd"> { 1011 let isLaneQ = 1; 1012} 1013def VFMS_LANE : IOpInst<"vfms_lane", "...qI", "fdQfQd", OP_FMS_LN>; 1014def VFMS_LANEQ : IOpInst<"vfms_laneq", "...QI", "fdQfQd", OP_FMS_LNQ> { 1015 let isLaneQ = 1; 1016} 1017 1018def VMLAL_LANEQ : SOpInst<"vmlal_laneq", "(>Q)(>Q).QI", "siUsUi", OP_MLAL_LN> { 1019 let isLaneQ = 1; 1020} 1021def VMLAL_HIGH_LANE : SOpInst<"vmlal_high_lane", "(>Q)(>Q)Q.I", "siUsUi", 1022 OP_MLALHi_LN>; 1023def VMLAL_HIGH_LANEQ : SOpInst<"vmlal_high_laneq", "(>Q)(>Q)QQI", "siUsUi", 1024 OP_MLALHi_LN> { 1025 let isLaneQ = 1; 1026} 1027def VMLSL_LANEQ : SOpInst<"vmlsl_laneq", "(>Q)(>Q).QI", "siUsUi", OP_MLSL_LN> { 1028 let isLaneQ = 1; 1029} 1030def VMLSL_HIGH_LANE : SOpInst<"vmlsl_high_lane", "(>Q)(>Q)Q.I", "siUsUi", 1031 OP_MLSLHi_LN>; 1032def VMLSL_HIGH_LANEQ : SOpInst<"vmlsl_high_laneq", "(>Q)(>Q)QQI", "siUsUi", 1033 OP_MLSLHi_LN> { 1034 let isLaneQ = 1; 1035} 1036 1037def VQDMLAL_LANEQ : SOpInst<"vqdmlal_laneq", "(>Q)(>Q).QI", "si", OP_QDMLAL_LN> { 1038 let isLaneQ = 1; 1039} 1040def VQDMLAL_HIGH_LANE : SOpInst<"vqdmlal_high_lane", "(>Q)(>Q)Q.I", "si", 1041 OP_QDMLALHi_LN>; 1042def VQDMLAL_HIGH_LANEQ : SOpInst<"vqdmlal_high_laneq", "(>Q)(>Q)QQI", "si", 1043 OP_QDMLALHi_LN> { 1044 let isLaneQ = 1; 1045} 1046def VQDMLSL_LANEQ : SOpInst<"vqdmlsl_laneq", "(>Q)(>Q).QI", "si", OP_QDMLSL_LN> { 1047 let isLaneQ = 1; 1048} 1049def VQDMLSL_HIGH_LANE : SOpInst<"vqdmlsl_high_lane", "(>Q)(>Q)Q.I", "si", 1050 OP_QDMLSLHi_LN>; 1051def VQDMLSL_HIGH_LANEQ : SOpInst<"vqdmlsl_high_laneq", "(>Q)(>Q)QQI", "si", 1052 OP_QDMLSLHi_LN> { 1053 let isLaneQ = 1; 1054} 1055 1056// Newly add double parameter for vmul_lane in aarch64 1057// Note: d type is handled by SCALAR_VMUL_LANE 1058def VMUL_LANE_A64 : IOpInst<"vmul_lane", "..qI", "Qd", OP_MUL_LN>; 1059 1060// Note: d type is handled by SCALAR_VMUL_LANEQ 1061def VMUL_LANEQ : IOpInst<"vmul_laneq", "..QI", 1062 "sifUsUiQsQiQUsQUiQfQd", OP_MUL_LN> { 1063 let isLaneQ = 1; 1064} 1065def VMULL_LANEQ : SOpInst<"vmull_laneq", "(>Q).QI", "siUsUi", OP_MULL_LN> { 1066 let isLaneQ = 1; 1067} 1068def VMULL_HIGH_LANE : SOpInst<"vmull_high_lane", "(>Q)Q.I", "siUsUi", 1069 OP_MULLHi_LN>; 1070def VMULL_HIGH_LANEQ : SOpInst<"vmull_high_laneq", "(>Q)QQI", "siUsUi", 1071 OP_MULLHi_LN> { 1072 let isLaneQ = 1; 1073} 1074 1075def VQDMULL_LANEQ : SOpInst<"vqdmull_laneq", "(>Q).QI", "si", OP_QDMULL_LN> { 1076 let isLaneQ = 1; 1077} 1078def VQDMULL_HIGH_LANE : SOpInst<"vqdmull_high_lane", "(>Q)Q.I", "si", 1079 OP_QDMULLHi_LN>; 1080def VQDMULL_HIGH_LANEQ : SOpInst<"vqdmull_high_laneq", "(>Q)QQI", "si", 1081 OP_QDMULLHi_LN> { 1082 let isLaneQ = 1; 1083} 1084 1085let isLaneQ = 1 in { 1086def VQDMULH_LANEQ : SInst<"vqdmulh_laneq", "..QI", "siQsQi">; 1087def VQRDMULH_LANEQ : SInst<"vqrdmulh_laneq", "..QI", "siQsQi">; 1088} 1089let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in { 1090def VQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "...QI", "siQsQi", OP_QRDMLAH_LN> { 1091 let isLaneQ = 1; 1092} 1093def VQRDMLSH_LANEQ : SOpInst<"vqrdmlsh_laneq", "...QI", "siQsQi", OP_QRDMLSH_LN> { 1094 let isLaneQ = 1; 1095} 1096} 1097 1098// Note: d type implemented by SCALAR_VMULX_LANE 1099def VMULX_LANE : IOpInst<"vmulx_lane", "..qI", "fQfQd", OP_MULX_LN>; 1100// Note: d type is implemented by SCALAR_VMULX_LANEQ 1101def VMULX_LANEQ : IOpInst<"vmulx_laneq", "..QI", "fQfQd", OP_MULX_LN> { 1102 let isLaneQ = 1; 1103} 1104 1105//////////////////////////////////////////////////////////////////////////////// 1106// Across vectors class 1107def VADDLV : SInst<"vaddlv", "(1>).", "csiUcUsUiQcQsQiQUcQUsQUi">; 1108def VMAXV : SInst<"vmaxv", "1.", "csifUcUsUiQcQsQiQUcQUsQUiQfQd">; 1109def VMINV : SInst<"vminv", "1.", "csifUcUsUiQcQsQiQUcQUsQUiQfQd">; 1110def VADDV : SInst<"vaddv", "1.", "csifUcUsUiQcQsQiQUcQUsQUiQfQdQlQUl">; 1111def FMAXNMV : SInst<"vmaxnmv", "1.", "fQfQd">; 1112def FMINNMV : SInst<"vminnmv", "1.", "fQfQd">; 1113 1114//////////////////////////////////////////////////////////////////////////////// 1115// Newly added Vector Extract for f64 1116def VEXT_A64 : WInst<"vext", "...I", "dQdPlQPl">; 1117 1118//////////////////////////////////////////////////////////////////////////////// 1119// Crypto 1120let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_CRYPTO)" in { 1121def AESE : SInst<"vaese", "...", "QUc">; 1122def AESD : SInst<"vaesd", "...", "QUc">; 1123def AESMC : SInst<"vaesmc", "..", "QUc">; 1124def AESIMC : SInst<"vaesimc", "..", "QUc">; 1125 1126def SHA1H : SInst<"vsha1h", "11", "Ui">; 1127def SHA1SU1 : SInst<"vsha1su1", "...", "QUi">; 1128def SHA256SU0 : SInst<"vsha256su0", "...", "QUi">; 1129 1130def SHA1C : SInst<"vsha1c", "..1.", "QUi">; 1131def SHA1P : SInst<"vsha1p", "..1.", "QUi">; 1132def SHA1M : SInst<"vsha1m", "..1.", "QUi">; 1133def SHA1SU0 : SInst<"vsha1su0", "....", "QUi">; 1134def SHA256H : SInst<"vsha256h", "....", "QUi">; 1135def SHA256H2 : SInst<"vsha256h2", "....", "QUi">; 1136def SHA256SU1 : SInst<"vsha256su1", "....", "QUi">; 1137} 1138 1139//////////////////////////////////////////////////////////////////////////////// 1140// Float -> Int conversions with explicit rounding mode 1141 1142let ArchGuard = "__ARM_ARCH >= 8" in { 1143def FCVTNS_S32 : SInst<"vcvtn_s32", "S.", "fQf">; 1144def FCVTNU_S32 : SInst<"vcvtn_u32", "U.", "fQf">; 1145def FCVTPS_S32 : SInst<"vcvtp_s32", "S.", "fQf">; 1146def FCVTPU_S32 : SInst<"vcvtp_u32", "U.", "fQf">; 1147def FCVTMS_S32 : SInst<"vcvtm_s32", "S.", "fQf">; 1148def FCVTMU_S32 : SInst<"vcvtm_u32", "U.", "fQf">; 1149def FCVTAS_S32 : SInst<"vcvta_s32", "S.", "fQf">; 1150def FCVTAU_S32 : SInst<"vcvta_u32", "U.", "fQf">; 1151} 1152 1153let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__)" in { 1154def FCVTNS_S64 : SInst<"vcvtn_s64", "S.", "dQd">; 1155def FCVTNU_S64 : SInst<"vcvtn_u64", "U.", "dQd">; 1156def FCVTPS_S64 : SInst<"vcvtp_s64", "S.", "dQd">; 1157def FCVTPU_S64 : SInst<"vcvtp_u64", "U.", "dQd">; 1158def FCVTMS_S64 : SInst<"vcvtm_s64", "S.", "dQd">; 1159def FCVTMU_S64 : SInst<"vcvtm_u64", "U.", "dQd">; 1160def FCVTAS_S64 : SInst<"vcvta_s64", "S.", "dQd">; 1161def FCVTAU_S64 : SInst<"vcvta_u64", "U.", "dQd">; 1162} 1163 1164//////////////////////////////////////////////////////////////////////////////// 1165// Round to Integral 1166 1167let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)" in { 1168def FRINTN_S32 : SInst<"vrndn", "..", "fQf">; 1169def FRINTA_S32 : SInst<"vrnda", "..", "fQf">; 1170def FRINTP_S32 : SInst<"vrndp", "..", "fQf">; 1171def FRINTM_S32 : SInst<"vrndm", "..", "fQf">; 1172def FRINTX_S32 : SInst<"vrndx", "..", "fQf">; 1173def FRINTZ_S32 : SInst<"vrnd", "..", "fQf">; 1174def FRINTI_S32 : SInst<"vrndi", "..", "fQf">; 1175} 1176 1177let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_DIRECTED_ROUNDING)" in { 1178def FRINTN_S64 : SInst<"vrndn", "..", "dQd">; 1179def FRINTA_S64 : SInst<"vrnda", "..", "dQd">; 1180def FRINTP_S64 : SInst<"vrndp", "..", "dQd">; 1181def FRINTM_S64 : SInst<"vrndm", "..", "dQd">; 1182def FRINTX_S64 : SInst<"vrndx", "..", "dQd">; 1183def FRINTZ_S64 : SInst<"vrnd", "..", "dQd">; 1184def FRINTI_S64 : SInst<"vrndi", "..", "dQd">; 1185} 1186 1187//////////////////////////////////////////////////////////////////////////////// 1188// MaxNum/MinNum Floating Point 1189 1190let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN)" in { 1191def FMAXNM_S32 : SInst<"vmaxnm", "...", "fQf">; 1192def FMINNM_S32 : SInst<"vminnm", "...", "fQf">; 1193} 1194 1195let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__) && defined(__ARM_FEATURE_NUMERIC_MAXMIN)" in { 1196def FMAXNM_S64 : SInst<"vmaxnm", "...", "dQd">; 1197def FMINNM_S64 : SInst<"vminnm", "...", "dQd">; 1198} 1199 1200//////////////////////////////////////////////////////////////////////////////// 1201// Permutation 1202def VTRN1 : SOpInst<"vtrn1", "...", 1203 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_TRN1>; 1204def VZIP1 : SOpInst<"vzip1", "...", 1205 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_ZIP1>; 1206def VUZP1 : SOpInst<"vuzp1", "...", 1207 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_UZP1>; 1208def VTRN2 : SOpInst<"vtrn2", "...", 1209 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_TRN2>; 1210def VZIP2 : SOpInst<"vzip2", "...", 1211 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_ZIP2>; 1212def VUZP2 : SOpInst<"vuzp2", "...", 1213 "csiUcUsUifPcPsQcQsQiQlQUcQUsQUiQUlQfQdQPcQPsQPl", OP_UZP2>; 1214 1215//////////////////////////////////////////////////////////////////////////////// 1216// Table lookup 1217let InstName = "vtbl" in { 1218def VQTBL1_A64 : WInst<"vqtbl1", ".QU", "UccPcQUcQcQPc">; 1219def VQTBL2_A64 : WInst<"vqtbl2", ".(2Q)U", "UccPcQUcQcQPc">; 1220def VQTBL3_A64 : WInst<"vqtbl3", ".(3Q)U", "UccPcQUcQcQPc">; 1221def VQTBL4_A64 : WInst<"vqtbl4", ".(4Q)U", "UccPcQUcQcQPc">; 1222} 1223let InstName = "vtbx" in { 1224def VQTBX1_A64 : WInst<"vqtbx1", "..QU", "UccPcQUcQcQPc">; 1225def VQTBX2_A64 : WInst<"vqtbx2", "..(2Q)U", "UccPcQUcQcQPc">; 1226def VQTBX3_A64 : WInst<"vqtbx3", "..(3Q)U", "UccPcQUcQcQPc">; 1227def VQTBX4_A64 : WInst<"vqtbx4", "..(4Q)U", "UccPcQUcQcQPc">; 1228} 1229 1230//////////////////////////////////////////////////////////////////////////////// 1231// Vector reinterpret cast operations 1232 1233// NeonEmitter implicitly takes the cartesian product of the type string with 1234// itself during generation so, unlike all other intrinsics, this one should 1235// include *all* types, not just additional ones. 1236def VVREINTERPRET : REINTERPRET_CROSS_SELF<"csilUcUsUiUlhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQdQPcQPsQPlQPk"> { 1237 let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__)"; 1238 let BigEndianSafe = 1; 1239} 1240 1241//////////////////////////////////////////////////////////////////////////////// 1242// Scalar Intrinsics 1243// Scalar Arithmetic 1244 1245// Scalar Addition 1246def SCALAR_ADD : SInst<"vadd", "111", "SlSUl">; 1247// Scalar Saturating Add 1248def SCALAR_QADD : SInst<"vqadd", "111", "ScSsSiSlSUcSUsSUiSUl">; 1249 1250// Scalar Subtraction 1251def SCALAR_SUB : SInst<"vsub", "111", "SlSUl">; 1252// Scalar Saturating Sub 1253def SCALAR_QSUB : SInst<"vqsub", "111", "ScSsSiSlSUcSUsSUiSUl">; 1254 1255let InstName = "vmov" in { 1256def VGET_HIGH_A64 : NoTestOpInst<"vget_high", ".Q", "dPl", OP_HI>; 1257def VGET_LOW_A64 : NoTestOpInst<"vget_low", ".Q", "dPl", OP_LO>; 1258} 1259 1260//////////////////////////////////////////////////////////////////////////////// 1261// Scalar Shift 1262// Scalar Shift Left 1263def SCALAR_SHL: SInst<"vshl", "11(S1)", "SlSUl">; 1264// Scalar Saturating Shift Left 1265def SCALAR_QSHL: SInst<"vqshl", "11(S1)", "ScSsSiSlSUcSUsSUiSUl">; 1266// Scalar Saturating Rounding Shift Left 1267def SCALAR_QRSHL: SInst<"vqrshl", "11(S1)", "ScSsSiSlSUcSUsSUiSUl">; 1268// Scalar Shift Rounding Left 1269def SCALAR_RSHL: SInst<"vrshl", "11(S1)", "SlSUl">; 1270 1271//////////////////////////////////////////////////////////////////////////////// 1272// Scalar Shift (Immediate) 1273let isScalarShift = 1 in { 1274// Signed/Unsigned Shift Right (Immediate) 1275def SCALAR_SSHR_N: SInst<"vshr_n", "11I", "SlSUl">; 1276// Signed/Unsigned Rounding Shift Right (Immediate) 1277def SCALAR_SRSHR_N: SInst<"vrshr_n", "11I", "SlSUl">; 1278 1279// Signed/Unsigned Shift Right and Accumulate (Immediate) 1280def SCALAR_SSRA_N: SInst<"vsra_n", "111I", "SlSUl">; 1281// Signed/Unsigned Rounding Shift Right and Accumulate (Immediate) 1282def SCALAR_SRSRA_N: SInst<"vrsra_n", "111I", "SlSUl">; 1283 1284// Shift Left (Immediate) 1285def SCALAR_SHL_N: SInst<"vshl_n", "11I", "SlSUl">; 1286// Signed/Unsigned Saturating Shift Left (Immediate) 1287def SCALAR_SQSHL_N: SInst<"vqshl_n", "11I", "ScSsSiSlSUcSUsSUiSUl">; 1288// Signed Saturating Shift Left Unsigned (Immediate) 1289def SCALAR_SQSHLU_N: SInst<"vqshlu_n", "11I", "ScSsSiSl">; 1290 1291// Shift Right And Insert (Immediate) 1292def SCALAR_SRI_N: SInst<"vsri_n", "111I", "SlSUl">; 1293// Shift Left And Insert (Immediate) 1294def SCALAR_SLI_N: SInst<"vsli_n", "111I", "SlSUl">; 1295 1296let isScalarNarrowShift = 1 in { 1297 // Signed/Unsigned Saturating Shift Right Narrow (Immediate) 1298 def SCALAR_SQSHRN_N: SInst<"vqshrn_n", "(1<)1I", "SsSiSlSUsSUiSUl">; 1299 // Signed/Unsigned Saturating Rounded Shift Right Narrow (Immediate) 1300 def SCALAR_SQRSHRN_N: SInst<"vqrshrn_n", "(1<)1I", "SsSiSlSUsSUiSUl">; 1301 // Signed Saturating Shift Right Unsigned Narrow (Immediate) 1302 def SCALAR_SQSHRUN_N: SInst<"vqshrun_n", "(1<)1I", "SsSiSl">; 1303 // Signed Saturating Rounded Shift Right Unsigned Narrow (Immediate) 1304 def SCALAR_SQRSHRUN_N: SInst<"vqrshrun_n", "(1<)1I", "SsSiSl">; 1305} 1306 1307//////////////////////////////////////////////////////////////////////////////// 1308// Scalar Signed/Unsigned Fixed-point Convert To Floating-Point (Immediate) 1309def SCALAR_SCVTF_N_F32: SInst<"vcvt_n_f32", "(1F)(1!)I", "SiSUi">; 1310def SCALAR_SCVTF_N_F64: SInst<"vcvt_n_f64", "(1F)(1!)I", "SlSUl">; 1311 1312//////////////////////////////////////////////////////////////////////////////// 1313// Scalar Floating-point Convert To Signed/Unsigned Fixed-point (Immediate) 1314def SCALAR_FCVTZS_N_S32 : SInst<"vcvt_n_s32", "(1S)1I", "Sf">; 1315def SCALAR_FCVTZU_N_U32 : SInst<"vcvt_n_u32", "(1U)1I", "Sf">; 1316def SCALAR_FCVTZS_N_S64 : SInst<"vcvt_n_s64", "(1S)1I", "Sd">; 1317def SCALAR_FCVTZU_N_U64 : SInst<"vcvt_n_u64", "(1U)1I", "Sd">; 1318} 1319 1320//////////////////////////////////////////////////////////////////////////////// 1321// Scalar Floating-point Round to Integral 1322let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING)" in { 1323def SCALAR_FRINTN_S32 : SInst<"vrndn", "11", "Sf">; 1324} 1325 1326//////////////////////////////////////////////////////////////////////////////// 1327// Scalar Reduce Pairwise Addition (Scalar and Floating Point) 1328def SCALAR_ADDP : SInst<"vpadd", "1.", "SfSHlSHdSHUl">; 1329 1330//////////////////////////////////////////////////////////////////////////////// 1331// Scalar Reduce Floating Point Pairwise Max/Min 1332def SCALAR_FMAXP : SInst<"vpmax", "1.", "SfSQd">; 1333 1334def SCALAR_FMINP : SInst<"vpmin", "1.", "SfSQd">; 1335 1336//////////////////////////////////////////////////////////////////////////////// 1337// Scalar Reduce Floating Point Pairwise maxNum/minNum 1338def SCALAR_FMAXNMP : SInst<"vpmaxnm", "1.", "SfSQd">; 1339def SCALAR_FMINNMP : SInst<"vpminnm", "1.", "SfSQd">; 1340 1341//////////////////////////////////////////////////////////////////////////////// 1342// Scalar Integer Saturating Doubling Multiply Half High 1343def SCALAR_SQDMULH : SInst<"vqdmulh", "111", "SsSi">; 1344 1345//////////////////////////////////////////////////////////////////////////////// 1346// Scalar Integer Saturating Rounding Doubling Multiply Half High 1347def SCALAR_SQRDMULH : SInst<"vqrdmulh", "111", "SsSi">; 1348 1349let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in { 1350//////////////////////////////////////////////////////////////////////////////// 1351// Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half 1352def SCALAR_SQRDMLAH : SOpInst<"vqrdmlah", "1111", "SsSi", OP_QRDMLAH>; 1353 1354//////////////////////////////////////////////////////////////////////////////// 1355// Signed Saturating Rounding Doubling Multiply Subtract Returning High Half 1356def SCALAR_SQRDMLSH : SOpInst<"vqrdmlsh", "1111", "SsSi", OP_QRDMLSH>; 1357} 1358 1359//////////////////////////////////////////////////////////////////////////////// 1360// Scalar Floating-point Multiply Extended 1361def SCALAR_FMULX : IInst<"vmulx", "111", "SfSd">; 1362 1363//////////////////////////////////////////////////////////////////////////////// 1364// Scalar Floating-point Reciprocal Step 1365def SCALAR_FRECPS : IInst<"vrecps", "111", "SfSd">; 1366 1367//////////////////////////////////////////////////////////////////////////////// 1368// Scalar Floating-point Reciprocal Square Root Step 1369def SCALAR_FRSQRTS : IInst<"vrsqrts", "111", "SfSd">; 1370 1371//////////////////////////////////////////////////////////////////////////////// 1372// Scalar Signed Integer Convert To Floating-point 1373def SCALAR_SCVTFS : SInst<"vcvt_f32", "(1F)(1!)", "Si">; 1374def SCALAR_SCVTFD : SInst<"vcvt_f64", "(1F)(1!)", "Sl">; 1375 1376//////////////////////////////////////////////////////////////////////////////// 1377// Scalar Unsigned Integer Convert To Floating-point 1378def SCALAR_UCVTFS : SInst<"vcvt_f32", "(1F)(1!)", "SUi">; 1379def SCALAR_UCVTFD : SInst<"vcvt_f64", "(1F)(1!)", "SUl">; 1380 1381//////////////////////////////////////////////////////////////////////////////// 1382// Scalar Floating-point Converts 1383def SCALAR_FCVTXN : IInst<"vcvtx_f32", "(1F<)(1!)", "Sd">; 1384def SCALAR_FCVTNSS : SInst<"vcvtn_s32", "(1S)1", "Sf">; 1385def SCALAR_FCVTNUS : SInst<"vcvtn_u32", "(1U)1", "Sf">; 1386def SCALAR_FCVTNSD : SInst<"vcvtn_s64", "(1S)1", "Sd">; 1387def SCALAR_FCVTNUD : SInst<"vcvtn_u64", "(1U)1", "Sd">; 1388def SCALAR_FCVTMSS : SInst<"vcvtm_s32", "(1S)1", "Sf">; 1389def SCALAR_FCVTMUS : SInst<"vcvtm_u32", "(1U)1", "Sf">; 1390def SCALAR_FCVTMSD : SInst<"vcvtm_s64", "(1S)1", "Sd">; 1391def SCALAR_FCVTMUD : SInst<"vcvtm_u64", "(1U)1", "Sd">; 1392def SCALAR_FCVTASS : SInst<"vcvta_s32", "(1S)1", "Sf">; 1393def SCALAR_FCVTAUS : SInst<"vcvta_u32", "(1U)1", "Sf">; 1394def SCALAR_FCVTASD : SInst<"vcvta_s64", "(1S)1", "Sd">; 1395def SCALAR_FCVTAUD : SInst<"vcvta_u64", "(1U)1", "Sd">; 1396def SCALAR_FCVTPSS : SInst<"vcvtp_s32", "(1S)1", "Sf">; 1397def SCALAR_FCVTPUS : SInst<"vcvtp_u32", "(1U)1", "Sf">; 1398def SCALAR_FCVTPSD : SInst<"vcvtp_s64", "(1S)1", "Sd">; 1399def SCALAR_FCVTPUD : SInst<"vcvtp_u64", "(1U)1", "Sd">; 1400def SCALAR_FCVTZSS : SInst<"vcvt_s32", "(1S)1", "Sf">; 1401def SCALAR_FCVTZUS : SInst<"vcvt_u32", "(1U)1", "Sf">; 1402def SCALAR_FCVTZSD : SInst<"vcvt_s64", "(1S)1", "Sd">; 1403def SCALAR_FCVTZUD : SInst<"vcvt_u64", "(1U)1", "Sd">; 1404 1405//////////////////////////////////////////////////////////////////////////////// 1406// Scalar Floating-point Reciprocal Estimate 1407def SCALAR_FRECPE : IInst<"vrecpe", "11", "SfSd">; 1408 1409//////////////////////////////////////////////////////////////////////////////// 1410// Scalar Floating-point Reciprocal Exponent 1411def SCALAR_FRECPX : IInst<"vrecpx", "11", "SfSd">; 1412 1413//////////////////////////////////////////////////////////////////////////////// 1414// Scalar Floating-point Reciprocal Square Root Estimate 1415def SCALAR_FRSQRTE : IInst<"vrsqrte", "11", "SfSd">; 1416 1417//////////////////////////////////////////////////////////////////////////////// 1418// Scalar Integer Comparison 1419def SCALAR_CMEQ : SInst<"vceq", "(U1)11", "SlSUl">; 1420def SCALAR_CMEQZ : SInst<"vceqz", "(U1)1", "SlSUl">; 1421def SCALAR_CMGE : SInst<"vcge", "(U1)11", "Sl">; 1422def SCALAR_CMGEZ : SInst<"vcgez", "(U1)1", "Sl">; 1423def SCALAR_CMHS : SInst<"vcge", "(U1)11", "SUl">; 1424def SCALAR_CMLE : SInst<"vcle", "(U1)11", "SlSUl">; 1425def SCALAR_CMLEZ : SInst<"vclez", "(U1)1", "Sl">; 1426def SCALAR_CMLT : SInst<"vclt", "(U1)11", "SlSUl">; 1427def SCALAR_CMLTZ : SInst<"vcltz", "(U1)1", "Sl">; 1428def SCALAR_CMGT : SInst<"vcgt", "(U1)11", "Sl">; 1429def SCALAR_CMGTZ : SInst<"vcgtz", "(U1)1", "Sl">; 1430def SCALAR_CMHI : SInst<"vcgt", "(U1)11", "SUl">; 1431def SCALAR_CMTST : SInst<"vtst", "(U1)11", "SlSUl">; 1432 1433//////////////////////////////////////////////////////////////////////////////// 1434// Scalar Floating-point Comparison 1435def SCALAR_FCMEQ : IInst<"vceq", "(1U)11", "SfSd">; 1436def SCALAR_FCMEQZ : IInst<"vceqz", "(1U)1", "SfSd">; 1437def SCALAR_FCMGE : IInst<"vcge", "(1U)11", "SfSd">; 1438def SCALAR_FCMGEZ : IInst<"vcgez", "(1U)1", "SfSd">; 1439def SCALAR_FCMGT : IInst<"vcgt", "(1U)11", "SfSd">; 1440def SCALAR_FCMGTZ : IInst<"vcgtz", "(1U)1", "SfSd">; 1441def SCALAR_FCMLE : IInst<"vcle", "(1U)11", "SfSd">; 1442def SCALAR_FCMLEZ : IInst<"vclez", "(1U)1", "SfSd">; 1443def SCALAR_FCMLT : IInst<"vclt", "(1U)11", "SfSd">; 1444def SCALAR_FCMLTZ : IInst<"vcltz", "(1U)1", "SfSd">; 1445 1446//////////////////////////////////////////////////////////////////////////////// 1447// Scalar Floating-point Absolute Compare Mask Greater Than Or Equal 1448def SCALAR_FACGE : IInst<"vcage", "(1U)11", "SfSd">; 1449def SCALAR_FACLE : IInst<"vcale", "(1U)11", "SfSd">; 1450 1451//////////////////////////////////////////////////////////////////////////////// 1452// Scalar Floating-point Absolute Compare Mask Greater Than 1453def SCALAR_FACGT : IInst<"vcagt", "(1U)11", "SfSd">; 1454def SCALAR_FACLT : IInst<"vcalt", "(1U)11", "SfSd">; 1455 1456//////////////////////////////////////////////////////////////////////////////// 1457// Scalar Absolute Value 1458def SCALAR_ABS : SInst<"vabs", "11", "Sl">; 1459 1460//////////////////////////////////////////////////////////////////////////////// 1461// Scalar Absolute Difference 1462def SCALAR_ABD : IInst<"vabd", "111", "SfSd">; 1463 1464//////////////////////////////////////////////////////////////////////////////// 1465// Scalar Signed Saturating Absolute Value 1466def SCALAR_SQABS : SInst<"vqabs", "11", "ScSsSiSl">; 1467 1468//////////////////////////////////////////////////////////////////////////////// 1469// Scalar Negate 1470def SCALAR_NEG : SInst<"vneg", "11", "Sl">; 1471 1472//////////////////////////////////////////////////////////////////////////////// 1473// Scalar Signed Saturating Negate 1474def SCALAR_SQNEG : SInst<"vqneg", "11", "ScSsSiSl">; 1475 1476//////////////////////////////////////////////////////////////////////////////// 1477// Scalar Signed Saturating Accumulated of Unsigned Value 1478def SCALAR_SUQADD : SInst<"vuqadd", "11(1U)", "ScSsSiSl">; 1479 1480//////////////////////////////////////////////////////////////////////////////// 1481// Scalar Unsigned Saturating Accumulated of Signed Value 1482def SCALAR_USQADD : SInst<"vsqadd", "11(1S)", "SUcSUsSUiSUl">; 1483 1484//////////////////////////////////////////////////////////////////////////////// 1485// Signed Saturating Doubling Multiply-Add Long 1486def SCALAR_SQDMLAL : SInst<"vqdmlal", "(1>)(1>)11", "SsSi">; 1487 1488//////////////////////////////////////////////////////////////////////////////// 1489// Signed Saturating Doubling Multiply-Subtract Long 1490def SCALAR_SQDMLSL : SInst<"vqdmlsl", "(1>)(1>)11", "SsSi">; 1491 1492//////////////////////////////////////////////////////////////////////////////// 1493// Signed Saturating Doubling Multiply Long 1494def SCALAR_SQDMULL : SInst<"vqdmull", "(1>)11", "SsSi">; 1495 1496//////////////////////////////////////////////////////////////////////////////// 1497// Scalar Signed Saturating Extract Unsigned Narrow 1498def SCALAR_SQXTUN : SInst<"vqmovun", "(U1<)1", "SsSiSl">; 1499 1500//////////////////////////////////////////////////////////////////////////////// 1501// Scalar Signed Saturating Extract Narrow 1502def SCALAR_SQXTN : SInst<"vqmovn", "(1<)1", "SsSiSl">; 1503 1504//////////////////////////////////////////////////////////////////////////////// 1505// Scalar Unsigned Saturating Extract Narrow 1506def SCALAR_UQXTN : SInst<"vqmovn", "(1<)1", "SUsSUiSUl">; 1507 1508// Scalar Floating Point multiply (scalar, by element) 1509def SCALAR_FMUL_LANE : IOpInst<"vmul_lane", "11.I", "SfSd", OP_SCALAR_MUL_LN>; 1510def SCALAR_FMUL_LANEQ : IOpInst<"vmul_laneq", "11QI", "SfSd", OP_SCALAR_MUL_LN> { 1511 let isLaneQ = 1; 1512} 1513 1514// Scalar Floating Point multiply extended (scalar, by element) 1515def SCALAR_FMULX_LANE : IOpInst<"vmulx_lane", "11.I", "SfSd", OP_SCALAR_MULX_LN>; 1516def SCALAR_FMULX_LANEQ : IOpInst<"vmulx_laneq", "11QI", "SfSd", OP_SCALAR_MULX_LN> { 1517 let isLaneQ = 1; 1518} 1519 1520def SCALAR_VMUL_N : IInst<"vmul_n", "..1", "d">; 1521 1522// VMUL_LANE_A64 d type implemented using scalar mul lane 1523def SCALAR_VMUL_LANE : IInst<"vmul_lane", "..qI", "d">; 1524 1525// VMUL_LANEQ d type implemented using scalar mul lane 1526def SCALAR_VMUL_LANEQ : IInst<"vmul_laneq", "..QI", "d"> { 1527 let isLaneQ = 1; 1528} 1529 1530// VMULX_LANE d type implemented using scalar vmulx_lane 1531def SCALAR_VMULX_LANE : IOpInst<"vmulx_lane", "..qI", "d", OP_SCALAR_VMULX_LN>; 1532 1533// VMULX_LANEQ d type implemented using scalar vmulx_laneq 1534def SCALAR_VMULX_LANEQ : IOpInst<"vmulx_laneq", "..QI", "d", OP_SCALAR_VMULX_LNQ> { 1535 let isLaneQ = 1; 1536} 1537 1538// Scalar Floating Point fused multiply-add (scalar, by element) 1539def SCALAR_FMLA_LANE : IInst<"vfma_lane", "111.I", "SfSd">; 1540def SCALAR_FMLA_LANEQ : IInst<"vfma_laneq", "111QI", "SfSd"> { 1541 let isLaneQ = 1; 1542} 1543 1544// Scalar Floating Point fused multiply-subtract (scalar, by element) 1545def SCALAR_FMLS_LANE : IOpInst<"vfms_lane", "111.I", "SfSd", OP_FMS_LN>; 1546def SCALAR_FMLS_LANEQ : IOpInst<"vfms_laneq", "111QI", "SfSd", OP_FMS_LNQ> { 1547 let isLaneQ = 1; 1548} 1549 1550// Signed Saturating Doubling Multiply Long (scalar by element) 1551def SCALAR_SQDMULL_LANE : SOpInst<"vqdmull_lane", "(1>)1.I", "SsSi", OP_SCALAR_QDMULL_LN>; 1552def SCALAR_SQDMULL_LANEQ : SOpInst<"vqdmull_laneq", "(1>)1QI", "SsSi", OP_SCALAR_QDMULL_LN> { 1553 let isLaneQ = 1; 1554} 1555 1556// Signed Saturating Doubling Multiply-Add Long (scalar by element) 1557def SCALAR_SQDMLAL_LANE : SInst<"vqdmlal_lane", "(1>)(1>)1.I", "SsSi">; 1558def SCALAR_SQDMLAL_LANEQ : SInst<"vqdmlal_laneq", "(1>)(1>)1QI", "SsSi"> { 1559 let isLaneQ = 1; 1560} 1561 1562// Signed Saturating Doubling Multiply-Subtract Long (scalar by element) 1563def SCALAR_SQDMLS_LANE : SInst<"vqdmlsl_lane", "(1>)(1>)1.I", "SsSi">; 1564def SCALAR_SQDMLS_LANEQ : SInst<"vqdmlsl_laneq", "(1>)(1>)1QI", "SsSi"> { 1565 let isLaneQ = 1; 1566} 1567 1568// Scalar Integer Saturating Doubling Multiply Half High (scalar by element) 1569def SCALAR_SQDMULH_LANE : SOpInst<"vqdmulh_lane", "11.I", "SsSi", OP_SCALAR_QDMULH_LN>; 1570def SCALAR_SQDMULH_LANEQ : SOpInst<"vqdmulh_laneq", "11QI", "SsSi", OP_SCALAR_QDMULH_LN> { 1571 let isLaneQ = 1; 1572} 1573 1574// Scalar Integer Saturating Rounding Doubling Multiply Half High 1575def SCALAR_SQRDMULH_LANE : SOpInst<"vqrdmulh_lane", "11.I", "SsSi", OP_SCALAR_QRDMULH_LN>; 1576def SCALAR_SQRDMULH_LANEQ : SOpInst<"vqrdmulh_laneq", "11QI", "SsSi", OP_SCALAR_QRDMULH_LN> { 1577 let isLaneQ = 1; 1578} 1579 1580let ArchGuard = "defined(__ARM_FEATURE_QRDMX) && defined(__aarch64__)" in { 1581// Signed Saturating Rounding Doubling Multiply Accumulate Returning High Half 1582def SCALAR_SQRDMLAH_LANE : SOpInst<"vqrdmlah_lane", "111.I", "SsSi", OP_SCALAR_QRDMLAH_LN>; 1583def SCALAR_SQRDMLAH_LANEQ : SOpInst<"vqrdmlah_laneq", "111QI", "SsSi", OP_SCALAR_QRDMLAH_LN> { 1584 let isLaneQ = 1; 1585} 1586 1587// Signed Saturating Rounding Doubling Multiply Subtract Returning High Half 1588def SCALAR_SQRDMLSH_LANE : SOpInst<"vqrdmlsh_lane", "111.I", "SsSi", OP_SCALAR_QRDMLSH_LN>; 1589def SCALAR_SQRDMLSH_LANEQ : SOpInst<"vqrdmlsh_laneq", "111QI", "SsSi", OP_SCALAR_QRDMLSH_LN> { 1590 let isLaneQ = 1; 1591} 1592} 1593 1594def SCALAR_VDUP_LANE : IInst<"vdup_lane", "1.I", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs">; 1595def SCALAR_VDUP_LANEQ : IInst<"vdup_laneq", "1QI", "ScSsSiSlSfSdSUcSUsSUiSUlSPcSPs"> { 1596 let isLaneQ = 1; 1597} 1598} 1599 1600// ARMv8.2-A FP16 vector intrinsics for A32/A64. 1601let ArchGuard = "defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in { 1602 1603 // ARMv8.2-A FP16 one-operand vector intrinsics. 1604 1605 // Comparison 1606 def CMEQH : SInst<"vceqz", "U.", "hQh">; 1607 def CMGEH : SInst<"vcgez", "U.", "hQh">; 1608 def CMGTH : SInst<"vcgtz", "U.", "hQh">; 1609 def CMLEH : SInst<"vclez", "U.", "hQh">; 1610 def CMLTH : SInst<"vcltz", "U.", "hQh">; 1611 1612 // Vector conversion 1613 def VCVT_F16 : SInst<"vcvt_f16", "F(.!)", "sUsQsQUs">; 1614 def VCVT_S16 : SInst<"vcvt_s16", "S.", "hQh">; 1615 def VCVT_U16 : SInst<"vcvt_u16", "U.", "hQh">; 1616 def VCVTA_S16 : SInst<"vcvta_s16", "S.", "hQh">; 1617 def VCVTA_U16 : SInst<"vcvta_u16", "U.", "hQh">; 1618 def VCVTM_S16 : SInst<"vcvtm_s16", "S.", "hQh">; 1619 def VCVTM_U16 : SInst<"vcvtm_u16", "U.", "hQh">; 1620 def VCVTN_S16 : SInst<"vcvtn_s16", "S.", "hQh">; 1621 def VCVTN_U16 : SInst<"vcvtn_u16", "U.", "hQh">; 1622 def VCVTP_S16 : SInst<"vcvtp_s16", "S.", "hQh">; 1623 def VCVTP_U16 : SInst<"vcvtp_u16", "U.", "hQh">; 1624 1625 // Vector rounding 1626 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_DIRECTED_ROUNDING) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in { 1627 def FRINTZH : SInst<"vrnd", "..", "hQh">; 1628 def FRINTNH : SInst<"vrndn", "..", "hQh">; 1629 def FRINTAH : SInst<"vrnda", "..", "hQh">; 1630 def FRINTPH : SInst<"vrndp", "..", "hQh">; 1631 def FRINTMH : SInst<"vrndm", "..", "hQh">; 1632 def FRINTXH : SInst<"vrndx", "..", "hQh">; 1633 } 1634 1635 // Misc. 1636 def VABSH : SInst<"vabs", "..", "hQh">; 1637 def VNEGH : SOpInst<"vneg", "..", "hQh", OP_NEG>; 1638 def VRECPEH : SInst<"vrecpe", "..", "hQh">; 1639 def FRSQRTEH : SInst<"vrsqrte", "..", "hQh">; 1640 1641 // ARMv8.2-A FP16 two-operands vector intrinsics. 1642 1643 // Misc. 1644 def VADDH : SOpInst<"vadd", "...", "hQh", OP_ADD>; 1645 def VABDH : SInst<"vabd", "...", "hQh">; 1646 def VSUBH : SOpInst<"vsub", "...", "hQh", OP_SUB>; 1647 1648 // Comparison 1649 let InstName = "vacge" in { 1650 def VCAGEH : SInst<"vcage", "U..", "hQh">; 1651 def VCALEH : SInst<"vcale", "U..", "hQh">; 1652 } 1653 let InstName = "vacgt" in { 1654 def VCAGTH : SInst<"vcagt", "U..", "hQh">; 1655 def VCALTH : SInst<"vcalt", "U..", "hQh">; 1656 } 1657 def VCEQH : SOpInst<"vceq", "U..", "hQh", OP_EQ>; 1658 def VCGEH : SOpInst<"vcge", "U..", "hQh", OP_GE>; 1659 def VCGTH : SOpInst<"vcgt", "U..", "hQh", OP_GT>; 1660 let InstName = "vcge" in 1661 def VCLEH : SOpInst<"vcle", "U..", "hQh", OP_LE>; 1662 let InstName = "vcgt" in 1663 def VCLTH : SOpInst<"vclt", "U..", "hQh", OP_LT>; 1664 1665 // Vector conversion 1666 let isVCVT_N = 1 in { 1667 def VCVT_N_F16 : SInst<"vcvt_n_f16", "F(.!)I", "sUsQsQUs">; 1668 def VCVT_N_S16 : SInst<"vcvt_n_s16", "S.I", "hQh">; 1669 def VCVT_N_U16 : SInst<"vcvt_n_u16", "U.I", "hQh">; 1670 } 1671 1672 // Max/Min 1673 def VMAXH : SInst<"vmax", "...", "hQh">; 1674 def VMINH : SInst<"vmin", "...", "hQh">; 1675 let ArchGuard = "__ARM_ARCH >= 8 && defined(__ARM_FEATURE_NUMERIC_MAXMIN) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in { 1676 def FMAXNMH : SInst<"vmaxnm", "...", "hQh">; 1677 def FMINNMH : SInst<"vminnm", "...", "hQh">; 1678 } 1679 1680 // Multiplication/Division 1681 def VMULH : SOpInst<"vmul", "...", "hQh", OP_MUL>; 1682 1683 // Pairwise addition 1684 def VPADDH : SInst<"vpadd", "...", "h">; 1685 1686 // Pairwise Max/Min 1687 def VPMAXH : SInst<"vpmax", "...", "h">; 1688 def VPMINH : SInst<"vpmin", "...", "h">; 1689 1690 // Reciprocal/Sqrt 1691 def VRECPSH : SInst<"vrecps", "...", "hQh">; 1692 def VRSQRTSH : SInst<"vrsqrts", "...", "hQh">; 1693 1694 // ARMv8.2-A FP16 three-operands vector intrinsics. 1695 1696 // Vector fused multiply-add operations 1697 def VFMAH : SInst<"vfma", "....", "hQh">; 1698 def VFMSH : SOpInst<"vfms", "....", "hQh", OP_FMLS>; 1699 1700 // ARMv8.2-A FP16 lane vector intrinsics. 1701 1702 // Mul lane 1703 def VMUL_LANEH : IOpInst<"vmul_lane", "..qI", "hQh", OP_MUL_LN>; 1704 def VMUL_NH : IOpInst<"vmul_n", "..1", "hQh", OP_MUL_N>; 1705 1706 // Data processing intrinsics - section 5 1707 1708 // Logical operations 1709 let isHiddenLInst = 1 in 1710 def VBSLH : SInst<"vbsl", ".U..", "hQh">; 1711 1712 // Transposition operations 1713 def VZIPH : WInst<"vzip", "2..", "hQh">; 1714 def VUZPH : WInst<"vuzp", "2..", "hQh">; 1715 def VTRNH : WInst<"vtrn", "2..", "hQh">; 1716 1717 1718 let ArchGuard = "!defined(__aarch64__)" in { 1719 // Set all lanes to same value. 1720 // Already implemented prior to ARMv8.2-A. 1721 def VMOV_NH : WOpInst<"vmov_n", ".1", "hQh", OP_DUP>; 1722 def VDUP_NH : WOpInst<"vdup_n", ".1", "hQh", OP_DUP>; 1723 def VDUP_LANE1H : WOpInst<"vdup_lane", ".qI", "hQh", OP_DUP_LN>; 1724 } 1725 1726 // Vector Extract 1727 def VEXTH : WInst<"vext", "...I", "hQh">; 1728 1729 // Reverse vector elements 1730 def VREV64H : WOpInst<"vrev64", "..", "hQh", OP_REV64>; 1731} 1732 1733// ARMv8.2-A FP16 vector intrinsics for A64 only. 1734let ArchGuard = "defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(__aarch64__)" in { 1735 1736 // Vector rounding 1737 def FRINTIH : SInst<"vrndi", "..", "hQh">; 1738 1739 // Misc. 1740 def FSQRTH : SInst<"vsqrt", "..", "hQh">; 1741 1742 // Multiplication/Division 1743 def MULXH : SInst<"vmulx", "...", "hQh">; 1744 def FDIVH : IOpInst<"vdiv", "...", "hQh", OP_DIV>; 1745 1746 // Pairwise addition 1747 def VPADDH1 : SInst<"vpadd", "...", "Qh">; 1748 1749 // Pairwise Max/Min 1750 def VPMAXH1 : SInst<"vpmax", "...", "Qh">; 1751 def VPMINH1 : SInst<"vpmin", "...", "Qh">; 1752 1753 // Pairwise MaxNum/MinNum 1754 def FMAXNMPH : SInst<"vpmaxnm", "...", "hQh">; 1755 def FMINNMPH : SInst<"vpminnm", "...", "hQh">; 1756 1757 // ARMv8.2-A FP16 lane vector intrinsics. 1758 1759 // FMA lane 1760 def VFMA_LANEH : IInst<"vfma_lane", "...qI", "hQh">; 1761 def VFMA_LANEQH : IInst<"vfma_laneq", "...QI", "hQh"> { 1762 let isLaneQ = 1; 1763 } 1764 1765 // FMA lane with scalar argument 1766 def FMLA_NH : SOpInst<"vfma_n", "...1", "hQh", OP_FMLA_N>; 1767 // Scalar floating point fused multiply-add (scalar, by element) 1768 def SCALAR_FMLA_LANEH : IInst<"vfma_lane", "111.I", "Sh">; 1769 def SCALAR_FMLA_LANEQH : IInst<"vfma_laneq", "111QI", "Sh"> { 1770 let isLaneQ = 1; 1771 } 1772 1773 // FMS lane 1774 def VFMS_LANEH : IOpInst<"vfms_lane", "...qI", "hQh", OP_FMS_LN>; 1775 def VFMS_LANEQH : IOpInst<"vfms_laneq", "...QI", "hQh", OP_FMS_LNQ> { 1776 let isLaneQ = 1; 1777 } 1778 // FMS lane with scalar argument 1779 def FMLS_NH : SOpInst<"vfms_n", "...1", "hQh", OP_FMLS_N>; 1780 // Scalar floating foint fused multiply-subtract (scalar, by element) 1781 def SCALAR_FMLS_LANEH : IOpInst<"vfms_lane", "111.I", "Sh", OP_FMS_LN>; 1782 def SCALAR_FMLS_LANEQH : IOpInst<"vfms_laneq", "111QI", "Sh", OP_FMS_LNQ> { 1783 let isLaneQ = 1; 1784 } 1785 1786 // Mul lane 1787 def VMUL_LANEQH : IOpInst<"vmul_laneq", "..QI", "hQh", OP_MUL_LN> { 1788 let isLaneQ = 1; 1789 } 1790 // Scalar floating point multiply (scalar, by element) 1791 def SCALAR_FMUL_LANEH : IOpInst<"vmul_lane", "11.I", "Sh", OP_SCALAR_MUL_LN>; 1792 def SCALAR_FMUL_LANEQH : IOpInst<"vmul_laneq", "11QI", "Sh", OP_SCALAR_MUL_LN> { 1793 let isLaneQ = 1; 1794 } 1795 1796 // Mulx lane 1797 def VMULX_LANEH : IOpInst<"vmulx_lane", "..qI", "hQh", OP_MULX_LN>; 1798 def VMULX_LANEQH : IOpInst<"vmulx_laneq", "..QI", "hQh", OP_MULX_LN> { 1799 let isLaneQ = 1; 1800 } 1801 def VMULX_NH : IOpInst<"vmulx_n", "..1", "hQh", OP_MULX_N>; 1802 // Scalar floating point mulx (scalar, by element) 1803 def SCALAR_FMULX_LANEH : IInst<"vmulx_lane", "11.I", "Sh">; 1804 def SCALAR_FMULX_LANEQH : IInst<"vmulx_laneq", "11QI", "Sh"> { 1805 let isLaneQ = 1; 1806 } 1807 1808 // ARMv8.2-A FP16 reduction vector intrinsics. 1809 def VMAXVH : SInst<"vmaxv", "1.", "hQh">; 1810 def VMINVH : SInst<"vminv", "1.", "hQh">; 1811 def FMAXNMVH : SInst<"vmaxnmv", "1.", "hQh">; 1812 def FMINNMVH : SInst<"vminnmv", "1.", "hQh">; 1813 1814 // Permutation 1815 def VTRN1H : SOpInst<"vtrn1", "...", "hQh", OP_TRN1>; 1816 def VZIP1H : SOpInst<"vzip1", "...", "hQh", OP_ZIP1>; 1817 def VUZP1H : SOpInst<"vuzp1", "...", "hQh", OP_UZP1>; 1818 def VTRN2H : SOpInst<"vtrn2", "...", "hQh", OP_TRN2>; 1819 def VZIP2H : SOpInst<"vzip2", "...", "hQh", OP_ZIP2>; 1820 def VUZP2H : SOpInst<"vuzp2", "...", "hQh", OP_UZP2>; 1821 1822 def SCALAR_VDUP_LANEH : IInst<"vdup_lane", "1.I", "Sh">; 1823 def SCALAR_VDUP_LANEQH : IInst<"vdup_laneq", "1QI", "Sh"> { 1824 let isLaneQ = 1; 1825 } 1826} 1827 1828// v8.2-A dot product instructions. 1829let ArchGuard = "defined(__ARM_FEATURE_DOTPROD)" in { 1830 def DOT : SInst<"vdot", "..(<<)(<<)", "iQiUiQUi">; 1831 def DOT_LANE : SOpInst<"vdot_lane", "..(<<)(<<q)I", "iUiQiQUi", OP_DOT_LN>; 1832} 1833let ArchGuard = "defined(__ARM_FEATURE_DOTPROD) && defined(__aarch64__)" in { 1834 // Variants indexing into a 128-bit vector are A64 only. 1835 def UDOT_LANEQ : SOpInst<"vdot_laneq", "..(<<)(<<Q)I", "iUiQiQUi", OP_DOT_LNQ> { 1836 let isLaneQ = 1; 1837 } 1838} 1839 1840// v8.2-A FP16 fused multiply-add long instructions. 1841let ArchGuard = "defined(__ARM_FEATURE_FP16FML) && defined(__aarch64__)" in { 1842 def VFMLAL_LOW : SInst<"vfmlal_low", ">>..", "hQh">; 1843 def VFMLSL_LOW : SInst<"vfmlsl_low", ">>..", "hQh">; 1844 def VFMLAL_HIGH : SInst<"vfmlal_high", ">>..", "hQh">; 1845 def VFMLSL_HIGH : SInst<"vfmlsl_high", ">>..", "hQh">; 1846 1847 def VFMLAL_LANE_LOW : SOpInst<"vfmlal_lane_low", "(F>)(F>)F(Fq)I", "hQh", OP_FMLAL_LN>; 1848 def VFMLSL_LANE_LOW : SOpInst<"vfmlsl_lane_low", "(F>)(F>)F(Fq)I", "hQh", OP_FMLSL_LN>; 1849 def VFMLAL_LANE_HIGH : SOpInst<"vfmlal_lane_high", "(F>)(F>)F(Fq)I", "hQh", OP_FMLAL_LN_Hi>; 1850 def VFMLSL_LANE_HIGH : SOpInst<"vfmlsl_lane_high", "(F>)(F>)F(Fq)I", "hQh", OP_FMLSL_LN_Hi>; 1851 1852 def VFMLAL_LANEQ_LOW : SOpInst<"vfmlal_laneq_low", "(F>)(F>)F(FQ)I", "hQh", OP_FMLAL_LN> { 1853 let isLaneQ = 1; 1854 } 1855 def VFMLSL_LANEQ_LOW : SOpInst<"vfmlsl_laneq_low", "(F>)(F>)F(FQ)I", "hQh", OP_FMLSL_LN> { 1856 let isLaneQ = 1; 1857 } 1858 def VFMLAL_LANEQ_HIGH : SOpInst<"vfmlal_laneq_high", "(F>)(F>)F(FQ)I", "hQh", OP_FMLAL_LN_Hi> { 1859 let isLaneQ = 1; 1860 } 1861 def VFMLSL_LANEQ_HIGH : SOpInst<"vfmlsl_laneq_high", "(F>)(F>)F(FQ)I", "hQh", OP_FMLSL_LN_Hi> { 1862 let isLaneQ = 1; 1863 } 1864} 1865 1866let ArchGuard = "defined(__ARM_FEATURE_MATMUL_INT8)" in { 1867 def VMMLA : SInst<"vmmla", "..(<<)(<<)", "QUiQi">; 1868 def VUSMMLA : SInst<"vusmmla", "..(<<U)(<<)", "Qi">; 1869 1870 def VUSDOT : SInst<"vusdot", "..(<<U)(<<)", "iQi">; 1871 1872 def VUSDOT_LANE : SOpInst<"vusdot_lane", "..(<<U)(<<q)I", "iQi", OP_USDOT_LN>; 1873 def VSUDOT_LANE : SOpInst<"vsudot_lane", "..(<<)(<<qU)I", "iQi", OP_SUDOT_LN>; 1874 1875 let ArchGuard = "defined(__aarch64__)" in { 1876 let isLaneQ = 1 in { 1877 def VUSDOT_LANEQ : SOpInst<"vusdot_laneq", "..(<<U)(<<Q)I", "iQi", OP_USDOT_LNQ>; 1878 def VSUDOT_LANEQ : SOpInst<"vsudot_laneq", "..(<<)(<<QU)I", "iQi", OP_SUDOT_LNQ>; 1879 } 1880 } 1881} 1882 1883let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in { 1884 def VDOT_BF : SInst<"vbfdot", "..BB", "fQf">; 1885 def VDOT_LANE_BF : SOpInst<"vbfdot_lane", "..B(Bq)I", "fQf", OP_BFDOT_LN>; 1886 def VDOT_LANEQ_BF : SOpInst<"vbfdot_laneq", "..B(BQ)I", "fQf", OP_BFDOT_LNQ> { 1887 let isLaneQ = 1; 1888 } 1889 1890 def VFMMLA_BF : SInst<"vbfmmla", "..BB", "Qf">; 1891 1892 def VFMLALB_BF : SInst<"vbfmlalb", "..BB", "Qf">; 1893 def VFMLALT_BF : SInst<"vbfmlalt", "..BB", "Qf">; 1894 1895 def VFMLALB_LANE_BF : SOpInst<"vbfmlalb_lane", "..B(Bq)I", "Qf", OP_BFMLALB_LN>; 1896 def VFMLALB_LANEQ_BF : SOpInst<"vbfmlalb_laneq", "..B(BQ)I", "Qf", OP_BFMLALB_LN>; 1897 1898 def VFMLALT_LANE_BF : SOpInst<"vbfmlalt_lane", "..B(Bq)I", "Qf", OP_BFMLALT_LN>; 1899 def VFMLALT_LANEQ_BF : SOpInst<"vbfmlalt_laneq", "..B(BQ)I", "Qf", OP_BFMLALT_LN>; 1900} 1901 1902multiclass VCMLA_ROTS<string type, string lanety, string laneqty> { 1903 foreach ROT = ["", "_rot90", "_rot180", "_rot270" ] in { 1904 def : SInst<"vcmla" # ROT, "....", type # "Q" # type>; 1905 1906 // vcmla{ROT}_lane 1907 def : SOpInst<"vcmla" # ROT # "_lane", "...qI", type, Op<(call "vcmla" # ROT, $p0, $p1, 1908 (bitcast $p0, (dup_typed lanety , (call "vget_lane", (bitcast lanety, $p2), $p3))))>>; 1909 1910 // vcmlaq{ROT}_lane 1911 def : SOpInst<"vcmla" # ROT # "_lane", "...qI", "Q" # type, Op<(call "vcmla" # ROT, $p0, $p1, 1912 (bitcast $p0, (dup_typed laneqty , (call "vget_lane", (bitcast lanety, $p2), $p3))))>>; 1913 1914 let isLaneQ = 1 in { 1915 // vcmla{ROT}_laneq 1916 def : SOpInst<"vcmla" # ROT # "_laneq", "...QI", type, Op<(call "vcmla" # ROT, $p0, $p1, 1917 (bitcast $p0, (dup_typed lanety, (call "vget_lane", (bitcast laneqty, $p2), $p3))))>>; 1918 1919 // vcmlaq{ROT}_laneq 1920 def : SOpInst<"vcmla" # ROT # "_laneq", "...QI", "Q" # type, Op<(call "vcmla" # ROT, $p0, $p1, 1921 (bitcast $p0, (dup_typed laneqty , (call "vget_lane", (bitcast laneqty, $p2), $p3))))>>; 1922 } 1923 } 1924} 1925 1926// v8.3-A Vector complex addition intrinsics 1927let ArchGuard = "defined(__ARM_FEATURE_COMPLEX) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in { 1928 def VCADD_ROT90_FP16 : SInst<"vcadd_rot90", "...", "h">; 1929 def VCADD_ROT270_FP16 : SInst<"vcadd_rot270", "...", "h">; 1930 def VCADDQ_ROT90_FP16 : SInst<"vcaddq_rot90", "QQQ", "h">; 1931 def VCADDQ_ROT270_FP16 : SInst<"vcaddq_rot270", "QQQ", "h">; 1932 1933 defm VCMLA_FP16 : VCMLA_ROTS<"h", "uint32x2_t", "uint32x4_t">; 1934} 1935let ArchGuard = "defined(__ARM_FEATURE_COMPLEX)" in { 1936 def VCADD_ROT90 : SInst<"vcadd_rot90", "...", "f">; 1937 def VCADD_ROT270 : SInst<"vcadd_rot270", "...", "f">; 1938 def VCADDQ_ROT90 : SInst<"vcaddq_rot90", "QQQ", "f">; 1939 def VCADDQ_ROT270 : SInst<"vcaddq_rot270", "QQQ", "f">; 1940 1941 defm VCMLA_F32 : VCMLA_ROTS<"f", "uint64x1_t", "uint64x2_t">; 1942} 1943let ArchGuard = "defined(__ARM_FEATURE_COMPLEX) && defined(__aarch64__)" in { 1944 def VCADDQ_ROT90_FP64 : SInst<"vcaddq_rot90", "QQQ", "d">; 1945 def VCADDQ_ROT270_FP64 : SInst<"vcaddq_rot270", "QQQ", "d">; 1946 1947 defm VCMLA_FP64 : VCMLA_ROTS<"d", "uint64x2_t", "uint64x2_t">; 1948} 1949 1950// V8.2-A BFloat intrinsics 1951let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in { 1952 def VCREATE_BF : NoTestOpInst<"vcreate", ".(IU>)", "b", OP_CAST> { 1953 let BigEndianSafe = 1; 1954 } 1955 1956 def VDUP_N_BF : WOpInst<"vdup_n", ".1", "bQb", OP_DUP>; 1957 1958 def VDUP_LANE_BF : WOpInst<"vdup_lane", ".qI", "bQb", OP_DUP_LN>; 1959 def VDUP_LANEQ_BF: WOpInst<"vdup_laneq", ".QI", "bQb", OP_DUP_LN> { 1960 let isLaneQ = 1; 1961 } 1962 1963 def VCOMBINE_BF : NoTestOpInst<"vcombine", "Q..", "b", OP_CONC>; 1964 1965 def VGET_HIGH_BF : NoTestOpInst<"vget_high", ".Q", "b", OP_HI>; 1966 def VGET_LOW_BF : NoTestOpInst<"vget_low", ".Q", "b", OP_LO>; 1967 1968 def VGET_LANE_BF : IInst<"vget_lane", "1.I", "bQb">; 1969 def VSET_LANE_BF : IInst<"vset_lane", ".1.I", "bQb">; 1970 def SCALAR_VDUP_LANE_BF : IInst<"vdup_lane", "1.I", "Sb">; 1971 def SCALAR_VDUP_LANEQ_BF : IInst<"vdup_laneq", "1QI", "Sb"> { 1972 let isLaneQ = 1; 1973 } 1974 1975 def VLD1_BF : WInst<"vld1", ".(c*!)", "bQb">; 1976 def VLD2_BF : WInst<"vld2", "2(c*!)", "bQb">; 1977 def VLD3_BF : WInst<"vld3", "3(c*!)", "bQb">; 1978 def VLD4_BF : WInst<"vld4", "4(c*!)", "bQb">; 1979 1980 def VST1_BF : WInst<"vst1", "v*(.!)", "bQb">; 1981 def VST2_BF : WInst<"vst2", "v*(2!)", "bQb">; 1982 def VST3_BF : WInst<"vst3", "v*(3!)", "bQb">; 1983 def VST4_BF : WInst<"vst4", "v*(4!)", "bQb">; 1984 1985 def VLD1_X2_BF : WInst<"vld1_x2", "2(c*!)", "bQb">; 1986 def VLD1_X3_BF : WInst<"vld1_x3", "3(c*!)", "bQb">; 1987 def VLD1_X4_BF : WInst<"vld1_x4", "4(c*!)", "bQb">; 1988 1989 def VST1_X2_BF : WInst<"vst1_x2", "v*(2!)", "bQb">; 1990 def VST1_X3_BF : WInst<"vst1_x3", "v*(3!)", "bQb">; 1991 def VST1_X4_BF : WInst<"vst1_x4", "v*(4!)", "bQb">; 1992 1993 def VLD1_LANE_BF : WInst<"vld1_lane", ".(c*!).I", "bQb">; 1994 def VLD2_LANE_BF : WInst<"vld2_lane", "2(c*!)2I", "bQb">; 1995 def VLD3_LANE_BF : WInst<"vld3_lane", "3(c*!)3I", "bQb">; 1996 def VLD4_LANE_BF : WInst<"vld4_lane", "4(c*!)4I", "bQb">; 1997 def VST1_LANE_BF : WInst<"vst1_lane", "v*(.!)I", "bQb">; 1998 def VST2_LANE_BF : WInst<"vst2_lane", "v*(2!)I", "bQb">; 1999 def VST3_LANE_BF : WInst<"vst3_lane", "v*(3!)I", "bQb">; 2000 def VST4_LANE_BF : WInst<"vst4_lane", "v*(4!)I", "bQb">; 2001 2002 def VLD1_DUP_BF : WInst<"vld1_dup", ".(c*!)", "bQb">; 2003 def VLD2_DUP_BF : WInst<"vld2_dup", "2(c*!)", "bQb">; 2004 def VLD3_DUP_BF : WInst<"vld3_dup", "3(c*!)", "bQb">; 2005 def VLD4_DUP_BF : WInst<"vld4_dup", "4(c*!)", "bQb">; 2006 2007 def VCVT_F32_BF16 : SOpInst<"vcvt_f32_bf16", "(F>)(Bq!)", "Qb", OP_VCVT_F32_BF16>; 2008 def VCVT_LOW_F32_BF16 : SOpInst<"vcvt_low_f32", "(F>)(BQ!)", "Qb", OP_VCVT_F32_BF16_LO>; 2009 def VCVT_HIGH_F32_BF16 : SOpInst<"vcvt_high_f32", "(F>)(BQ!)", "Qb", OP_VCVT_F32_BF16_HI>; 2010 2011 def SCALAR_CVT_BF16_F32 : SInst<"vcvth_bf16", "(1B)1", "f">; 2012 def SCALAR_CVT_F32_BF16 : SOpInst<"vcvtah_f32", "(1F>)(1!)", "b", OP_CVT_F32_BF16>; 2013} 2014 2015let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && !defined(__aarch64__)" in { 2016 def VCVT_BF16_F32_A32_INTERNAL : WInst<"__a32_vcvt_bf16", "BQ", "f">; 2017 def VCVT_BF16_F32_A32 : SOpInst<"vcvt_bf16", "BQ", "f", OP_VCVT_BF16_F32_A32>; 2018 def VCVT_LOW_BF16_F32_A32 : SOpInst<"vcvt_low_bf16", "BQ", "Qf", OP_VCVT_BF16_F32_LO_A32>; 2019 def VCVT_HIGH_BF16_F32_A32 : SOpInst<"vcvt_high_bf16", "BBQ", "Qf", OP_VCVT_BF16_F32_HI_A32>; 2020} 2021 2022let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) && defined(__aarch64__)" in { 2023 def VCVT_LOW_BF16_F32_A64_INTERNAL : WInst<"__a64_vcvtq_low_bf16", "BQ", "Hf">; 2024 def VCVT_LOW_BF16_F32_A64 : SOpInst<"vcvt_low_bf16", "BQ", "Qf", OP_VCVT_BF16_F32_LO_A64>; 2025 def VCVT_HIGH_BF16_F32_A64 : SInst<"vcvt_high_bf16", "BBQ", "Qf">; 2026 def VCVT_BF16_F32 : SOpInst<"vcvt_bf16", "BQ", "f", OP_VCVT_BF16_F32_A64>; 2027 2028 def COPY_LANE_BF16 : IOpInst<"vcopy_lane", "..I.I", "b", OP_COPY_LN>; 2029 def COPYQ_LANE_BF16 : IOpInst<"vcopy_lane", "..IqI", "Qb", OP_COPY_LN>; 2030 def COPY_LANEQ_BF16 : IOpInst<"vcopy_laneq", "..IQI", "b", OP_COPY_LN>; 2031 def COPYQ_LANEQ_BF16 : IOpInst<"vcopy_laneq", "..I.I", "Qb", OP_COPY_LN>; 2032} 2033 2034let ArchGuard = "defined(__ARM_FEATURE_BF16) && !defined(__aarch64__)" in { 2035 let BigEndianSafe = 1 in { 2036 defm VREINTERPRET_BF : REINTERPRET_CROSS_TYPES< 2037 "csilUcUsUiUlhfPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQPcQPsQPl", "bQb">; 2038 } 2039} 2040 2041let ArchGuard = "defined(__ARM_FEATURE_BF16) && defined(__aarch64__)" in { 2042 let BigEndianSafe = 1 in { 2043 defm VVREINTERPRET_BF : REINTERPRET_CROSS_TYPES< 2044 "csilUcUsUiUlhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQdQPcQPsQPlQPk", "bQb">; 2045 } 2046} 2047