diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index fc6a6a88b4fc08..a53973bad92e25 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -2423,14 +2423,6 @@ let Predicates = [HasSVEorSME] in { } let AddedComplexity = 5 in { - def : Pat<(add GPR64:$op, (vscale (sve_rdvl_imm i32:$imm))), - (ADDVL_XXI GPR64:$op, $imm)>; - - def : Pat<(add GPR32:$op, (i32 (trunc (vscale (sve_rdvl_imm i32:$imm))))), - (i32 (EXTRACT_SUBREG (ADDVL_XXI (INSERT_SUBREG (i64 (IMPLICIT_DEF)), - GPR32:$op, sub_32), $imm), - sub_32))>; - def : Pat<(nxv8i16 (add ZPR:$op, (nxv8i16 (splat_vector (i32 (trunc (vscale (sve_cnth_imm i32:$imm)))))))), (INCH_ZPiI ZPR:$op, 31, $imm)>; def : Pat<(nxv4i32 (add ZPR:$op, (nxv4i32 (splat_vector (i32 (trunc (vscale (sve_cntw_imm i32:$imm)))))))), @@ -2447,6 +2439,14 @@ let Predicates = [HasSVEorSME] in { } let Predicates = [HasSVEorSME, UseScalarIncVL], AddedComplexity = 5 in { + def : Pat<(add GPR64:$op, (vscale (sve_rdvl_imm i32:$imm))), + (ADDVL_XXI GPR64:$op, $imm)>; + + def : Pat<(add GPR32:$op, (i32 (trunc (vscale (sve_rdvl_imm i32:$imm))))), + (i32 (EXTRACT_SUBREG (ADDVL_XXI (INSERT_SUBREG (i64 (IMPLICIT_DEF)), + GPR32:$op, sub_32), $imm), + sub_32))>; + def : Pat<(add GPR64:$op, (vscale (sve_cnth_imm i32:$imm))), (INCH_XPiI GPR64:$op, 31, $imm)>; def : Pat<(add GPR64:$op, (vscale (sve_cntw_imm i32:$imm))), @@ -2488,9 +2488,6 @@ let Predicates = [HasSVEorSME] in { sub_32))>; } - def : Pat<(add GPR64:$op, (vscale (sve_rdvl_imm i32:$imm))), - (ADDVL_XXI GPR64:$op, $imm)>; - // FIXME: BigEndian requires an additional REV instruction to satisfy the // constraint that none of the bits change when stored to memory as one // type, and reloaded as another type. diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll index aefacc605474fa..bb58248c6f60e0 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll @@ -195,13 +195,14 @@ define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: neg x10, x9 ; CHECK-NEXT: mov w11, #1000 // =0x3e8 +; CHECK-NEXT: rdvl x13, #2 ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: and x10, x10, x11 -; CHECK-NEXT: rdvl x11, #4 ; CHECK-NEXT: zip2 z0.d, z1.d, z1.d ; CHECK-NEXT: zip1 z1.d, z1.d, z1.d -; CHECK-NEXT: addvl x12, x1, #2 -; CHECK-NEXT: addvl x13, x0, #2 +; CHECK-NEXT: rdvl x11, #4 +; CHECK-NEXT: add x12, x1, x13 +; CHECK-NEXT: add x13, x0, x13 ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: .LBB2_1: // %vector.body diff --git a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll index f1e95ca9c206f0..06570b4539cc11 100644 --- a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll +++ b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll @@ -351,9 +351,9 @@ define @splice_nxv16f32_16( %a, @splice_nxv16i8_neg17( %a, @splice_nxv8i16_neg9( %a, @splice_nxv8f16_neg9( %a, @splice_nxv8i32( %a, @splice_nxv16f32_neg17( %a, @extract_v16i8_nxv16i8_idx16( %vec) nounwind ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov x8, #-16 // =0xfffffffffffffff0 +; CHECK-NEXT: rdvl x8, #1 ; CHECK-NEXT: mov w9, #16 // =0x10 -; CHECK-NEXT: addvl x8, x8, #1 +; CHECK-NEXT: sub x8, x8, #16 ; CHECK-NEXT: cmp x8, #16 ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp diff --git a/llvm/test/CodeGen/AArch64/sve-gep.ll b/llvm/test/CodeGen/AArch64/sve-gep.ll index fd93e43613c52c..8ebd0198e9099a 100644 --- a/llvm/test/CodeGen/AArch64/sve-gep.ll +++ b/llvm/test/CodeGen/AArch64/sve-gep.ll @@ -4,7 +4,8 @@ define * @scalar_of_scalable_1(* %base) { ; CHECK-LABEL: scalar_of_scalable_1: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x0, x0, #4 +; CHECK-NEXT: rdvl x8, #4 +; CHECK-NEXT: add x0, x0, x8 ; CHECK-NEXT: ret %d = getelementptr , * %base, i64 4 ret * %d @@ -202,7 +203,8 @@ define @scalable_of_fixed_5_i64(i64* %base, *> @scalable_of_scalable_1(* %base) { ; CHECK-LABEL: scalable_of_scalable_1: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #1 +; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: add x8, x0, x8 ; CHECK-NEXT: mov z0.d, x8 ; CHECK-NEXT: ret %idx = shufflevector insertelement ( undef, i64 1, i32 0), zeroinitializer, zeroinitializer diff --git a/llvm/test/CodeGen/AArch64/sve-insert-element.ll b/llvm/test/CodeGen/AArch64/sve-insert-element.ll index f327e32c92e006..2aa298f6d9173f 100644 --- a/llvm/test/CodeGen/AArch64/sve-insert-element.ll +++ b/llvm/test/CodeGen/AArch64/sve-insert-element.ll @@ -590,10 +590,10 @@ define @test_predicate_insert_32xi1( %val, ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: ptrue p2.b -; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff +; CHECK-NEXT: rdvl x8, #2 ; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1 ; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1 -; CHECK-NEXT: addvl x8, x8, #2 +; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: mov w9, w1 ; CHECK-NEXT: cmp x9, x8 ; CHECK-NEXT: csel x8, x9, x8, lo diff --git a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll index 9ca928c00299fe..de2efe288ac470 100644 --- a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll @@ -120,9 +120,9 @@ define @insert_v16i8_nxv16i8_idx16( %vec, < ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov x8, #-16 // =0xfffffffffffffff0 +; CHECK-NEXT: rdvl x8, #1 ; CHECK-NEXT: mov w9, #16 // =0x10 -; CHECK-NEXT: addvl x8, x8, #1 +; CHECK-NEXT: sub x8, x8, #16 ; CHECK-NEXT: mov x10, sp ; CHECK-NEXT: cmp x8, #16 ; CHECK-NEXT: csel x8, x8, x9, lo diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-nf.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-nf.ll index a774af20f2e93c..f86d999340184e 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-nf.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-loads-nf.ll @@ -17,7 +17,8 @@ define @ldnf1b( %pg, ptr %a) { define @ldnf1b_out_of_lower_bound( %pg, ptr %a) { ; CHECK-LABEL: ldnf1b_out_of_lower_bound: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #-9 +; CHECK-NEXT: rdvl x8, #-9 +; CHECK-NEXT: add x8, x0, x8 ; CHECK-NEXT: ldnf1b { z0.b }, p0/z, [x8] ; CHECK-NEXT: ret %base = getelementptr , * %a, i64 -9 @@ -62,7 +63,8 @@ define @ldnf1b_upper_bound( %pg, ptr %a) { define @ldnf1b_out_of_upper_bound( %pg, ptr %a) { ; CHECK-LABEL: ldnf1b_out_of_upper_bound: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #8 +; CHECK-NEXT: rdvl x8, #8 +; CHECK-NEXT: add x8, x0, x8 ; CHECK-NEXT: ldnf1b { z0.b }, p0/z, [x8] ; CHECK-NEXT: ret %base = getelementptr , * %a, i64 8 diff --git a/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-imm.ll index c85779b99b048a..c5a3945cd04748 100644 --- a/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve-pred-contiguous-ldst-addressing-mode-reg-imm.ll @@ -9,9 +9,11 @@ define void @imm_out_of_range( * %base, %mask) nounwind { ; CHECK-LABEL: imm_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #8 +; CHECK-NEXT: rdvl x8, #8 +; CHECK-NEXT: add x8, x0, x8 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x8] -; CHECK-NEXT: addvl x8, x0, #-9 +; CHECK-NEXT: rdvl x8, #-9 +; CHECK-NEXT: add x8, x0, x8 ; CHECK-NEXT: st1d { z0.d }, p0, [x8] ; CHECK-NEXT: ret %base_load = getelementptr , * %base, i64 8 diff --git a/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-imm.ll index 73f23c3952ccc1..c06921ee96a0d4 100644 --- a/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve-pred-non-temporal-ldst-addressing-mode-reg-imm.ll @@ -9,9 +9,11 @@ define void @imm_out_of_range( * %base, %mask) nounwind { ; CHECK-LABEL: imm_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #8 +; CHECK-NEXT: rdvl x8, #8 +; CHECK-NEXT: add x8, x0, x8 ; CHECK-NEXT: ldnt1d { z0.d }, p0/z, [x8] -; CHECK-NEXT: addvl x8, x0, #-9 +; CHECK-NEXT: rdvl x8, #-9 +; CHECK-NEXT: add x8, x0, x8 ; CHECK-NEXT: stnt1d { z0.d }, p0, [x8] ; CHECK-NEXT: ret %base_load = getelementptr , * %base, i64 8 diff --git a/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll index d79990e9e9616e..a1c2ec9c7e1d42 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll @@ -23,9 +23,9 @@ define i8 @split_extract_32i8_idx( %a, i32 %idx) { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff +; CHECK-NEXT: rdvl x8, #2 ; CHECK-NEXT: mov w9, w0 -; CHECK-NEXT: addvl x8, x8, #2 +; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x9, x8 ; CHECK-NEXT: csel x8, x9, x8, lo ; CHECK-NEXT: mov x9, sp @@ -47,9 +47,9 @@ define i16 @split_extract_16i16_idx( %a, i32 %idx) { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff +; CHECK-NEXT: rdvl x8, #1 ; CHECK-NEXT: mov w9, w0 -; CHECK-NEXT: addvl x8, x8, #1 +; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x9, x8 ; CHECK-NEXT: csel x8, x9, x8, lo ; CHECK-NEXT: mov x9, sp @@ -141,9 +141,9 @@ define i16 @split_extract_16i16( %a) { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff +; CHECK-NEXT: rdvl x8, #1 ; CHECK-NEXT: mov w9, #128 // =0x80 -; CHECK-NEXT: addvl x8, x8, #1 +; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, #128 ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp @@ -165,10 +165,10 @@ define i32 @split_extract_16i32( %a) { ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff +; CHECK-NEXT: rdvl x8, #1 ; CHECK-NEXT: mov w9, #34464 // =0x86a0 ; CHECK-NEXT: movk w9, #1, lsl #16 -; CHECK-NEXT: addvl x8, x8, #1 +; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, x9 ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp diff --git a/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll b/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll index 7984057241c847..5441659fa5cb45 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll @@ -24,9 +24,9 @@ define @split_insert_32i8_idx( %a, i8 %elt, ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff +; CHECK-NEXT: rdvl x8, #2 ; CHECK-NEXT: mov x9, sp -; CHECK-NEXT: addvl x8, x8, #2 +; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x1, x8 ; CHECK-NEXT: csel x8, x1, x8, lo ; CHECK-NEXT: st1b { z1.b }, p0, [sp, #1, mul vl] @@ -136,9 +136,9 @@ define @split_insert_32i16( %a, i16 %elt) ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff +; CHECK-NEXT: rdvl x8, #2 ; CHECK-NEXT: mov w9, #128 // =0x80 -; CHECK-NEXT: addvl x8, x8, #2 +; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, #128 ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp diff --git a/llvm/test/CodeGen/AArch64/sve-vl-arith.ll b/llvm/test/CodeGen/AArch64/sve-vl-arith.ll index cd11972efc8011..dd4294c8d3bdcc 100644 --- a/llvm/test/CodeGen/AArch64/sve-vl-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-vl-arith.ll @@ -116,7 +116,8 @@ define @decd_vec( %a) { define i64 @incb_scalar_i64(i64 %a) { ; NO_SCALAR_INC-LABEL: incb_scalar_i64: ; NO_SCALAR_INC: // %bb.0: -; NO_SCALAR_INC-NEXT: addvl x0, x0, #1 +; NO_SCALAR_INC-NEXT: rdvl x8, #1 +; NO_SCALAR_INC-NEXT: add x0, x0, x8 ; NO_SCALAR_INC-NEXT: ret ; ; CHECK-LABEL: incb_scalar_i64: @@ -185,7 +186,8 @@ define i64 @incd_scalar_i64(i64 %a) { define i64 @decb_scalar_i64(i64 %a) { ; NO_SCALAR_INC-LABEL: decb_scalar_i64: ; NO_SCALAR_INC: // %bb.0: -; NO_SCALAR_INC-NEXT: addvl x0, x0, #-2 +; NO_SCALAR_INC-NEXT: rdvl x8, #-2 +; NO_SCALAR_INC-NEXT: add x0, x0, x8 ; NO_SCALAR_INC-NEXT: ret ; ; CHECK-LABEL: decb_scalar_i64: @@ -257,9 +259,8 @@ define i64 @decd_scalar_i64(i64 %a) { define i32 @incb_scalar_i32(i32 %a) { ; NO_SCALAR_INC-LABEL: incb_scalar_i32: ; NO_SCALAR_INC: // %bb.0: -; NO_SCALAR_INC-NEXT: // kill: def $w0 killed $w0 def $x0 -; NO_SCALAR_INC-NEXT: addvl x0, x0, #3 -; NO_SCALAR_INC-NEXT: // kill: def $w0 killed $w0 killed $x0 +; NO_SCALAR_INC-NEXT: rdvl x8, #3 +; NO_SCALAR_INC-NEXT: add w0, w0, w8 ; NO_SCALAR_INC-NEXT: ret ; ; CHECK-LABEL: incb_scalar_i32: @@ -344,9 +345,8 @@ define i32 @incd_scalar_i32(i32 %a) { define i32 @decb_scalar_i32(i32 %a) { ; NO_SCALAR_INC-LABEL: decb_scalar_i32: ; NO_SCALAR_INC: // %bb.0: -; NO_SCALAR_INC-NEXT: // kill: def $w0 killed $w0 def $x0 -; NO_SCALAR_INC-NEXT: addvl x0, x0, #-4 -; NO_SCALAR_INC-NEXT: // kill: def $w0 killed $w0 killed $x0 +; NO_SCALAR_INC-NEXT: rdvl x8, #-4 +; NO_SCALAR_INC-NEXT: add w0, w0, w8 ; NO_SCALAR_INC-NEXT: ret ; ; CHECK-LABEL: decb_scalar_i32: