diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp index 6e0f429c34b2f6..35c46157c2eb9c 100644 --- a/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp +++ b/llvm/lib/Target/RISCV/RISCVCodeGenPrepare.cpp @@ -18,9 +18,11 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsRISCV.h" #include "llvm/IR/PatternMatch.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" @@ -35,6 +37,7 @@ namespace { class RISCVCodeGenPrepare : public FunctionPass, public InstVisitor { const DataLayout *DL; + const DominatorTree *DT; const RISCVSubtarget *ST; public: @@ -48,12 +51,14 @@ class RISCVCodeGenPrepare : public FunctionPass, void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); + AU.addRequired(); AU.addRequired(); } bool visitInstruction(Instruction &I) { return false; } bool visitAnd(BinaryOperator &BO); bool visitIntrinsicInst(IntrinsicInst &I); + bool expandVPStrideLoad(IntrinsicInst &I); }; } // end anonymous namespace @@ -128,6 +133,9 @@ bool RISCVCodeGenPrepare::visitAnd(BinaryOperator &BO) { // Which eliminates the scalar -> vector -> scalar crossing during instruction // selection. bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) { + if (expandVPStrideLoad(I)) + return true; + if (I.getIntrinsicID() != Intrinsic::vector_reduce_fadd) return false; @@ -155,6 +163,48 @@ bool RISCVCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) { return true; } +bool RISCVCodeGenPrepare::expandVPStrideLoad(IntrinsicInst &II) { + if (ST->hasOptimizedZeroStrideLoad()) + return false; + + Value *BasePtr, *VL; + + using namespace PatternMatch; + if (!match(&II, m_Intrinsic( + m_Value(BasePtr), m_Zero(), m_AllOnes(), m_Value(VL)))) + return false; + + if (!isKnownNonZero(VL, {*DL, DT, nullptr, &II})) + return false; + + auto *VTy = cast(II.getType()); + + IRBuilder<> Builder(&II); + + // Extend VL from i32 to XLen if needed. + if (ST->is64Bit()) + VL = Builder.CreateZExt(VL, Builder.getInt64Ty()); + + Type *STy = VTy->getElementType(); + Value *Val = Builder.CreateLoad(STy, BasePtr); + const auto &TLI = *ST->getTargetLowering(); + Value *Res; + + // TODO: Also support fixed/illegal vector types to splat with evl = vl. + if (isa(VTy) && TLI.isTypeLegal(EVT::getEVT(VTy))) { + unsigned VMVOp = STy->isFloatingPointTy() ? Intrinsic::riscv_vfmv_v_f + : Intrinsic::riscv_vmv_v_x; + Res = Builder.CreateIntrinsic(VMVOp, {VTy, VL->getType()}, + {PoisonValue::get(VTy), Val, VL}); + } else { + Res = Builder.CreateVectorSplat(VTy->getElementCount(), Val); + } + + II.replaceAllUsesWith(Res); + II.eraseFromParent(); + return true; +} + bool RISCVCodeGenPrepare::runOnFunction(Function &F) { if (skipFunction(F)) return false; @@ -164,6 +214,7 @@ bool RISCVCodeGenPrepare::runOnFunction(Function &F) { ST = &TM.getSubtarget(F); DL = &F.getDataLayout(); + DT = &getAnalysis().getDomTree(); bool MadeChange = false; for (auto &BB : F) diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll index 5e64e9fbc1a2f5..41c7d1f5fd64c7 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll @@ -1,10 +1,16 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+v,+zvfh,+optimized-zero-stride-load \ +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,CHECK-OPT +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+v,+zvfh,+optimized-zero-stride-load \ +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,CHECK-OPT ; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+v,+zvfh \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV32 +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,CHECK-NO-OPT ; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+v,+zvfh \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV64 +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,CHECK-NO-OPT declare <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i8(ptr, i8, <2 x i1>, i32) @@ -626,3 +632,41 @@ define <33 x double> @strided_load_v33f64(ptr %ptr, i64 %stride, <33 x i1> %mask } declare <33 x double> @llvm.experimental.vp.strided.load.v33f64.p0.i64(ptr, i64, <33 x i1>, i32) + +; TODO: Use accurate evl. +; Test unmasked integer zero strided +define <4 x i8> @zero_strided_unmasked_vpload_4i8_i8(ptr %ptr) { +; CHECK-OPT-LABEL: zero_strided_unmasked_vpload_4i8_i8: +; CHECK-OPT: # %bb.0: +; CHECK-OPT-NEXT: vsetivli zero, 3, e8, mf4, ta, ma +; CHECK-OPT-NEXT: vlse8.v v8, (a0), zero +; CHECK-OPT-NEXT: ret +; +; CHECK-NO-OPT-LABEL: zero_strided_unmasked_vpload_4i8_i8: +; CHECK-NO-OPT: # %bb.0: +; CHECK-NO-OPT-NEXT: lbu a0, 0(a0) +; CHECK-NO-OPT-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NO-OPT-NEXT: vmv.v.x v8, a0 +; CHECK-NO-OPT-NEXT: ret + %load = call <4 x i8> @llvm.experimental.vp.strided.load.4i8.p0.i8(ptr %ptr, i8 0, <4 x i1> splat (i1 true), i32 3) + ret <4 x i8> %load +} + +; TODO: Use accurate evl. +; Test unmasked float zero strided +define <4 x half> @zero_strided_unmasked_vpload_4f16(ptr %ptr) { +; CHECK-OPT-LABEL: zero_strided_unmasked_vpload_4f16: +; CHECK-OPT: # %bb.0: +; CHECK-OPT-NEXT: vsetivli zero, 3, e16, mf2, ta, ma +; CHECK-OPT-NEXT: vlse16.v v8, (a0), zero +; CHECK-OPT-NEXT: ret +; +; CHECK-NO-OPT-LABEL: zero_strided_unmasked_vpload_4f16: +; CHECK-NO-OPT: # %bb.0: +; CHECK-NO-OPT-NEXT: flh fa5, 0(a0) +; CHECK-NO-OPT-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NO-OPT-NEXT: vfmv.v.f v8, fa5 +; CHECK-NO-OPT-NEXT: ret + %load = call <4 x half> @llvm.experimental.vp.strided.load.4f16.p0.i32(ptr %ptr, i32 0, <4 x i1> splat (i1 true), i32 3) + ret <4 x half> %load +} diff --git a/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll index 4d3bced0bcb50f..6b8ded49142262 100644 --- a/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll +++ b/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll @@ -1,10 +1,16 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+v,+zvfh,+optimized-zero-stride-load \ +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,CHECK-OPT +; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+v,+zvfh,+optimized-zero-stride-load \ +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,CHECK-OPT ; RUN: llc -mtriple=riscv32 -mattr=+m,+d,+zfh,+v,+zvfh \ -; RUN: -verify-machineinstrs < %s | FileCheck %s \ -; RUN: -check-prefixes=CHECK,CHECK-RV32 +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV32,CHECK-NO-OPT ; RUN: llc -mtriple=riscv64 -mattr=+m,+d,+zfh,+v,+zvfh \ -; RUN: -verify-machineinstrs < %s | FileCheck %s \ -; RUN: -check-prefixes=CHECK,CHECK-RV64 +; RUN: -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-RV64,CHECK-NO-OPT declare @llvm.experimental.vp.strided.load.nxv1i8.p0.i8(ptr, i8, , i32) @@ -780,3 +786,39 @@ define @strided_load_nxv17f64(ptr %ptr, i64 %stride, @llvm.experimental.vp.strided.load.nxv17f64.p0.i64(ptr, i64, , i32) declare @llvm.experimental.vector.extract.nxv1f64( %vec, i64 %idx) declare @llvm.experimental.vector.extract.nxv16f64( %vec, i64 %idx) + +; Test unmasked integer zero strided +define @zero_strided_unmasked_vpload_nxv1i8_i8(ptr %ptr) { +; CHECK-OPT-LABEL: zero_strided_unmasked_vpload_nxv1i8_i8: +; CHECK-OPT: # %bb.0: +; CHECK-OPT-NEXT: vsetivli zero, 4, e8, mf8, ta, ma +; CHECK-OPT-NEXT: vlse8.v v8, (a0), zero +; CHECK-OPT-NEXT: ret +; +; CHECK-NO-OPT-LABEL: zero_strided_unmasked_vpload_nxv1i8_i8: +; CHECK-NO-OPT: # %bb.0: +; CHECK-NO-OPT-NEXT: lbu a0, 0(a0) +; CHECK-NO-OPT-NEXT: vsetivli zero, 4, e8, mf8, ta, ma +; CHECK-NO-OPT-NEXT: vmv.v.x v8, a0 +; CHECK-NO-OPT-NEXT: ret + %load = call @llvm.experimental.vp.strided.load.nxv1i8.p0.i8(ptr %ptr, i8 0, splat (i1 true), i32 4) + ret %load +} + +; Test unmasked float zero strided +define @zero_strided_unmasked_vpload_nxv1f16(ptr %ptr) { +; CHECK-OPT-LABEL: zero_strided_unmasked_vpload_nxv1f16: +; CHECK-OPT: # %bb.0: +; CHECK-OPT-NEXT: vsetivli zero, 4, e16, mf4, ta, ma +; CHECK-OPT-NEXT: vlse16.v v8, (a0), zero +; CHECK-OPT-NEXT: ret +; +; CHECK-NO-OPT-LABEL: zero_strided_unmasked_vpload_nxv1f16: +; CHECK-NO-OPT: # %bb.0: +; CHECK-NO-OPT-NEXT: flh fa5, 0(a0) +; CHECK-NO-OPT-NEXT: vsetivli zero, 4, e16, mf4, ta, ma +; CHECK-NO-OPT-NEXT: vfmv.v.f v8, fa5 +; CHECK-NO-OPT-NEXT: ret + %load = call @llvm.experimental.vp.strided.load.nxv1f16.p0.i32(ptr %ptr, i32 0, splat (i1 true), i32 4) + ret %load +}