[VP] Custom strided VLD/VST VPBuilder path

Author: Simon Moll

llvm/include/llvm/IR/VPBuilder.h

@@ -26,8 +26,10 @@ class VPBuilder {
   Value& RequestPred();
   Value& RequestEVL();
 
-  // Cast to vector of element type.
+  // Cast to pointer to vector of \p Ptr/
   Value& GetAsVectorPointer(Value& Ptr);
+  // Cast to pointer of element of vector type.
+  Value& GetAsElementPointer(Value &Ptr);
 
 public:
   VPBuilder(IRBuilder<> &_builder)
@@ -73,6 +75,10 @@ class VPBuilder {
                                MaybeAlign Alignment);
   Value &CreateContiguousLoad(Type *ReturnTy, Value &Pointer,
                               MaybeAlign Alignment);
+  Value &CreateStridedStore(Value &Val, Value &Pointer, MaybeAlign Alignment,
+                            int64_t Stride);
+  Value &CreateStridedLoad(Type *ReturnTy, Value &Pointer, MaybeAlign Alignment,
+                           int64_t Stride);
   Value &CreateScatter(Value &Val, Value &PointerVec, MaybeAlign Alignment);
   Value &CreateGather(Type *RetTy, Value &PointerVec, MaybeAlign AlignOpt);
   Value &createSelect(Value &OnTrue, Value &OnFalse, Value &Mask, Value &Pivot,

llvm/lib/IR/IntrinsicInst.cpp

@@ -618,13 +618,13 @@ static VPIntrinsic::ShortTypeVec getVPIntrinsicTypes(Intrinsic::ID ID,
     return VPIntrinsic::ShortTypeVec{VectorTy};
 
   case Intrinsic::experimental_vp_strided_load:
-    return VPIntrinsic::ShortTypeVec{VecRetTy,
+    return VPIntrinsic::ShortTypeVec{VecRetTy, VecPtrTy,
                                      Type::getInt64Ty(VectorTy->getContext())};
   case Intrinsic::vp_gather:
   case Intrinsic::vp_load:
     return VPIntrinsic::ShortTypeVec{VecRetTy, VecPtrTy};
   case Intrinsic::experimental_vp_strided_store:
-    return VPIntrinsic::ShortTypeVec{VectorTy,
+    return VPIntrinsic::ShortTypeVec{VectorTy, VecPtrTy,
                                      Type::getInt64Ty(VectorTy->getContext())};
   case Intrinsic::vp_scatter:
   case Intrinsic::vp_store:

llvm/lib/IR/VPBuilder.cpp

@@ -130,6 +130,19 @@ VectorType &VPBuilder::getVectorType(Type &ElementTy) {
   return *VectorType::get(&ElementTy, StaticVectorLength);
 }
 
+Value &VPBuilder::GetAsElementPointer(Value &Ptr) {
+  auto *ElemTy = Ptr.getType()->getPointerElementType();
+  if (!ElemTy->isVectorTy())
+    return Ptr;
+  auto &PointerTy = cast<PointerType>(*Ptr.getType());
+  if (auto *VecTy = dyn_cast<VectorType>(PointerTy.getPointerElementType())) {
+    auto *ElemPtrTy =
+        VecTy->getElementType()->getPointerTo(PointerTy.getAddressSpace());
+    return *Builder.CreatePointerCast(&Ptr, ElemPtrTy);
+  }
+  return Ptr;
+}
+
 Value &VPBuilder::GetAsVectorPointer(Value &Ptr) {
   auto *ElemTy = Ptr.getType()->getPointerElementType();
   if (ElemTy->isVectorTy())
@@ -140,6 +153,46 @@ Value &VPBuilder::GetAsVectorPointer(Value &Ptr) {
   return *Builder.CreatePointerCast(&Ptr, VecPtrTy);
 }
 
+Value &VPBuilder::CreateStridedStore(Value &Val, Value &Ptr,
+                                     MaybeAlign AlignOpt, int64_t Stride) {
+  auto &ElemPtr = GetAsElementPointer(Ptr);
+  auto &VecTy = getVectorType(*ElemPtr.getType()->getPointerElementType());
+  auto *StoreFunc = Intrinsic::getDeclaration(
+      &getModule(), Intrinsic::experimental_vp_strided_store,
+      {&VecTy, Type::getInt64Ty(Builder.getContext())});
+  auto *StrideConst = Builder.getInt64((uint64_t)Stride);
+
+  ShortValueVec Args{&Val, &ElemPtr, StrideConst, &RequestPred(),
+                     &RequestEVL()};
+  CallInst &StoreCall = *Builder.CreateCall(StoreFunc, Args);
+  if (AlignOpt.hasValue()) {
+    unsigned PtrPos =
+        VPIntrinsic::getMemoryPointerParamPos(Intrinsic::vp_store).getValue();
+    StoreCall.addParamAttr(
+        PtrPos, Attribute::getWithAlignment(getContext(), AlignOpt.getValue()));
+  }
+  return StoreCall;
+}
+
+Value &VPBuilder::CreateStridedLoad(Type *ReturnTy, Value &Ptr,
+                                    MaybeAlign AlignOpt, int64_t Stride) {
+  auto &ElemPtr = GetAsElementPointer(Ptr);
+  // auto &VecTy = getVectorType(*ElemPtr.getType()->getPointerElementType());
+  auto *LoadFunc = VPIntrinsic::getDeclarationForParams(
+      &getModule(), Intrinsic::experimental_vp_strided_load, ReturnTy,
+      {&ElemPtr});
+  auto *StrideConst = Builder.getInt64((uint64_t)Stride);
+  ShortValueVec Args{&ElemPtr, StrideConst, &RequestPred(), &RequestEVL()};
+  CallInst &LoadCall = *Builder.CreateCall(LoadFunc, Args);
+  if (AlignOpt.hasValue()) {
+    unsigned PtrPos =
+        VPIntrinsic::getMemoryPointerParamPos(Intrinsic::vp_load).getValue();
+    LoadCall.addParamAttr(
+        PtrPos, Attribute::getWithAlignment(getContext(), AlignOpt.getValue()));
+  }
+  return LoadCall;
+}
+
 Value &VPBuilder::CreateContiguousStore(Value &Val, Value &ElemPointer,
                                         MaybeAlign AlignOpt) {
   auto &VecPtr = GetAsVectorPointer(ElemPointer);