Add Efficient 64b Addressing support

Add Efficient 64b Addressing support

Author: matborzyszkowski

IGC/AdaptorCommon/ImplicitArgs.cpp

@@ -167,6 +167,11 @@ static const std::vector<ImplicitArg> IMPLICIT_ARGS = {
     ImplicitArg(ImplicitArg::ASSERT_BUFFER_POINTER, "assertBufferPointer", ImplicitArg::GLOBALPTR,
                 WIAnalysis::UNIFORM_GLOBAL, 1, ImplicitArg::ALIGN_GRF, false,
                 GenISAIntrinsic::GenISA_getAssertBufferPtr),
+    ImplicitArg(ImplicitArg::INDIRECT_DATA_POINTER, "indirectDataPointer", ImplicitArg::GLOBALPTR,
+                WIAnalysis::UNIFORM_GLOBAL, 1, ImplicitArg::ALIGN_GRF, true,
+                GenISAIntrinsic::GenISA_getIndirectDataPtr),
+    ImplicitArg(ImplicitArg::SCRATCH_POINTER, "scratchPointer", ImplicitArg::GLOBALPTR, WIAnalysis::UNIFORM_GLOBAL, 1,
+                ImplicitArg::ALIGN_PTR, true, GenISAIntrinsic::GenISA_getScratchPtr),
 
     // BufferBoundsChecking
     ImplicitArg(ImplicitArg::BUFFER_SIZE, "bufferSize", ImplicitArg::LONG, WIAnalysis::UNIFORM_GLOBAL, 1,

IGC/AdaptorCommon/ImplicitArgs.hpp

@@ -124,6 +124,8 @@ class ImplicitArg {
     IMPLICIT_ARG_BUFFER_PTR,
 
     ASSERT_BUFFER_POINTER,
+    INDIRECT_DATA_POINTER,
+    SCRATCH_POINTER,
 
     // BufferBoundsChecking
     BUFFER_SIZE,

IGC/AdaptorCommon/RayTracing/API/RayDispatchGlobalData.h

@@ -115,6 +115,7 @@ class RayDispatchGlobalDataAdaptor {
   uint64_t GetStatelessScratchPtr() const { return 0; };
   uint32_t GetBaseSSHOffset() const { return 0; };
   uint32_t GetUberTilesMap() const { return 0; };
+  uint64_t GetBaseSurfaceStatePointer() const { return 0; };
 };
 
 // Layout used to pass global data to the shaders
@@ -155,6 +156,15 @@ struct RayDispatchGlobalData {
         uint32_t paddingBits3;  // 32-bits of padding
       };
 
+      // For Efficient64 there is no baseSSH, the full address of the
+      // resource must calculated in the shader. Instead of delivering
+      // separately the offset to the first Stack SurfaceState, and
+      // the pointer to the baseSurfaceState (in Constants), UMD
+      // will deliver baseSurfaceStatePointer + OffsetToTheFirstStackSS
+      // in baseSurfaceStatePointer.
+      struct {
+        uint64_t baseSurfaceStatePointer;
+      };
     };
 
     uint64_t uberTilesMap; // base address of the uber tiles map used for AtomicPull model
@@ -177,6 +187,11 @@ struct RayDispatchGlobalData {
       pMissShaderStride = umd.GetMissStride();
       pRtMemBasePtr = umd.GetRayStackBufferAddress();
       baseSSHOffset = umd.GetBaseSSHOffset();
+      // Check if baseSurfaceStatePointer is delivered from the UMD, as
+      // it shares the same memory with baseSSHOffset in the RayDispatchGlobalDataCommon.
+      // baseSurfaceStatePointer will be written only if it is not zero.
+      if (uint64_t Tmp = umd.GetBaseSurfaceStatePointer())
+        baseSurfaceStatePointer = Tmp;
       pStackSizePerRay = umd.GetStackSizePerRay();
       swStackSizePerRay = umd.GetSWStackSizePerRay();
       pNumDSSRTStacks = umd.GetNumDSSRTStacks();

IGC/AdaptorCommon/RayTracing/RTBuilder.cpp

@@ -116,6 +116,13 @@ Value *RTBuilder::getMaxBVHLevels(void) {
 Value *RTBuilder::getStatelessScratchPtr(void) { return _get_statelessScratchPtr(VALUE_NAME("statelessScratchPtr")); }
 
 
+Value *RTBuilder::getBaseSurfaceStatePointer(Value *rayDispatchGlobalDataPtr) {
+  // For non-RT shaders, which use RayQuery GlobalBufferPointer is delivered
+  // in pushConstants. It must be read and passed to this function.
+  return _getBaseSurfaceStatePointerFromPointerToGlobals(rayDispatchGlobalDataPtr,
+                                                         VALUE_NAME("BaseSurfaceStatePointerFromPointerToGlobals"));
+}
+
 Value *RTBuilder::getIsFrontFace(RTBuilder::StackPointerVal *StackPointer, Value *ShaderTy) {
   auto *isCommitted = CreateICmpEQ(ShaderTy, getInt32(CallableShaderTypeMD::ClosestHit));
   switch (getMemoryStyle()) {

IGC/AdaptorCommon/RayTracing/RTBuilder.h

@@ -204,6 +204,8 @@ class RTBuilder : public IGCIRBuilder<> {
   Value *getRtMemBasePtr(Value *globalBufferPtr = nullptr);
   Value *getStackSizePerRay(void);
   Value *getNumDSSRTStacks(void);
+  Value *getBaseSurfaceStatePointer(void);
+  Value *getBaseSurfaceStatePointer(Value *RayDispatchGlobalData);
   Value *getMaxBVHLevels(void);
   Value *getStatelessScratchPtr(void);
   Value *getLeafType(StackPointerVal *StackPointer, Value *CommittedHit);

IGC/AdaptorCommon/RayTracing/RTStackReflectionIRBG/reflection.cpp

@@ -118,6 +118,10 @@ RTGlobals(maxBVHLevels, Xe3, xe3.rt_data_info);
 RTGlobalsCommon(statelessScratchPtr);
 
 
+CREATE_PRIVATE auto
+_getBaseSurfaceStatePointerFromPointerToGlobals(RTGAS RayDispatchGlobalData *__restrict__ GlobalsPtr) {
+  return GlobalsPtr->rt.xe3.common.baseSurfaceStatePointer;
+}
 
 //////////// End Common Fields ////////////
 

IGC/AdaptorOCL/Utils/CacheControlsHelper.h

@@ -76,6 +76,54 @@ const CacheControlMapTy<LoadCacheControl> supportedLoadConfigs = {
         { LSC_L1IAR_L3IAR,     { LoadCacheControl::InvalidateAfterRead, LoadCacheControl::InvalidateAfterRead } },
     // clang-format on
 };
+template <typename T> struct SeparateCacheControlsL1L2L3 {
+  T L1;
+  T L2;
+  T L3;
+};
+
+template <typename T>
+using L1L2L3StoreCacheControlMapTy =
+    std::unordered_map<LSC_STCC_L1_L2_L3, SeparateCacheControlsL1L2L3<T>, std::hash<int>>;
+const L1L2L3StoreCacheControlMapTy<StoreCacheControl> supportedL1L2L3StoreConfigs = {
+    // clang-format off
+        { LSC_STCC_L1UC_L2UC_L3UC, { StoreCacheControl::Uncached,     StoreCacheControl::Uncached,  StoreCacheControl::Uncached } },
+        { LSC_STCC_L1UC_L2UC_L3WB, { StoreCacheControl::Uncached,     StoreCacheControl::Uncached,  StoreCacheControl::WriteBack } },
+        { LSC_STCC_L1UC_L2WB_L3UC, { StoreCacheControl::Uncached,     StoreCacheControl::WriteBack, StoreCacheControl::Uncached } },
+        { LSC_STCC_L1UC_L2WB_L3WB, { StoreCacheControl::Uncached,     StoreCacheControl::WriteBack, StoreCacheControl::WriteBack } },
+        { LSC_STCC_L1WT_L2UC_L3UC, { StoreCacheControl::WriteThrough, StoreCacheControl::Uncached,  StoreCacheControl::Uncached } },
+        { LSC_STCC_L1WT_L2UC_L3WB, { StoreCacheControl::WriteThrough, StoreCacheControl::Uncached,  StoreCacheControl::WriteBack } },
+        { LSC_STCC_L1WT_L2WB_L3UC, { StoreCacheControl::WriteThrough, StoreCacheControl::WriteBack, StoreCacheControl::Uncached } },
+        { LSC_STCC_L1WT_L2WB_L3WB, { StoreCacheControl::WriteThrough, StoreCacheControl::WriteBack, StoreCacheControl::WriteBack } },
+        { LSC_STCC_L1S_L2UC_L3UC,  { StoreCacheControl::Streaming,    StoreCacheControl::Uncached,  StoreCacheControl::Uncached } },
+        { LSC_STCC_L1S_L2UC_L3WB,  { StoreCacheControl::Streaming,    StoreCacheControl::Uncached,  StoreCacheControl::WriteBack } },
+        { LSC_STCC_L1S_L2WB_L3UC,  { StoreCacheControl::Streaming,    StoreCacheControl::WriteBack, StoreCacheControl::Uncached } },
+        { LSC_STCC_L1WB_L2UC_L3UC, { StoreCacheControl::WriteBack,    StoreCacheControl::Uncached,  StoreCacheControl::Uncached } },
+        { LSC_STCC_L1WB_L2WB_L3UC, { StoreCacheControl::WriteBack,    StoreCacheControl::WriteBack, StoreCacheControl::Uncached } },
+        { LSC_STCC_L1WB_L2UC_L3WB, { StoreCacheControl::WriteBack,    StoreCacheControl::Uncached,  StoreCacheControl::WriteBack } },
+    // clang-format on
+};
+
+template <typename T>
+using L1L2L3LoadCacheControlMapTy =
+    std::unordered_map<LSC_LDCC_L1_L2_L3, SeparateCacheControlsL1L2L3<T>, std::hash<int>>;
+const L1L2L3LoadCacheControlMapTy<LoadCacheControl> supportedL1L2L3LoadConfigs = {
+    // clang-format off
+        { LSC_LDCC_L1UC_L2UC_L3UC,    { LoadCacheControl::Uncached,  LoadCacheControl::Uncached, LoadCacheControl::Uncached } },
+        { LSC_LDCC_L1UC_L2UC_L3C,     { LoadCacheControl::Uncached,  LoadCacheControl::Uncached, LoadCacheControl::Cached } },
+        { LSC_LDCC_L1UC_L2C_L3UC,     { LoadCacheControl::Uncached,  LoadCacheControl::Cached,   LoadCacheControl::Uncached } },
+        { LSC_LDCC_L1UC_L2C_L3C,      { LoadCacheControl::Uncached,  LoadCacheControl::Cached,   LoadCacheControl::Cached } },
+        { LSC_LDCC_L1C_L2UC_L3UC,     { LoadCacheControl::Cached,    LoadCacheControl::Uncached, LoadCacheControl::Uncached } },
+        { LSC_LDCC_L1C_L2UC_L3C,      { LoadCacheControl::Cached,    LoadCacheControl::Uncached, LoadCacheControl::Cached } },
+        { LSC_LDCC_L1C_L2C_L3UC,      { LoadCacheControl::Cached,    LoadCacheControl::Cached,   LoadCacheControl::Uncached } },
+        { LSC_LDCC_L1C_L2C_L3C,       { LoadCacheControl::Cached,    LoadCacheControl::Cached,   LoadCacheControl::Cached } },
+        { LSC_LDCC_L1S_L2UC_L3UC,     { LoadCacheControl::Streaming, LoadCacheControl::Uncached, LoadCacheControl::Uncached } },
+        { LSC_LDCC_L1S_L2UC_L3C,      { LoadCacheControl::Streaming, LoadCacheControl::Uncached, LoadCacheControl::Cached } },
+        { LSC_LDCC_L1S_L2C_L3UC,      { LoadCacheControl::Streaming, LoadCacheControl::Cached,   LoadCacheControl::Uncached } },
+        { LSC_LDCC_L1S_L2C_L3C,       { LoadCacheControl::Streaming, LoadCacheControl::Cached,   LoadCacheControl::Cached } },
+        { LSC_LDCC_L1IAR_L2IAR_L3IAR, { LoadCacheControl::InvalidateAfterRead, LoadCacheControl::InvalidateAfterRead, LoadCacheControl::InvalidateAfterRead } },
+    // clang-format on
+};
 
 using CacheLevel = uint64_t;
 

IGC/AdaptorOCL/dllInterfaceCompute.cpp

@@ -1418,6 +1418,9 @@ bool TranslateBuild(const STB_TranslateInputArgs *pInputArgs, STB_TranslateOutpu
                     TB_DATA_FORMAT inputDataFormatTemp, const IGC::CPlatform &IGCPlatform,
                     float profilingTimerResolution) {
   ShaderHash inputShHash;
+  if (IGCPlatform.hasEfficient64bEnabled()) {
+    IGC_SET_FLAG_VALUE(EnableEfficient64b, true);
+  }
   if (IGC_IS_FLAG_ENABLED(EnableKernelNamesBasedHash)) {
     // Create the hash based on kernel names.
     // This takes the names and concatenates them into a string

IGC/AdaptorOCL/ocl_igc_interface/igc_features_and_workarounds.h

@@ -87,7 +87,13 @@ CIF_DEFINE_INTERFACE_VER_WITH_COMPATIBILITY(IgcFeaturesAndWorkarounds, 2, 1) {
   OCL_API_CALL virtual void SetMaxOCLParamSize(uint32_t s);
   OCL_API_CALL virtual uint32_t GetMaxOCLParamSize() const;
 };
-CIF_DEFINE_INTERFACE_VER_WITH_COMPATIBILITY(IgcFeaturesAndWorkarounds, 3, 2) { CIF_INHERIT_CONSTRUCTOR(); };
+
+CIF_DEFINE_INTERFACE_VER_WITH_COMPATIBILITY(IgcFeaturesAndWorkarounds, 3, 2) {
+  CIF_INHERIT_CONSTRUCTOR();
+
+  virtual void SetFtrEfficient64BitAddressing(bool v);
+  virtual bool GetFtrEfficient64BitAddressing() const;
+};
 
 CIF_GENERATE_VERSIONS_LIST(IgcFeaturesAndWorkarounds);
 CIF_MARK_LATEST_VERSION(IgcFeaturesAndWorkaroundsLatest, IgcFeaturesAndWorkarounds);

IGC/AdaptorOCL/ocl_igc_interface/impl/igc_features_and_workarounds_impl.cpp

@@ -53,6 +53,12 @@ void CIF_GET_INTERFACE_CLASS(IgcFeaturesAndWorkarounds, 2)::SetMaxOCLParamSize(u
   IGC_ASSERT(s >= OCLCaps::MINIMAL_MAX_PARAMETER_SIZE);
   CIF_GET_PIMPL()->OCLCaps.MaxParameterSize = s;
 }
+bool CIF_GET_INTERFACE_CLASS(IgcFeaturesAndWorkarounds, 3)::GetFtrEfficient64BitAddressing() const {
+  return CIF_GET_PIMPL()->FeTable.FtrEfficient64BitAddressing;
+}
+void CIF_GET_INTERFACE_CLASS(IgcFeaturesAndWorkarounds, 3)::SetFtrEfficient64BitAddressing(bool v) {
+  CIF_GET_PIMPL()->FeTable.FtrEfficient64BitAddressing = v;
+}
 
 } // namespace IGC