[cublas] set CUBLAS_POINTER_MODE_HOST after setting it to device (uxlfoundation#94)

* [cublas] set CUBLAS_POINTER_MODE_HOST after setting it to device

Since we are reusing the cudaHandle we need to reset this after it has been set to DEVICE. Otherwise we get illigal memory acces later when we expect the defualt value ( CUBLAS_POINTER_MODE_HOST ) to be set.

* [cublas] reset CUBLAS_POINTER_MODE to host for rot and add description

* [cublas] Don't set pointer mode for rot function

Author: sbalint98

src/blas/backends/cublas/cublas_level1.cpp

@@ -56,6 +56,10 @@ inline void asum(Func func, cl::sycl::queue &queue, int64_t n, cl::sycl::buffer<
             cublasStatus_t err;
             // ASUM does not support negative index
             CUBLAS_ERROR_FUNC(func, err, handle, n, x_, std::abs(incx), res_);
+            // Higher level BLAS functions expect CUBLAS_POINTER_MODE_HOST
+            // to be set, therfore we need to reset this to the default value
+            // in order to avoid CUDA_ERROR_ILLEGAL_ADRESS errors
+            cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
         });
     });
 }
@@ -160,6 +164,10 @@ inline void rotg(Func func, cl::sycl::queue &queue, cl::sycl::buffer<T1, 1> &a,
             auto s_ = sc.get_mem<cuDataType1 *>(ih, s_acc);
             cublasStatus_t err;
             CUBLAS_ERROR_FUNC(func, err, handle, a_, b_, c_, s_);
+            // Higher level BLAS functions expect CUBLAS_POINTER_MODE_HOST
+            // to be set, therfore we need to reset this to the default value
+            // in order to avoid CUDA_ERROR_ILLEGAL_ADRESS errors
+            cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
         });
     });
 }
@@ -201,6 +209,10 @@ inline void rotm(Func func, cl::sycl::queue &queue, int64_t n, cl::sycl::buffer<
             auto param_ = sc.get_mem<cuDataType *>(ih, param_acc);
             cublasStatus_t err;
             CUBLAS_ERROR_FUNC(func, err, handle, n, x_, incx, y_, incy, param_);
+            // Higher level BLAS functions expect CUBLAS_POINTER_MODE_HOST
+            // to be set, therfore we need to reset this to the default value
+            // in order to avoid CUDA_ERROR_ILLEGAL_ADRESS errors
+            cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
         });
     });
 }
@@ -270,6 +282,10 @@ inline void dot(Func func, cl::sycl::queue &queue, int64_t n, cl::sycl::buffer<T
             auto res_ = sc.get_mem<cuDataType *>(ih, res_acc);
             cublasStatus_t err;
             CUBLAS_ERROR_FUNC(func, err, handle, n, x_, incx, y_, incy, res_);
+            // Higher level BLAS functions expect CUBLAS_POINTER_MODE_HOST
+            // to be set, therfore we need to reset this to the default value
+            // in order to avoid CUDA_ERROR_ILLEGAL_ADRESS errors
+            cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
         });
     });
 }
@@ -301,12 +317,6 @@ inline void rot(Func func, cl::sycl::queue &queue, int64_t n, cl::sycl::buffer<T
         cgh.interop_task([=](cl::sycl::interop_handler ih) {
             auto sc = CublasScopedContextHandler(queue);
             auto handle = sc.get_handle(queue);
-            // By default the pointer mode is the CUBLAS_POINTER_MODE_HOST
-            // when the data is on buffer, it must be set to
-            // CUBLAS_POINTER_MODE_DEVICE mode otherwise it causes the segmentation
-            // fault. When it is set to device it is users responsibility to
-            // synchronise as the function is completely asynchronous.
-            // cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_DEVICE);
             auto x_ = sc.get_mem<cuDataType1 *>(ih, x_acc);
             auto y_ = sc.get_mem<cuDataType1 *>(ih, y_acc);
             cublasStatus_t err;
@@ -351,6 +361,10 @@ void sdsdot(cl::sycl::queue &queue, int64_t n, float sb, cl::sycl::buffer<float,
             auto res_ = sc.get_mem<float *>(ih, res_acc);
             cublasStatus_t err;
             CUBLAS_ERROR_FUNC(cublasSdot, err, handle, n, x_, incx, y_, incy, res_);
+            // Higher level BLAS functions expect CUBLAS_POINTER_MODE_HOST
+            // to be set, therfore we need to reset this to the default value
+            // in order to avoid CUDA_ERROR_ILLEGAL_ADRESS errors
+            cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
         });
     });
     // Since SB is a host pointer we need to bring the result back to the host and
@@ -391,6 +405,10 @@ inline void rotmg(Func func, cl::sycl::queue &queue, cl::sycl::buffer<T, 1> &d1,
             auto param_ = sc.get_mem<cuDataType *>(ih, param_acc);
             cublasStatus_t err;
             CUBLAS_ERROR_FUNC(func, err, handle, d1_, d2_, x1_, y1_, param_);
+            // Higher level BLAS functions expect CUBLAS_POINTER_MODE_HOST
+            // to be set, therfore we need to reset this to the default value
+            // in order to avoid CUDA_ERROR_ILLEGAL_ADRESS errors
+            cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
         });
     });
 }
@@ -437,6 +455,10 @@ inline void iamax(Func func, cl::sycl::queue &queue, int64_t n, cl::sycl::buffer
             // For negative incx, iamax returns 0. This behaviour is similar to that of
             // reference netlib BLAS.
             CUBLAS_ERROR_FUNC(func, err, handle, n, x_, incx, int_res_);
+            // Higher level BLAS functions expect CUBLAS_POINTER_MODE_HOST
+            // to be set, therfore we need to reset this to the default value
+            // in order to avoid CUDA_ERROR_ILLEGAL_ADRESS errors
+            cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
         });
     });
     // This requires to bring the data to host, copy it, and return it back to
@@ -519,6 +541,10 @@ inline void iamin(Func func, cl::sycl::queue &queue, int64_t n, cl::sycl::buffer
             // For negative incx, iamin returns 0. This behaviour is similar to that of
             // implemented as a reference IAMIN.
             CUBLAS_ERROR_FUNC(func, err, handle, n, x_, incx, int_res_);
+            // Higher level BLAS functions expect CUBLAS_POINTER_MODE_HOST
+            // to be set, therfore we need to reset this to the default value
+            // in order to avoid CUDA_ERROR_ILLEGAL_ADRESS errors
+            cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
         });
     });
     result.template get_access<cl::sycl::access::mode::write>()[0] =
@@ -561,6 +587,10 @@ inline void nrm2(Func func, cl::sycl::queue &queue, int64_t n, cl::sycl::buffer<
             cublasStatus_t err;
             // NRM2 does not support negative index
             CUBLAS_ERROR_FUNC(func, err, handle, n, x_, std::abs(incx), res_);
+            // Higher level BLAS functions expect CUBLAS_POINTER_MODE_HOST
+            // to be set, therfore we need to reset this to the default value
+            // in order to avoid CUDA_ERROR_ILLEGAL_ADRESS errors
+            cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST);
         });
     });
 }