Added an atomic flag to ensure correct order on capture start/end

Author: Przemog1

examples_tests

@@ -21,17 +21,6 @@ class CThreadSafeQueueAdapter final : public IQueue
             : IQueue(originDevice,original->getFamilyIndex(),original->getFlags(),original->getPriority()), originalQueue(original) {}        
         inline CThreadSafeQueueAdapter() : IQueue(nullptr, 0, CREATE_FLAGS::PROTECTED_BIT, 0.f) {}
 
-        inline bool startCapture() override
-        { 
-            std::lock_guard g(m);
-            return originalQueue->startCapture();
-        }
-        inline bool endCapture() override
-        { 
-            std::lock_guard g(m);
-            return originalQueue->endCapture(); 
-        }
-
         virtual bool insertDebugMarker(const char* name, const core::vector4df_SIMD& color=core::vector4df_SIMD(1.0, 1.0, 1.0, 1.0)) override
         {
             std::lock_guard g(m);
@@ -94,7 +83,6 @@ class CThreadSafeQueueAdapter final : public IQueue
             return originalQueue->waitIdle_impl();
         }
 
-
         // used to use unique_ptr here, but it needed `~IQueue` to be public, which requires a custom deleter, etc.
         IQueue* const originalQueue = nullptr;
         mutable std::mutex m;

include/nbl/video/CThreadSafeQueueAdapter.h

@@ -32,14 +32,18 @@ class NBL_API2 CVulkanConnection final : public IAPIConnection
 
     protected:
         explicit inline CVulkanConnection(const VkInstance instance, const SFeatures& enabledFeatures, std::unique_ptr<CVulkanDebugCallback>&& debugCallback, const VkDebugUtilsMessengerEXT vk_debugMessenger)
-            : IAPIConnection(enabledFeatures), m_vkInstance(instance), m_debugCallback(std::move(debugCallback)), m_vkDebugUtilsMessengerEXT(vk_debugMessenger) {}
+            : IAPIConnection(enabledFeatures), m_vkInstance(instance), m_debugCallback(std::move(debugCallback)), m_vkDebugUtilsMessengerEXT(vk_debugMessenger)
+        {
+            flag.clear();
+        }
 
         virtual ~CVulkanConnection();
 
     private:
         const VkInstance m_vkInstance;
         const VkDebugUtilsMessengerEXT m_vkDebugUtilsMessengerEXT;
         const std::unique_ptr<CVulkanDebugCallback> m_debugCallback; // this needs to live longer than VkDebugUtilsMessengerEXT handle above
+        std::atomic_flag flag;
 };
 
 }

include/nbl/video/CVulkanConnection.h

@@ -60,8 +60,6 @@ class IQueue : public core::Interface, public core::Unmovable
 #endif
 
         // for renderdoc and friends
-        virtual bool startCapture() = 0;
-        virtual bool endCapture() = 0;
         virtual bool insertDebugMarker(const char* name, const core::vector4df_SIMD& color = core::vector4df_SIMD(1.0, 1.0, 1.0, 1.0)) = 0;
         virtual bool beginDebugMarker(const char* name, const core::vector4df_SIMD& color = core::vector4df_SIMD(1.0, 1.0, 1.0, 1.0)) = 0;
         virtual bool endDebugMarker() = 0;

include/nbl/video/IQueue.h

@@ -325,6 +325,16 @@ bool CVulkanConnection::startCapture()
 {
     if (!isRunningInRenderdoc())
         return false;
+    if (flag.test())
+    {
+        auto logger = m_debugCallback->getLogger();
+        if(logger)
+            logger->log("Only one capture can be running at a time.", system::ILogger::ELL_ERROR);
+
+        return false;
+    }
+
+    flag.test_and_set();
     m_rdoc_api->StartFrameCapture(RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(m_vkInstance), NULL);
     return true;
 }
@@ -333,7 +343,17 @@ bool CVulkanConnection::endCapture()
 {
     if (!isRunningInRenderdoc())
         return false;
+    if (!flag.test())
+    {
+        auto logger = m_debugCallback->getLogger();
+        if (logger)
+            logger->log("No ongoing caputre to end.", system::ILogger::ELL_ERROR);
+
+        return false;
+    }
+
     m_rdoc_api->EndFrameCapture(RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(m_vkInstance), NULL);
+    flag.clear();
     return true;
 }
 

src/nbl/video/CVulkanConnection.cpp

@@ -34,8 +34,6 @@ class CVulkanQueue final : public IQueue
             return RESULT::OTHER_ERROR;
         }
 
-        bool startCapture() override { return false; }
-        bool endCapture() override { return false; }
         bool insertDebugMarker(const char* name, const core::vector4df_SIMD& color) override;
         bool beginDebugMarker(const char* name, const core::vector4df_SIMD& color) override;
         bool endDebugMarker() override;