Decouples acquire/present from SubmitScheduler

SubmitScheduler doesn't have to know anything about Acquire/Present as
its responsibility to schedule and submit command buffers to a queue.

source/renderer/backend/vulkan/Device.cpp

@@ -626,7 +626,7 @@ std::unique_ptr<CDevice> CDevice::Create(SDL_Window* window)
 	device->m_SamplerManager = std::make_unique<CSamplerManager>(device.get());
 
 	device->m_SubmitScheduler = CSubmitScheduler::Create(
-		device.get(), device->m_GraphicsQueueFamilyIndex, device->m_GraphicsQueue);
+		device.get(), device->m_GraphicsQueue);
 	if (!device->m_SubmitScheduler)
 		return nullptr;
 
@@ -742,7 +742,10 @@ std::unique_ptr<ISwapChain> CDevice::CreateSwapChain(
 	if (window)
 		SDL_Vulkan_GetDrawableSize(window, &surfaceDrawableWidth, &surfaceDrawableHeight);
 	return CSwapChain::Create(
-		this, m_SubmitScheduler.get(), name, m_Surface, surfaceDrawableWidth, surfaceDrawableHeight,
+		this, m_SubmitScheduler.get(),
+		m_GraphicsQueueFamilyIndex, m_GraphicsQueue,
+		name, m_Surface,
+		surfaceDrawableWidth, surfaceDrawableHeight,
 		vsync, std::move(oldSwapChain));
 }
 

source/renderer/backend/vulkan/SubmitScheduler.cpp

@@ -24,7 +24,6 @@
 #include "ps/ConfigDB.h"
 #include "renderer/backend/vulkan/Device.h"
 #include "renderer/backend/vulkan/RingCommandContext.h"
-#include "renderer/backend/vulkan/SwapChain.h"
 #include "renderer/backend/vulkan/Utilities.h"
 
 #include <cstddef>
@@ -40,7 +39,7 @@ namespace Vulkan
 {
 
 std::unique_ptr<CSubmitScheduler> CSubmitScheduler::Create(
-	CDevice* device, const uint32_t queueFamilyIndex, VkQueue queue)
+	CDevice* device, VkQueue queue)
 {
 	std::unique_ptr<CSubmitScheduler> submitScheduler{new CSubmitScheduler{device, queue}};
 
@@ -57,26 +56,6 @@ std::unique_ptr<CSubmitScheduler> CSubmitScheduler::Create(
 			device->GetVkDevice(), &fenceCreateInfo, nullptr, &fence));
 		submitScheduler->m_Fences.push_back({fence, INVALID_SUBMIT_HANDLE});
 	}
-
-	submitScheduler->m_AcquireCommandContext = CRingCommandContext::Create(
-		device, NUMBER_OF_FRAMES_IN_FLIGHT, queueFamilyIndex, *submitScheduler);
-	if (!submitScheduler->m_AcquireCommandContext)
-		return nullptr;
-	submitScheduler->m_PresentCommandContext = CRingCommandContext::Create(
-		device, NUMBER_OF_FRAMES_IN_FLIGHT, queueFamilyIndex, *submitScheduler);
-	if (!submitScheduler->m_PresentCommandContext)
-		return nullptr;
-
-	submitScheduler->m_DebugWaitIdleBeforeAcquire = g_ConfigDB.Get(
-		"renderer.backend.vulkan.debugwaitidlebeforeacquire",
-		submitScheduler->m_DebugWaitIdleBeforeAcquire);
-	submitScheduler->m_DebugWaitIdleBeforePresent = g_ConfigDB.Get(
-		"renderer.backend.vulkan.debugwaitidlebeforepresent",
-		submitScheduler->m_DebugWaitIdleBeforePresent);
-	submitScheduler->m_DebugWaitIdleAfterPresent = g_ConfigDB.Get(
-		"renderer.backend.vulkan.debugwaitidleafterpresent",
-		submitScheduler->m_DebugWaitIdleAfterPresent);
-
 	return submitScheduler;
 }
 
@@ -94,39 +73,6 @@ CSubmitScheduler::~CSubmitScheduler()
 			vkDestroyFence(device, fence.value, nullptr);
 }
 
-bool CSubmitScheduler::AcquireNextImage(CSwapChain& swapChain, VkSemaphore acquireImageSemaphore)
-{
-	if (m_DebugWaitIdleBeforeAcquire)
-		vkDeviceWaitIdle(m_Device->GetVkDevice());
-
-	if (!swapChain.AcquireNextImage())
-		return false;
-	swapChain.SubmitCommandsAfterAcquireNextImage(*m_AcquireCommandContext);
-
-	m_NextWaitSemaphore = acquireImageSemaphore;
-	m_NextWaitDstStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
-	m_AcquireCommandContext->Flush();
-	return true;
-}
-
-CSubmitScheduler::SubmitHandle CSubmitScheduler::Present(CSwapChain& swapChain, VkSemaphore submitDone)
-{
-	swapChain.SubmitCommandsBeforePresent(*m_PresentCommandContext);
-	m_NextSubmitSignalSemaphore = submitDone;
-	m_PresentCommandContext->Flush();
-	const SubmitHandle submitHandle{Flush()};
-
-	if (m_DebugWaitIdleBeforePresent)
-		vkDeviceWaitIdle(m_Device->GetVkDevice());
-
-	swapChain.Present(submitDone, m_Queue);
-
-	if (m_DebugWaitIdleAfterPresent)
-		vkDeviceWaitIdle(m_Device->GetVkDevice());
-
-	return submitHandle;
-}
-
 CSubmitScheduler::SubmitHandle CSubmitScheduler::Submit(VkCommandBuffer commandBuffer)
 {
 	m_SubmittedCommandBuffers.emplace_back(commandBuffer);
@@ -167,31 +113,45 @@ CSubmitScheduler::SubmitHandle CSubmitScheduler::Flush()
 
 	VkSubmitInfo submitInfo{};
 	submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
-	if (m_NextWaitSemaphore != VK_NULL_HANDLE)
+	if (!m_NextWaitSemaphores.empty())
 	{
-		submitInfo.waitSemaphoreCount = 1;
+		ENSURE(m_NextWaitSemaphores.size() == m_NextWaitDstStageMasks.size());
-		submitInfo.pWaitSemaphores = &m_NextWaitSemaphore;
+		submitInfo.waitSemaphoreCount = m_NextWaitSemaphores.size();
-		submitInfo.pWaitDstStageMask = &m_NextWaitDstStageMask;
+		submitInfo.pWaitSemaphores = m_NextWaitSemaphores.data();
+		submitInfo.pWaitDstStageMask = m_NextWaitDstStageMasks.data();
 	}
-	if (m_NextSubmitSignalSemaphore != VK_NULL_HANDLE)
+	if (!m_NextSubmitSignalSemaphores.empty())
 	{
-		submitInfo.signalSemaphoreCount = 1;
+		submitInfo.signalSemaphoreCount = m_NextSubmitSignalSemaphores.size();
-		submitInfo.pSignalSemaphores = &m_NextSubmitSignalSemaphore;
+		submitInfo.pSignalSemaphores = m_NextSubmitSignalSemaphores.data();
 	}
 	submitInfo.commandBufferCount = m_SubmittedCommandBuffers.size();
 	submitInfo.pCommandBuffers = m_SubmittedCommandBuffers.data();
 
 	ENSURE_VK_SUCCESS(vkQueueSubmit(m_Queue, 1, &submitInfo, fence.value));
 
-	m_NextWaitSemaphore = VK_NULL_HANDLE;
+	m_NextWaitSemaphores.clear();
-	m_NextWaitDstStageMask = 0;
+	m_NextWaitDstStageMasks.clear();
-	m_NextSubmitSignalSemaphore = VK_NULL_HANDLE;
+	m_NextSubmitSignalSemaphores.clear();
 
 	m_SubmittedCommandBuffers.clear();
 
 	return fence.lastUsedHandle;
 }
 
+void CSubmitScheduler::EnqueueWaitOnNextSubmit(
+	VkSemaphore semaphore, const VkPipelineStageFlags stageMask)
+{
+	m_NextWaitSemaphores.emplace_back(semaphore);
+	m_NextWaitDstStageMasks.emplace_back(stageMask);
+}
+
+void CSubmitScheduler::EnqueueSignalOnNextSubmit(
+	VkSemaphore semaphore)
+{
+	m_NextSubmitSignalSemaphores.emplace_back(semaphore);
+}
+
 } // namespace Vulkan
 
 } // namespace Backend

source/renderer/backend/vulkan/SubmitScheduler.h

@@ -28,7 +28,6 @@
 #include <vector>
 
 namespace Renderer::Backend::Vulkan { class CRingCommandContext; }
-namespace Renderer::Backend::Vulkan { class CSwapChain; }
 
 namespace Renderer
 {
@@ -50,13 +49,9 @@ public:
 	static constexpr SubmitHandle INVALID_SUBMIT_HANDLE = 0;
 
 	static std::unique_ptr<CSubmitScheduler> Create(
-		CDevice* device, const uint32_t queueFamilyIndex, VkQueue queue);
+		CDevice* device, VkQueue queue);
 	~CSubmitScheduler();
 
-	bool AcquireNextImage(CSwapChain& swapChain, VkSemaphore acquireImageSemaphore);
-
-	SubmitHandle Present(CSwapChain& swapChain, VkSemaphore submitDone);
-
 	SubmitHandle Submit(VkCommandBuffer commandBuffer);
 
 	void WaitUntilFree(const SubmitHandle handle);
@@ -65,6 +60,12 @@ public:
 
 	SubmitHandle Flush();
 
+	/**
+	 * It's a caller responsibility to guarantee a semaphore lifespan.
+	 */
+	void EnqueueWaitOnNextSubmit(VkSemaphore semaphore, const VkPipelineStageFlags stageMask);
+	void EnqueueSignalOnNextSubmit(VkSemaphore semaphore);
+
 private:
 	CSubmitScheduler(CDevice* device, VkQueue queue);
 
@@ -90,18 +91,11 @@ private:
 	};
 	std::queue<SubmittedHandle> m_SubmittedHandles;
 
-	VkSemaphore m_NextWaitSemaphore = VK_NULL_HANDLE;
+	std::vector<VkSemaphore> m_NextWaitSemaphores;
-	VkPipelineStageFlags m_NextWaitDstStageMask = 0;
+	std::vector<VkPipelineStageFlags> m_NextWaitDstStageMasks;
-	VkSemaphore m_NextSubmitSignalSemaphore = VK_NULL_HANDLE;
+	std::vector<VkSemaphore> m_NextSubmitSignalSemaphores;
 
 	std::vector<VkCommandBuffer> m_SubmittedCommandBuffers;
-
-	std::unique_ptr<CRingCommandContext> m_AcquireCommandContext;
-	std::unique_ptr<CRingCommandContext> m_PresentCommandContext;
-
-	bool m_DebugWaitIdleBeforeAcquire = false;
-	bool m_DebugWaitIdleBeforePresent = false;
-	bool m_DebugWaitIdleAfterPresent = false;
 };
 
 } // namespace Vulkan

source/renderer/backend/vulkan/SwapChain.cpp

@@ -54,6 +54,7 @@ namespace Vulkan
 // static
 std::unique_ptr<CSwapChain> CSwapChain::Create(
 	CDevice* device, CSubmitScheduler* submitScheduler,
+	const uint32_t queueFamilyIndex, VkQueue queue,
 	const char* name, VkSurfaceKHR surface,
 	int surfaceDrawableWidth, int surfaceDrawableHeight,
 	const bool vsync, std::unique_ptr<ISwapChain> oldSwapChain)
@@ -200,6 +201,7 @@ std::unique_ptr<CSwapChain> CSwapChain::Create(
 	std::unique_ptr<CSwapChain> swapChain(new CSwapChain());
 	swapChain->m_Device = device;
 	swapChain->m_SubmitScheduler = submitScheduler;
+	swapChain->m_Queue = queue;
 
 	RETURN_NULLPTR_IF_NOT_VK_SUCCESS(vkCreateSwapchainKHR(
 		device->GetVkDevice(), &swapChainCreateInfo, nullptr, &swapChain->m_SwapChain));
@@ -279,6 +281,25 @@ std::unique_ptr<CSwapChain> CSwapChain::Create(
 		swapChain->m_SubmitSemaphores.emplace_back(semaphore);
 	}
 
+	swapChain->m_AcquireCommandContext = CRingCommandContext::Create(
+		device, NUMBER_OF_FRAMES_IN_FLIGHT, queueFamilyIndex, *submitScheduler);
+	if (!swapChain->m_AcquireCommandContext)
+		return nullptr;
+	swapChain->m_PresentCommandContext = CRingCommandContext::Create(
+		device, NUMBER_OF_FRAMES_IN_FLIGHT, queueFamilyIndex, *submitScheduler);
+	if (!swapChain->m_PresentCommandContext)
+		return nullptr;
+
+	swapChain->m_DebugWaitIdleBeforeAcquire = g_ConfigDB.Get(
+		"renderer.backend.vulkan.debugwaitidlebeforeacquire",
+		swapChain->m_DebugWaitIdleBeforeAcquire);
+	swapChain->m_DebugWaitIdleBeforePresent = g_ConfigDB.Get(
+		"renderer.backend.vulkan.debugwaitidlebeforepresent",
+		swapChain->m_DebugWaitIdleBeforePresent);
+	swapChain->m_DebugWaitIdleAfterPresent = g_ConfigDB.Get(
+		"renderer.backend.vulkan.debugwaitidleafterpresent",
+		swapChain->m_DebugWaitIdleAfterPresent);
+
 	swapChain->m_IsValid = true;
 
 	return swapChain;
@@ -331,7 +352,17 @@ bool CSwapChain::AcquireNextBackbuffer()
 		m_SubmitScheduler->WaitUntilFree(frameObject.submitHandle);
 	}
 
-	return m_SubmitScheduler->AcquireNextImage(*this, frameObject.acquireImageSemaphore);
+	if (m_DebugWaitIdleBeforeAcquire)
+		vkDeviceWaitIdle(m_Device->GetVkDevice());
+
+	if (!AcquireNextImage())
+		return false;
+	SubmitCommandsAfterAcquireNextImage(*m_AcquireCommandContext);
+
+	m_SubmitScheduler->EnqueueWaitOnNextSubmit(
+		frameObject.acquireImageSemaphore, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
+	m_AcquireCommandContext->Flush();
+	return true;
 }
 
 void CSwapChain::Present()
@@ -342,7 +373,20 @@ void CSwapChain::Present()
 
 	PROFILE3("Present");
 	FrameObject& frameObject{m_FrameObjects[m_FrameID % m_FrameObjects.size()]};
-	frameObject.submitHandle = m_SubmitScheduler->Present(*this, m_SubmitSemaphores[m_CurrentImageIndex]);
+
+	SubmitCommandsBeforePresent(*m_PresentCommandContext);
+	m_SubmitScheduler->EnqueueSignalOnNextSubmit(m_SubmitSemaphores[m_CurrentImageIndex]);
+	m_PresentCommandContext->Flush();
+	frameObject.submitHandle = m_SubmitScheduler->Flush();
+
+	if (m_DebugWaitIdleBeforePresent)
+		vkDeviceWaitIdle(m_Device->GetVkDevice());
+
+	Present(m_SubmitSemaphores[m_CurrentImageIndex], m_Queue);
+
+	if (m_DebugWaitIdleAfterPresent)
+		vkDeviceWaitIdle(m_Device->GetVkDevice());
+
 	m_Device->OnPresent();
 	++m_FrameID;
 }

source/renderer/backend/vulkan/SwapChain.h

@@ -84,6 +84,7 @@ private:
 
 	static std::unique_ptr<CSwapChain> Create(
 		CDevice* device, CSubmitScheduler* submitScheduler,
+		const uint32_t queueFamilyIndex, VkQueue queue,
 		const char* name, VkSurfaceKHR surface,
 		int surfaceDrawableWidth, int surfaceDrawableHeight,
 		const bool vsync, std::unique_ptr<ISwapChain> oldSwapChain);
@@ -92,6 +93,7 @@ private:
 
 	CDevice* m_Device = nullptr;
 	CSubmitScheduler* m_SubmitScheduler{nullptr};
+	VkQueue m_Queue{VK_NULL_HANDLE};
 
 	bool m_IsValid = false;
 	VkSwapchainKHR m_SwapChain = VK_NULL_HANDLE;
@@ -156,6 +158,13 @@ private:
 	std::vector<SwapChainBackbuffer> m_Backbuffers;
 
 	std::unique_ptr<CTexture> m_BackbufferReadbackTexture;
+
+	std::unique_ptr<CRingCommandContext> m_AcquireCommandContext;
+	std::unique_ptr<CRingCommandContext> m_PresentCommandContext;
+
+	bool m_DebugWaitIdleBeforeAcquire = false;
+	bool m_DebugWaitIdleBeforePresent = false;
+	bool m_DebugWaitIdleAfterPresent = false;
 };
 
 } // namespace Vulkan