Feature: Added completion states on command buffers, so callers know when they finished executing on the GPU

 - Also updated the GPU profiler so individual views can be profiled independently (profiling entire frames was causing issues with queries across multiple command buffers)

Author: BearishSun

Source/Foundation/bsfCore/Profiling/BsProfilerGPU.cpp

@@ -19,9 +19,9 @@ namespace bs
 	{
 		while (!mUnresolvedFrames.empty())
 		{
-			ProfiledSample& frameSample = mUnresolvedFrames.front();
+			ProfiledFrame& frame = mUnresolvedFrames.front();
 
-			freeSample(frameSample);
+			freeFrame(mActiveFrame);
 			mUnresolvedFrames.pop();
 		}
 
@@ -36,11 +36,9 @@ namespace bs
 			return;
 		}
 
-		mFrameSample = ProfiledSample();
-		mFrameSample.name = "Frame";
-		beginSampleInternal(mFrameSample, true);
-
 		mIsFrameActive = true;
+		mActiveFrame.uncategorizedSamples.clear();
+		mActiveFrame.viewSamples.clear();
 	}
 
 	void ProfilerGPU::endFrame(bool discard)
@@ -51,19 +49,61 @@ namespace bs
 			return;
 		}
 
-		if (!mIsFrameActive)
+		if (mIsViewActive)
+		{
+			BS_LOG(Error, Profiler, "Attempting to end a frame while a view is active.");
 			return;
+		}
 
-		endSampleInternal(mFrameSample);
+		if (!mIsFrameActive)
+			return;
 
 		if(!discard)
-			mUnresolvedFrames.push(mFrameSample);
+			mUnresolvedFrames.push(mActiveFrame);
 		else
-			freeSample(mFrameSample);
+			freeFrame(mActiveFrame);
 
 		mIsFrameActive = false;
 	}
 
+	void ProfilerGPU::beginView(UINT64 id, ProfilerString title)
+	{
+		if (!mIsFrameActive)
+		{
+			BS_LOG(Error, Profiler, "Cannot begin a view because no frame is active.");
+			return;
+		}
+
+		if (mIsViewActive)
+		{
+			BS_LOG(Error, Profiler, "Cannot begin a view because another view is active.");
+			return;
+		}
+
+		auto sample = mViewSamplePool.construct<ProfiledViewSample>();
+		sample->viewId = id;
+
+		mActiveFrame.viewSamples.push_back(sample);
+
+		beginSampleInternal(*sample, true);
+		mIsViewActive = true;
+	}
+
+	void ProfilerGPU::endView()
+	{
+		if (!mActiveSamples.empty())
+		{
+			BS_LOG(Error, Profiler, "Attempting to end a view while a sample is active.");
+			return;
+		}
+
+		if (!mIsViewActive)
+			return;
+
+		endSampleInternal(*mActiveFrame.viewSamples.back());
+		mIsViewActive = false;
+	}
+
 	void ProfilerGPU::beginSample(ProfilerString name)
 	{
 		if (!mIsFrameActive)
@@ -76,8 +116,13 @@ namespace bs
 		sample->name = std::move(name);
 		beginSampleInternal(*sample, false);
 
-		if(mActiveSamples.empty())
-			mFrameSample.children.push_back(sample);
+		if (mActiveSamples.empty())
+		{
+			if (mIsViewActive)
+				mActiveFrame.viewSamples.back()->children.push_back(sample);
+			else
+				mActiveFrame.uncategorizedSamples.push_back(sample);
+		}
 		else
 		{
 			ProfiledSample* parent = mActiveSamples.top();
@@ -133,29 +178,53 @@ namespace bs
 	{
 		while (!mUnresolvedFrames.empty())
 		{
-			ProfiledSample& frameSample = mUnresolvedFrames.front();
+			ProfiledFrame& frame = mUnresolvedFrames.front();
 
-			// Frame sample timer query is the last query we issued
-			// so if it is complete, we may assume all queries are complete.
-			if (frameSample.activeTimeQuery->isReady())
+			// Make sure all the top-level queries have finished. If they have that implies
+			// all their children have finished as well
+			bool isReady = true;
+			for(auto& entry : frame.viewSamples)
 			{
-				GPUProfilerReport report;
-				resolveSample(frameSample, report.frameSample);
-
-				freeSample(frameSample);
-				mUnresolvedFrames.pop();
+				if (!entry->activeTimeQuery->isReady())
+				{
+					isReady = false;
+					break;
+				}
+			}
 
+			for(auto& entry : frame.uncategorizedSamples)
+			{
+				if (!entry->activeTimeQuery->isReady())
 				{
-					Lock lock(mMutex);
-					mReadyReports[(mReportHeadPos + mReportCount) % MAX_QUEUE_ELEMENTS] = report;
-					if (mReportCount == MAX_QUEUE_ELEMENTS)
-						mReportHeadPos = (mReportHeadPos + 1) % MAX_QUEUE_ELEMENTS;
-					else
-						mReportCount++;
+					isReady = false;
+					break;
 				}
 			}
-			else
+
+			if (!isReady)
 				break;
+			
+			GPUProfilerReport report;
+			report.viewSamples.resize(frame.viewSamples.size());
+			report.uncategorizedSamples.resize(frame.uncategorizedSamples.size());
+			
+			for (size_t i = 0; i < frame.viewSamples.size(); i++)
+				resolveSample(*frame.viewSamples[i], report.viewSamples[i]);
+				
+			for (size_t i = 0; i < frame.uncategorizedSamples.size(); i++)
+				resolveSample(*frame.uncategorizedSamples[i], report.uncategorizedSamples[i]);
+				
+			freeFrame(mActiveFrame);
+			mUnresolvedFrames.pop();
+
+			{
+				Lock lock(mMutex);
+				mReadyReports[(mReportHeadPos + mReportCount) % MAX_QUEUE_ELEMENTS] = report;
+				if (mReportCount == MAX_QUEUE_ELEMENTS)
+					mReportHeadPos = (mReportHeadPos + 1) % MAX_QUEUE_ELEMENTS;
+				else
+					mReportCount++;
+			}
 		}
 	}
 
@@ -175,6 +244,24 @@ namespace bs
 			mFreeOcclusionQueries.push(sample.activeOcclusionQuery);
 	}
 
+	void ProfilerGPU::freeFrame(ProfiledFrame& frame)
+	{
+		for (size_t i = 0; i < frame.viewSamples.size(); i++)
+		{
+			freeSample(*frame.viewSamples[i]);
+			mViewSamplePool.destruct(frame.viewSamples[i]);
+		}
+			
+		for (size_t i = 0; i < frame.uncategorizedSamples.size(); i++)
+		{
+			freeSample(*frame.uncategorizedSamples[i]);
+			mSamplePool.destruct(frame.uncategorizedSamples[i]);
+		}
+
+		frame.viewSamples.clear();
+		frame.uncategorizedSamples.clear();
+	}
+
 	void ProfilerGPU::resolveSample(const ProfiledSample& sample, GPUProfileSample& reportSample)
 	{
 		reportSample.name.assign(sample.name.data(), sample.name.size());

Source/Foundation/bsfCore/Profiling/BsProfilerGPU.h

@@ -43,10 +43,17 @@ namespace bs
 		Vector<GPUProfileSample> children;
 	};
 
+	/** Contains various profiler statistics for a particular view. */
+	struct GPUProfileViewSample : GPUProfileSample
+	{
+		UINT64 viewId;
+	};
+
 	/** Profiler report containing information about GPU sampling data from a single frame. */
 	struct GPUProfilerReport
 	{
-		GPUProfileSample frameSample; /**< Sample containing data for entire frame. */
+		Vector<GPUProfileViewSample> viewSamples; /**< Profiler samples belonging to a particular view. */
+		Vector<GPUProfileSample> uncategorizedSamples; /**< Profiler samples not grouped under a particular view. */
 	};
 
 	/**
@@ -68,6 +75,17 @@ namespace bs
 			Vector<ProfiledSample*> children;
 		};
 
+		struct ProfiledViewSample : ProfiledSample
+		{
+			UINT64 viewId;
+		};
+
+		struct ProfiledFrame
+		{
+			Vector<ProfiledViewSample*> viewSamples;
+			Vector<ProfiledSample*> uncategorizedSamples;
+		};
+
 	public:
 		ProfilerGPU();
 		~ProfilerGPU();
@@ -87,6 +105,19 @@ namespace bs
 		 */
 		void endFrame(bool discard = false);
 
+		/**
+		 * Signals that all following sample calls are used for rendering a particular view represented with the provided
+		 * id. A top-level timing and occlusion query is issued for the entire view and all following samples will
+		 * be grouped under the view in the output report. Must be followed by endView() when done sampling.
+		 *
+		 * @param[in]	id			Identifier that can be used to uniquely identify the view.
+		 * @param[in]	title		Title describing the view.
+		 */
+		void beginView(UINT64 id, ProfilerString title);
+
+		/** Signals the end of rendering for a particular view. Must match the corresponding beginView() call. */
+		void endView();
+
 		/**
 		 * Begins sample measurement. Must be followed by endSample().
 		 *
@@ -155,21 +186,26 @@ namespace bs
 		/** Frees the memory used by all the child samples. */
 		void freeSample(ProfiledSample& sample);
 
+		/** Frees the memory used by all the samples in the frame. */
+		void freeFrame(ProfiledFrame& frame);
+
 		/** Resolves an active sample and converts it to report sample. */
 		void resolveSample(const ProfiledSample& sample, GPUProfileSample& reportSample);
 
 	private:
-		ProfiledSample mFrameSample;
 		bool mIsFrameActive = false;
+		bool mIsViewActive = false;
 		Stack<ProfiledSample*> mActiveSamples;
+		ProfiledFrame mActiveFrame;
 
-		Queue<ProfiledSample> mUnresolvedFrames;
+		Queue<ProfiledFrame> mUnresolvedFrames;
 		GPUProfilerReport* mReadyReports = nullptr;
 
 		static const UINT32 MAX_QUEUE_ELEMENTS;
 		UINT32 mReportHeadPos = 0;
 		UINT32 mReportCount = 0;
 
+		PoolAlloc<sizeof(ProfiledViewSample), 16> mViewSamplePool;
 		PoolAlloc<sizeof(ProfiledSample), 256> mSamplePool;
 
 		mutable Stack<SPtr<ct::TimerQuery>> mFreeTimerQueries;

Source/Foundation/bsfCore/RenderAPI/BsCommandBuffer.h

@@ -36,6 +36,25 @@ namespace bs { namespace ct
 		UINT32 mMask = 0;
 	};
 
+	/** Possible states that a CommandBuffer can be in. */
+	enum class CommandBufferState
+	{
+		/** Command buffer doesn't have any commands recorded, nor has it been queued for execution. */
+		Empty,
+
+		/** Command buffer has one or multiple commands recorded, but they haven't been queued for execution. */
+		Recording,
+
+		/**
+		 * Command buffer has been queued for execution, but still hasn't finished executing. Buffer that is
+		 * executing cannot be modified or re-submitted for execution until done executing.
+		 */
+		Executing,
+
+		/** Command buffer has been queued for execution and has finished executing. */
+		Done
+	};
+
 	/**
 	 * Contains a list of render API commands that can be queued for execution on the GPU. User is allowed to populate the
 	 * command buffer from any thread, ensuring render API command generation can be multi-threaded. Command buffers
@@ -79,6 +98,15 @@ namespace bs { namespace ct
 		/** Returns the device index this buffer will execute on. */
 		UINT32 getDeviceIdx() const { return mDeviceIdx; }
 
+		/** Returns the current state of the command buffer. */
+		virtual CommandBufferState getState() const = 0;
+
+		/**
+		 * Resets the command buffer back into initial state. Must only be used if the command buffer is
+		 * not in the executing state.
+		 */
+		virtual void reset() = 0;
+
 	protected:
 		CommandBuffer(GpuQueueType type, UINT32 deviceIdx, UINT32 queueIdx, bool secondary);
 

Source/Foundation/bsfEngine/GUI/BsProfilerOverlay.cpp

@@ -538,7 +538,11 @@ namespace bs
 
 		if (ProfilerGPU::instance().getNumAvailableReports() > 0)
 		{
-			updateGPUSampleContents(ProfilerGPU::instance().getNextReport());
+			GPUProfilerReport report = ProfilerGPU::instance().getNextReport();
+
+			// TODO - Currently displaying just the first view. I need to add a way to toggle between views
+			if(!report.viewSamples.empty())
+				updateGPUSampleContents(report.viewSamples[0]);
 		}
 	}
 
@@ -705,26 +709,26 @@ namespace bs
 		}
 	}
 
-	void ProfilerOverlay::updateGPUSampleContents(const GPUProfilerReport& gpuReport)
+	void ProfilerOverlay::updateGPUSampleContents(const GPUProfileSample& frameSample)
 	{
 		mGPUFrameNumStr.setParameter(0, toString((UINT64)gTime().getFrameIdx()));
-		mGPUTimeStr.setParameter(0, toString(gpuReport.frameSample.timeMs));
-		mGPUDrawCallsStr.setParameter(0, toString(gpuReport.frameSample.numDrawCalls));
-		mGPURenTargetChangesStr.setParameter(0, toString(gpuReport.frameSample.numRenderTargetChanges));
-		mGPUPresentsStr.setParameter(0, toString(gpuReport.frameSample.numPresents));
-		mGPUClearsStr.setParameter(0, toString(gpuReport.frameSample.numClears));
-		mGPUVerticesStr.setParameter(0, toString(gpuReport.frameSample.numVertices));
-		mGPUPrimitivesStr.setParameter(0, toString(gpuReport.frameSample.numPrimitives));
-		mGPUSamplesStr.setParameter(0, toString(gpuReport.frameSample.numDrawnSamples));
-		mGPUPipelineStateChangesStr.setParameter(0, toString(gpuReport.frameSample.numPipelineStateChanges));
-
-		mGPUObjectsCreatedStr.setParameter(0, toString(gpuReport.frameSample.numObjectsCreated));
-		mGPUObjectsDestroyedStr.setParameter(0, toString(gpuReport.frameSample.numObjectsDestroyed));
-		mGPUResourceWritesStr.setParameter(0, toString(gpuReport.frameSample.numResourceWrites));
-		mGPUResourceReadsStr.setParameter(0, toString(gpuReport.frameSample.numResourceReads));
-		mGPUParamBindsStr.setParameter(0, toString(gpuReport.frameSample.numGpuParamBinds));
-		mGPUVertexBufferBindsStr.setParameter(0, toString(gpuReport.frameSample.numVertexBufferBinds));
-		mGPUIndexBufferBindsStr.setParameter(0, toString(gpuReport.frameSample.numIndexBufferBinds));
+		mGPUTimeStr.setParameter(0, toString(frameSample.timeMs));
+		mGPUDrawCallsStr.setParameter(0, toString(frameSample.numDrawCalls));
+		mGPURenTargetChangesStr.setParameter(0, toString(frameSample.numRenderTargetChanges));
+		mGPUPresentsStr.setParameter(0, toString(frameSample.numPresents));
+		mGPUClearsStr.setParameter(0, toString(frameSample.numClears));
+		mGPUVerticesStr.setParameter(0, toString(frameSample.numVertices));
+		mGPUPrimitivesStr.setParameter(0, toString(frameSample.numPrimitives));
+		mGPUSamplesStr.setParameter(0, toString(frameSample.numDrawnSamples));
+		mGPUPipelineStateChangesStr.setParameter(0, toString(frameSample.numPipelineStateChanges));
+
+		mGPUObjectsCreatedStr.setParameter(0, toString(frameSample.numObjectsCreated));
+		mGPUObjectsDestroyedStr.setParameter(0, toString(frameSample.numObjectsDestroyed));
+		mGPUResourceWritesStr.setParameter(0, toString(frameSample.numResourceWrites));
+		mGPUResourceReadsStr.setParameter(0, toString(frameSample.numResourceReads));
+		mGPUParamBindsStr.setParameter(0, toString(frameSample.numGpuParamBinds));
+		mGPUVertexBufferBindsStr.setParameter(0, toString(frameSample.numVertexBufferBinds));
+		mGPUIndexBufferBindsStr.setParameter(0, toString(frameSample.numIndexBufferBinds));
 
 		mGPUFrameNumLbl->setContent(mGPUFrameNumStr);
 		mGPUTimeLbl->setContent(mGPUTimeStr);
@@ -766,7 +770,7 @@ namespace bs
 		UINT32 currentCount = 0;
 
 		Stack<Todo> todo;
-		todo.push(Todo(gpuReport.frameSample, 0));
+		todo.push(Todo(frameSample, 0));
 
 		while (!todo.empty())
 		{