Feature: Track auto-exposure progress on the CPU so we can determine how long to keep redrawing an on-demand view for

Author: BearishSun

Source/Plugins/bsfRenderBeast/BsRenderBeast.cpp

@@ -486,6 +486,8 @@ namespace bs { namespace ct
 		for (UINT32 i = 0; i < numViews; i++)
 		{
 			RendererView* view = viewGroup.getView(i);
+			view->updateAsyncOperations();
+			
 			auto viewId = (UINT64)view;
 			const RendererViewTargetData& viewTarget = view->getProperties().target;
 			String title = StringUtil::format("({0} x {1})", viewTarget.targetWidth, viewTarget.targetHeight);

Source/Plugins/bsfRenderBeast/BsRenderBeast.h

@@ -21,23 +21,14 @@ namespace bs
 	 *  @{
 	 */
 
-	/** Information about current time and frame index. */
-	struct FrameTimings
-	{
-		float time;
-		float timeDelta;
-		UINT64 frameIdx;
-	};
-
 	/** Contains information global to an entire frame. */
 	struct FrameInfo
 	{
 		FrameInfo(const FrameTimings& timings, PerFrameData perFrameData)
-			:timeDelta(timings.timeDelta), frameIdx(timings.frameIdx), perFrameData(perFrameData)
+			:timings(timings), perFrameData(perFrameData)
 		{ }
 
-		float timeDelta;
-		UINT64 frameIdx;
+		FrameTimings timings;
 		PerFrameData perFrameData;
 	};
 

Source/Plugins/bsfRenderBeast/BsRenderBeastPrerequisites.h

@@ -55,6 +55,14 @@ namespace bs { namespace ct
 		Full,
 	};
 
+	/** Information about current time and frame index. */
+	struct FrameTimings
+	{
+		float time = 0.0f;
+		float timeDelta = 0.0f;
+		UINT64 frameIdx = 0;
+	};
+
 	/** @} */
 
 	struct RenderBeastOptions;

Source/Plugins/bsfRenderBeast/BsRenderCompositor.cpp

@@ -603,7 +603,7 @@ namespace bs { namespace ct
 			gbuffer.depth = sceneDepthNode->depthTex->texture;
 
 			GpuParticleSimulation::instance().simulate(inputs.scene, inputs.frameInfo.perFrameData.particles,
-				inputs.view.getPerViewBuffer(), gbuffer, inputs.frameInfo.timeDelta);
+				inputs.view.getPerViewBuffer(), gbuffer, inputs.frameInfo.timings.timeDelta);
 		}
 
 		GpuParticleSimulation::instance().sort(inputs.view);
@@ -1742,7 +1742,7 @@ namespace bs { namespace ct
 				eyeAdaptationMat->execute(
 					reducedHistogram->texture,
 					output->renderTexture,
-					inputs.frameInfo.timeDelta,
+					inputs.frameInfo.timings.timeDelta,
 					settings.autoExposure,
 					settings.exposureScale);
 			}
@@ -1756,7 +1756,7 @@ namespace bs { namespace ct
 				setupMat->execute(
 					downsampledScene->texture,
 					luminanceTex->renderTexture,
-					inputs.frameInfo.timeDelta,
+					inputs.frameInfo.timings.timeDelta,
 					settings.autoExposure,
 					settings.exposureScale);
 
@@ -1786,10 +1786,16 @@ namespace bs { namespace ct
 					downsampleInput,
 					prevFrameEyeAdaptation,
 					output->renderTexture,
-					inputs.frameInfo.timeDelta,
+					inputs.frameInfo.timings.timeDelta,
 					settings.autoExposure,
 					settings.exposureScale);
 			}
+
+			const RendererView& view = inputs.view;
+
+			// Notify the view eye adaptation value will change
+			SPtr<CommandBuffer> cb = RenderAPI::instance().getMainCommandBuffer();
+			view._notifyLuminanceUpdated(inputs.frameInfo.timings.frameIdx, cb, output);
 		}
 		else
 		{

Source/Plugins/bsfRenderBeast/BsRendererView.cpp

@@ -12,6 +12,7 @@
 #include "BsRenderBeast.h"
 #include "BsRendererDecal.h"
 #include "Animation/BsAnimationManager.h"
+#include "RenderAPI/BsCommandBuffer.h"
 
 namespace bs { namespace ct
 {
@@ -164,8 +165,18 @@ namespace bs { namespace ct
 
 		gPerCameraParamDef.gNDCToPrevNDC.set(mParamBuffer, NDCToPrevNDC);
 
-		mFrameDelta = frameInfo.timeDelta;
+		mFrameTimings = frameInfo.timings;
 		mAsyncAnim = frameInfo.perFrameData.animation ? frameInfo.perFrameData.animation->async : false;
+
+		// Account for auto-exposure taking multiple frames
+		if (mRedrawThisFrame)
+		{
+			// Note: Doing this here instead of _notifyNeedsRedraw because we need an up-to-date frame index
+			if (mRenderSettings->enableHDR && mRenderSettings->enableAutoExposure)
+				mWaitingOnAutoExposureFrame = mFrameTimings.frameIdx;
+			else
+				mWaitingOnAutoExposureFrame = std::numeric_limits<UINT64>::max();
+		}
 	}
 
 	void RendererView::endFrame()
@@ -186,36 +197,92 @@ namespace bs { namespace ct
 			mRedrawForFrames--;
 
 		if (mRedrawForSeconds > 0.0f)
-			mRedrawForSeconds -= mFrameDelta;
+			mRedrawForSeconds -= mFrameTimings.timeDelta;
+
+		mRedrawThisFrame = false;
 	}
 
 	void RendererView::_notifyNeedsRedraw()
 	{
+		mRedrawThisFrame = true;
+		
 		// If doing async animation there is a one frame delay
 		mRedrawForFrames = mAsyncAnim ? 2 : 1;
 
-		// Account for auto-exposure taking multiple frames
-		if (mRenderSettings->enableAutoExposure)
-		{
-			AutoExposureSettings& aeSettings = mRenderSettings->autoExposure;
-			float maxSpeed = std::max(aeSettings.eyeAdaptationSpeedUp, aeSettings.eyeAdaptationSpeedDown);
-
-			// TODO - We should get feedback from the renderer on when the exposure stabilises, as this number will
-			// be too high for a large majority of cases
-			mRedrawForSeconds = maxSpeed;
-		}
-		else
-			mRedrawForSeconds = 0.0f;
+		// This will be set once we get the new luminance data from the GPU
+		mRedrawForSeconds = 0.0f;
 	}
 
 	bool RendererView::shouldDraw() const
 	{
 		if (!mProperties.onDemand)
 			return true;
 
+		if(mRenderSettings->enableHDR && mRenderSettings->enableAutoExposure)
+		{
+			constexpr float AUTO_EXPOSURE_TOLERANCE = 0.01f;
+			
+			// The view was redrawn but we still haven't received the eye adaptation results from the GPU, so
+			// we keep redrawing until we do
+			if (mWaitingOnAutoExposureFrame != std::numeric_limits<UINT64>::max())
+				return true;
+			
+			// Need to render until the auto-exposure reaches the target exposure
+			float eyeAdaptationDiff = Math::abs(mCurrentEyeAdaptation - mPreviousEyeAdaptation);
+			if (eyeAdaptationDiff > AUTO_EXPOSURE_TOLERANCE)
+				return true;
+		}
+
 		return mRedrawForFrames > 0 || mRedrawForSeconds > 0.0f;
 	}
 
+	void RendererView::updateAsyncOperations()
+	{
+		// Find most recent available frame
+		auto lastFinishedIter = mLuminanceUpdates.end();
+		for(auto iter = mLuminanceUpdates.begin(); iter != mLuminanceUpdates.end(); ++iter)
+		{
+			if (iter->commandBuffer->getState() == CommandBufferState::Executing)
+				break;
+
+			lastFinishedIter = iter;
+		}
+
+		if (lastFinishedIter != mLuminanceUpdates.end())
+		{
+			// Get new luminance value
+			mPreviousEyeAdaptation = mCurrentEyeAdaptation;
+
+			PixelData data = lastFinishedIter->outputTexture->texture->lock(GBL_READ_ONLY);
+			mCurrentEyeAdaptation = data.getColorAt(0, 0).r;
+			lastFinishedIter->outputTexture->texture->unlock();
+
+			// We've received information about eye adaptation, use that to determine if redrawing
+			// is required (technically we're drawing a few frames extra, as this information is always
+			// a few frames too late).
+			if (lastFinishedIter->frameIdx == mWaitingOnAutoExposureFrame)
+				mWaitingOnAutoExposureFrame = std::numeric_limits<UINT64>::max();
+
+			mLuminanceUpdates.erase(mLuminanceUpdates.begin(), lastFinishedIter + 1);
+		}
+	}
+	
+	RendererViewRedrawReason RendererView::getRedrawReason() const
+	{
+		if (!mProperties.onDemand)
+			return RendererViewRedrawReason::PerFrame;
+
+		if (mRedrawThisFrame)
+			return RendererViewRedrawReason::OnDemandThisFrame;
+
+		return RendererViewRedrawReason::OnDemandLingering;
+	}
+
+	void RendererView::_notifyLuminanceUpdated(UINT64 frameIdx, SPtr<CommandBuffer> cb, SPtr<PooledRenderTexture> texture) const
+	{
+		mLuminanceUpdates.emplace_back(frameIdx, cb, texture);
+	}
+
 	void RendererView::determineVisible(const Vector<RendererRenderable*>& renderables, const Vector<CullInfo>& cullInfos,
 		Vector<bool>* visibility)
 	{

Source/Plugins/bsfRenderBeast/BsRendererView.h

@@ -10,7 +10,6 @@
 #include "Math/BsConvexVolume.h"
 #include "Shading/BsLightGrid.h"
 #include "Shading/BsShadowRendering.h"
-#include "BsRendererView.h"
 #include "BsRendererRenderable.h"
 #include "BsRenderCompositor.h"
 #include "BsRendererParticles.h"
@@ -221,6 +220,22 @@ namespace bs { namespace ct
 		Vector<Camera*> cameras;
 	};
 
+	/** Returns the reason why is a RendererView being redrawn. */
+	enum class RendererViewRedrawReason
+	{
+		/** This particular view isn't on-demand and is redrawn every frame. */
+		PerFrame,
+
+		/** Draws on demand and on-demand drawing was triggered this frame. */
+		OnDemandThisFrame,
+
+		/**
+		 * Draws on demand and on-demand drawing was triggered during an earlier frame but a multi-frame effect is
+		 * requiring the view to get redrawn in later frames.
+		 */
+		OnDemandLingering
+	};
+
 	/** Contains information about a single view into the scene, used by the renderer. */
 	class RendererView
 	{
@@ -275,7 +290,7 @@ namespace bs { namespace ct
 
 		/** Returns the compositor in charge of rendering for this view. */
 		const RenderCompositor& getCompositor() const { return mCompositor; }
-
+		
 		/**
 		 * Populates view render queues by determining visible renderable objects.
 		 *
@@ -417,6 +432,18 @@ namespace bs { namespace ct
 		/** Determines if view's 3D geometry should be rendered this frame. */
 		bool shouldDraw3D() const { return !mRenderSettings->overlayOnly && shouldDraw(); }
 
+		/**
+		 * Determines if any async operations have completed executing and finalizes them. Should be called once
+		 * per frame.
+		 */
+		void updateAsyncOperations();
+
+		/**
+		 * Returns the reason that explains why is the view getting redrawn this frame. Only relevant if shouldDraw() returned
+		 * true previously during the frame.
+		 */
+		RendererViewRedrawReason getRedrawReason() const;
+
 		/** Assigns a view index to the view. To be called by the parent view group when the view is added to it. */
 		void _setViewIdx(UINT32 viewIdx) { mViewIdx = viewIdx; }
 
@@ -431,6 +458,12 @@ namespace bs { namespace ct
 		 */
 		void _notifyCompositorTargetChanged(const SPtr<RenderTarget>& target) const { mContext.currentTarget = target; }
 
+		/**
+		 * Notifies the view that a new average luminance is being calculated on the provided command buffer. The results
+		 * will be read from the provided texture when the command buffer finishes executing.
+		 */
+		void _notifyLuminanceUpdated(UINT64 frameIdx, SPtr<CommandBuffer> cb, SPtr<PooledRenderTexture> texture) const;
+		
 		/**
 		 * Extracts the necessary values from the projection matrix that allow you to transform device Z value (range [0, 1]
 		 * into view Z value.
@@ -457,6 +490,17 @@ namespace bs { namespace ct
 		 */
 		static Vector2 getNDCZToDeviceZ();
 	private:
+		struct LuminanceUpdate
+		{
+			LuminanceUpdate(UINT64 frameIdx, SPtr<CommandBuffer> commandBuffer, const SPtr<PooledRenderTexture>& outputTexture)
+				: frameIdx(frameIdx), commandBuffer(std::move(commandBuffer)), outputTexture(std::move(outputTexture))
+			{ }
+
+			UINT64 frameIdx;
+			SPtr<CommandBuffer> commandBuffer;
+			SPtr<PooledRenderTexture> outputTexture;
+		};
+		
 		RendererViewProperties mProperties;
 		mutable RendererViewContext mContext;
 		Camera* mCamera;
@@ -476,11 +520,18 @@ namespace bs { namespace ct
 		UINT32 mViewIdx;
 
 		// On-demand drawing
+		bool mRedrawThisFrame = false;
 		float mRedrawForSeconds = 0.0f;
 		UINT32 mRedrawForFrames = 0;
+		UINT64 mWaitingOnAutoExposureFrame = std::numeric_limits<UINT64>::max();
+		mutable Vector<LuminanceUpdate> mLuminanceUpdates;
+
+		// Exposure
+		float mPreviousEyeAdaptation = 0.0f;
+		float mCurrentEyeAdaptation = 0.0f;
 
 		// Current frame info
-		float mFrameDelta = 0.0f;
+		FrameTimings mFrameTimings;
 		bool mAsyncAnim = false;
 	};