Re-add optional OIDN denoise as an external dynamic library.

bruvzg · bruvzg · commit 37aabd0c1b3a · 2023-10-05T15:30:38.000+03:00
diff --git a/doc/classes/EditorSettings.xml b/doc/classes/EditorSettings.xml
@@ -500,6 +500,9 @@
 			If [code]true[/code], when saving a file, the editor will rename the old file to a different name, save a new file, then only remove the old file once the new file has been saved. This makes loss of data less likely to happen if the editor or operating system exits unexpectedly while saving (e.g. due to a crash or power outage).
 			[b]Note:[/b] On Windows, this feature can interact negatively with certain antivirus programs. In this case, you may have to set this to [code]false[/code] to prevent file locking issues.
 		</member>
+		<member name="filesystem/tools/oidn/oidn_library_path" type="String" setter="" getter="">
+			The path to the directory containing the OIDN denoise dynamic library.
+		</member>
 		<member name="interface/editor/accept_dialog_cancel_ok_buttons" type="int" setter="" getter="">
 			How to position the Cancel and OK buttons in the editor's [AcceptDialog]s. Different platforms have different standard behaviors for this, which can be overridden using this setting. This is useful if you use Godot both on Windows and macOS/Linux and your Godot muscle memory is stronger than your OS specific one.
 			- [b]Auto[/b] follows the platform convention: Cancel first on macOS and Linux, OK first on Windows.
diff --git a/editor/editor_settings.cpp b/editor/editor_settings.cpp
@@ -519,6 +519,9 @@ void EditorSettings::_load_defaults(Ref<ConfigFile> p_extra_config) {
 	EDITOR_SETTING_USAGE(Variant::FLOAT, PROPERTY_HINT_RANGE, "filesystem/import/blender/rpc_server_uptime", 5, "0,300,1,or_greater,suffix:s", PROPERTY_USAGE_DEFAULT | PROPERTY_USAGE_RESTART_IF_CHANGED)
 	EDITOR_SETTING_USAGE(Variant::STRING, PROPERTY_HINT_GLOBAL_FILE, "filesystem/import/fbx/fbx2gltf_path", "", "", PROPERTY_USAGE_DEFAULT | PROPERTY_USAGE_RESTART_IF_CHANGED)
 
+	// Tools (denoise)
+	EDITOR_SETTING_USAGE(Variant::STRING, PROPERTY_HINT_GLOBAL_DIR, "filesystem/tools/oidn/oidn_library_path", "", "", PROPERTY_USAGE_DEFAULT)
+
 	/* Docks */
 
 	// SceneTree
diff --git a/modules/lightmapper_rd/lightmapper_rd.cpp b/modules/lightmapper_rd/lightmapper_rd.cpp
@@ -36,6 +36,7 @@
 
 #include "core/config/project_settings.h"
 #include "core/math/geometry_2d.h"
+#include "editor/editor_settings.h"
 #include "servers/rendering/rendering_device_binds.h"
 
 //uncomment this if you want to see textures from all the process saved
@@ -671,6 +672,118 @@ LightmapperRD::BakeError LightmapperRD::_dilate(RenderingDevice *rd, Ref<RDShade
 	return BAKE_OK;
 }
 
+bool LightmapperRD::_load_oidn(const String &p_library_path) {
+	if (oidn_lib_path == p_library_path) {
+		return oidn_lib_handle != nullptr;
+	}
+	oidn_lib_path = p_library_path;
+
+	String lib_path;
+	if (OS::get_singleton()->get_name() == "macOS") {
+		lib_path = oidn_lib_path.path_join("libOpenImageDenoise.dylib");
+	} else if (OS::get_singleton()->get_name() == "Windows") {
+		lib_path = oidn_lib_path.path_join("OpenImageDenoise.dll");
+	} else {
+		lib_path = oidn_lib_path.path_join("libOpenImageDenoise.so");
+	}
+
+	_unload_oidn();
+
+	if (OS::get_singleton()->open_dynamic_library(lib_path, oidn_lib_handle, true) != OK) {
+		oidn_lib_handle = nullptr;
+		ERR_PRINT(vformat("Failed to load %s.", lib_path));
+		return false;
+	}
+	bool symbols_ok = true;
+	symbols_ok = symbols_ok && (OS::get_singleton()->get_dynamic_library_symbol_handle(oidn_lib_handle, "oidnNewDevice", (void *&)oidnNewDevice) == OK);
+	symbols_ok = symbols_ok && (OS::get_singleton()->get_dynamic_library_symbol_handle(oidn_lib_handle, "oidnCommitDevice", (void *&)oidnCommitDevice) == OK);
+	symbols_ok = symbols_ok && (OS::get_singleton()->get_dynamic_library_symbol_handle(oidn_lib_handle, "oidnNewFilter", (void *&)oidnNewFilter) == OK);
+	symbols_ok = symbols_ok && (OS::get_singleton()->get_dynamic_library_symbol_handle(oidn_lib_handle, "oidnSetSharedFilterImage", (void *&)oidnSetSharedFilterImage) == OK);
+	symbols_ok = symbols_ok && (OS::get_singleton()->get_dynamic_library_symbol_handle(oidn_lib_handle, "oidnSetFilterBool", (void *&)oidnSetFilterBool) == OK);
+	symbols_ok = symbols_ok && (OS::get_singleton()->get_dynamic_library_symbol_handle(oidn_lib_handle, "oidnCommitFilter", (void *&)oidnCommitFilter) == OK);
+	symbols_ok = symbols_ok && (OS::get_singleton()->get_dynamic_library_symbol_handle(oidn_lib_handle, "oidnExecuteFilter", (void *&)oidnExecuteFilter) == OK);
+	symbols_ok = symbols_ok && (OS::get_singleton()->get_dynamic_library_symbol_handle(oidn_lib_handle, "oidnGetDeviceError", (void *&)oidnGetDeviceError) == OK);
+	symbols_ok = symbols_ok && (OS::get_singleton()->get_dynamic_library_symbol_handle(oidn_lib_handle, "oidnReleaseFilter", (void *&)oidnReleaseFilter) == OK);
+	symbols_ok = symbols_ok && (OS::get_singleton()->get_dynamic_library_symbol_handle(oidn_lib_handle, "oidnReleaseDevice", (void *&)oidnReleaseDevice) == OK);
+	if (!symbols_ok) {
+		ERR_PRINT("Failed to load OIDN symbols.");
+		_unload_oidn();
+		return false;
+	}
+
+	oidn_device = oidnNewDevice(OIDN_DEVICE_TYPE_CPU);
+	oidnCommitDevice(oidn_device);
+	const char *msg = nullptr;
+	if (oidnGetDeviceError(oidn_device, &msg) != OIDN_ERROR_NONE) {
+		ERR_PRINT("Failed to load OIDN device.");
+		_unload_oidn();
+		return false;
+	}
+
+	return true;
+}
+
+void LightmapperRD::_unload_oidn() {
+	if (oidn_lib_handle) {
+		if (oidn_device) {
+			oidnReleaseDevice(oidn_device);
+			oidn_device = nullptr;
+		}
+
+		OS::get_singleton()->close_dynamic_library(oidn_lib_handle);
+		oidn_lib_handle = nullptr;
+	}
+}
+
+LightmapperRD::BakeError LightmapperRD::_denoise_oidn(RenderingDevice *p_rd, RID p_source_light_tex, RID p_source_normal_tex, RID p_dest_light_tex, const Size2i &p_atlas_size, int p_atlas_slices, bool p_bake_sh) {
+	for (int i = 0; i < p_atlas_slices; i++) {
+		Vector<uint8_t> sn = p_rd->texture_get_data(p_source_normal_tex, i);
+		Ref<Image> imgn = Image::create_from_data(p_atlas_size.width, p_atlas_size.height, false, Image::FORMAT_RGBAH, sn);
+		imgn->convert(Image::FORMAT_RGBF);
+		Vector<uint8_t> datan = imgn->get_data();
+
+		for (int j = 0; j < (p_bake_sh ? 4 : 1); j++) {
+			int index = i * (p_bake_sh ? 4 : 1) + j;
+
+			Vector<uint8_t> sl = p_rd->texture_get_data(p_source_light_tex, index);
+
+			Ref<Image> imgl = Image::create_from_data(p_atlas_size.width, p_atlas_size.height, false, Image::FORMAT_RGBAH, sl);
+			imgl->convert(Image::FORMAT_RGBF);
+			Vector<uint8_t> datal = imgl->get_data();
+
+			void *filter = oidnNewFilter(oidn_device, "RTLightmap");
+			oidnSetSharedFilterImage(filter, "color", (void *)datal.ptrw(), OIDN_FORMAT_FLOAT3, imgl->get_width(), imgl->get_height(), 0, 0, 0);
+			oidnSetSharedFilterImage(filter, "normal", (void *)datan.ptrw(), OIDN_FORMAT_FLOAT3, imgn->get_width(), imgn->get_height(), 0, 0, 0);
+			oidnSetSharedFilterImage(filter, "output", (void *)datal.ptrw(), OIDN_FORMAT_FLOAT3, imgl->get_width(), imgl->get_height(), 0, 0, 0);
+			oidnSetFilterBool(filter, "hdr", true);
+			oidnCommitFilter(filter);
+			oidnExecuteFilter(filter);
+
+			const char *msg = nullptr;
+			if (oidnGetDeviceError(oidn_device, &msg) != OIDN_ERROR_NONE) {
+				oidnReleaseFilter(filter);
+				ERR_FAIL_V_MSG(BAKE_ERROR_LIGHTMAP_CANT_PRE_BAKE_MESHES, String(msg));
+			}
+			oidnReleaseFilter(filter);
+
+			imgl->set_data(imgl->get_width(), imgl->get_height(), false, imgl->get_format(), datal);
+			imgl->convert(Image::FORMAT_RGBAH);
+			Vector<uint8_t> ds = imgl->get_data();
+			imgl.unref(); // Avoid copy on write.
+			{ // Restore alpha.
+				uint32_t count = sl.size() / 2;
+				const uint16_t *src = (const uint16_t *)sl.ptr();
+				uint16_t *dst = (uint16_t *)ds.ptrw();
+				for (uint32_t k = 0; k < count; k += 4) {
+					dst[k + 3] = src[k + 3];
+				}
+			}
+			p_rd->texture_update(p_dest_light_tex, index, ds);
+		}
+	}
+	return BAKE_OK;
+}
+
 LightmapperRD::BakeError LightmapperRD::_denoise(RenderingDevice *p_rd, Ref<RDShaderFile> &p_compute_shader, const RID &p_compute_base_uniform_set, PushConstant &p_push_constant, RID p_source_light_tex, RID p_source_normal_tex, RID p_dest_light_tex, float p_denoiser_strength, const Size2i &p_atlas_size, int p_atlas_slices, bool p_bake_sh, BakeStepFunc p_step_function) {
 	RID denoise_params_buffer = p_rd->uniform_buffer_create(sizeof(DenoiseParams));
 	DenoiseParams denoise_params;
@@ -1501,8 +1614,14 @@ LightmapperRD::BakeError LightmapperRD::bake(BakeQuality p_quality, bool p_use_d
 		}
 
 		{
-			SWAP(light_accum_tex, light_accum_tex2);
-			BakeError error = _denoise(rd, compute_shader, compute_base_uniform_set, push_constant, light_accum_tex2, normal_tex, light_accum_tex, p_denoiser_strength, atlas_size, atlas_slices, p_bake_sh, p_step_function);
+			String oidn_path = EDITOR_GET("filesystem/tools/oidn/oidn_library_path");
+			BakeError error;
+			if (!oidn_path.is_empty() && _load_oidn(oidn_path)) {
+				error = _denoise_oidn(rd, light_accum_tex, normal_tex, light_accum_tex, atlas_size, atlas_slices, p_bake_sh);
+			} else {
+				SWAP(light_accum_tex, light_accum_tex2);
+				error = _denoise(rd, compute_shader, compute_base_uniform_set, push_constant, light_accum_tex2, normal_tex, light_accum_tex, p_denoiser_strength, atlas_size, atlas_slices, p_bake_sh, p_step_function);
+			}
 			if (unlikely(error != BAKE_OK)) {
 				return error;
 			}
@@ -1766,3 +1885,7 @@ Vector<Color> LightmapperRD::get_bake_probe_sh(int p_probe) const {
 
 LightmapperRD::LightmapperRD() {
 }
+
+LightmapperRD::~LightmapperRD() {
+	_unload_oidn();
+}
diff --git a/modules/lightmapper_rd/lightmapper_rd.h b/modules/lightmapper_rd/lightmapper_rd.h
@@ -36,10 +36,70 @@
 #include "scene/resources/mesh.h"
 #include "servers/rendering/rendering_device.h"
 
+typedef enum {
+	OIDN_DEVICE_TYPE_DEFAULT = 0, // select device automatically
+
+	OIDN_DEVICE_TYPE_CPU = 1, // CPU device
+	OIDN_DEVICE_TYPE_SYCL = 2, // SYCL device
+	OIDN_DEVICE_TYPE_CUDA = 3, // CUDA device
+	OIDN_DEVICE_TYPE_HIP = 4, // HIP device
+} OIDNDeviceType;
+
+typedef enum {
+	OIDN_FORMAT_UNDEFINED = 0,
+
+	// 32-bit single-precision floating-point scalar and vector formats
+	OIDN_FORMAT_FLOAT = 1,
+	OIDN_FORMAT_FLOAT2,
+	OIDN_FORMAT_FLOAT3,
+	OIDN_FORMAT_FLOAT4,
+
+	// 16-bit half-precision floating-point scalar and vector formats
+	OIDN_FORMAT_HALF = 257,
+	OIDN_FORMAT_HALF2,
+	OIDN_FORMAT_HALF3,
+	OIDN_FORMAT_HALF4,
+} OIDNFormat;
+
+typedef enum {
+	OIDN_ERROR_NONE = 0, // no error occurred
+	OIDN_ERROR_UNKNOWN = 1, // an unknown error occurred
+	OIDN_ERROR_INVALID_ARGUMENT = 2, // an invalid argument was specified
+	OIDN_ERROR_INVALID_OPERATION = 3, // the operation is not allowed
+	OIDN_ERROR_OUT_OF_MEMORY = 4, // not enough memory to execute the operation
+	OIDN_ERROR_UNSUPPORTED_HARDWARE = 5, // the hardware (e.g. CPU) is not supported
+	OIDN_ERROR_CANCELLED = 6, // the operation was cancelled by the user
+} OIDNError;
+
+typedef void *(*oidnNewDevicePtr)(OIDNDeviceType type);
+typedef void (*oidnCommitDevicePtr)(void *device);
+typedef void *(*oidnNewFilterPtr)(void *device, const char *type);
+typedef void (*oidnSetSharedFilterImagePtr)(void *filter, const char *name, void *devPtr, OIDNFormat format, size_t width, size_t height, size_t byteOffset, size_t pixelByteStride, size_t rowByteStride);
+typedef void (*oidnSetFilterBoolPtr)(void *filter, const char *name, bool value);
+typedef void (*oidnCommitFilterPtr)(void *filter);
+typedef void (*oidnExecuteFilterPtr)(void *filter);
+typedef OIDNError (*oidnGetDeviceErrorPtr)(void *device, const char **outMessage);
+typedef void (*oidnReleaseFilterPtr)(void *filter);
+typedef void (*oidnReleaseDevicePtr)(void *device);
+
 class RDShaderFile;
 class LightmapperRD : public Lightmapper {
 	GDCLASS(LightmapperRD, Lightmapper)
 
+	String oidn_lib_path;
+	void *oidn_lib_handle = nullptr;
+	void *oidn_device = nullptr;
+	oidnNewDevicePtr oidnNewDevice = nullptr;
+	oidnCommitDevicePtr oidnCommitDevice = nullptr;
+	oidnNewFilterPtr oidnNewFilter = nullptr;
+	oidnSetSharedFilterImagePtr oidnSetSharedFilterImage = nullptr;
+	oidnSetFilterBoolPtr oidnSetFilterBool = nullptr;
+	oidnCommitFilterPtr oidnCommitFilter = nullptr;
+	oidnExecuteFilterPtr oidnExecuteFilter = nullptr;
+	oidnGetDeviceErrorPtr oidnGetDeviceError = nullptr;
+	oidnReleaseFilterPtr oidnReleaseFilter = nullptr;
+	oidnReleaseDevicePtr oidnReleaseDevice = nullptr;
+
 	struct MeshInstance {
 		MeshData data;
 		int slice = 0;
@@ -246,6 +306,10 @@ class LightmapperRD : public Lightmapper {
 	BakeError _dilate(RenderingDevice *rd, Ref<RDShaderFile> &compute_shader, RID &compute_base_uniform_set, PushConstant &push_constant, RID &source_light_tex, RID &dest_light_tex, const Size2i &atlas_size, int atlas_slices);
 	BakeError _denoise(RenderingDevice *p_rd, Ref<RDShaderFile> &p_compute_shader, const RID &p_compute_base_uniform_set, PushConstant &p_push_constant, RID p_source_light_tex, RID p_source_normal_tex, RID p_dest_light_tex, float p_denoiser_strength, const Size2i &p_atlas_size, int p_atlas_slices, bool p_bake_sh, BakeStepFunc p_step_function);
 
+	bool _load_oidn(const String &p_library_path);
+	void _unload_oidn();
+	BakeError _denoise_oidn(RenderingDevice *p_rd, RID p_source_light_tex, RID p_source_normal_tex, RID p_dest_light_tex, const Size2i &p_atlas_size, int p_atlas_slices, bool p_bake_sh);
+
 public:
 	virtual void add_mesh(const MeshData &p_mesh) override;
 	virtual void add_directional_light(bool p_static, const Vector3 &p_direction, const Color &p_color, float p_energy, float p_angular_distance, float p_shadow_blur) override;
@@ -265,6 +329,7 @@ class LightmapperRD : public Lightmapper {
 	Vector<Color> get_bake_probe_sh(int p_probe) const override;
 
 	LightmapperRD();
+	~LightmapperRD();
 };
 
 #endif // LIGHTMAPPER_RD_H
