Transparency support for LightmapGI

Author: guerro323

doc/classes/ProjectSettings.xml

@@ -2513,6 +2513,9 @@
 		<member name="rendering/lightmapping/bake_performance/max_rays_per_probe_pass" type="int" setter="" getter="" default="64">
 			The maximum number of rays that can be thrown per pass when baking dynamic object lighting in [LightmapProbe]s with [LightmapGI]. Depending on the scene, adjusting this value may result in higher GPU utilization when baking lightmaps, leading to faster bake times.
 		</member>
+		<member name="rendering/lightmapping/bake_performance/max_transparency_rays" type="int" setter="" getter="" default="64">
+			The maximum number of retry rays that can be thrown per pass when hitting a transparent surface when baking lightmaps with [LightmapGI]. Depending on the scene, adjusting this value may result in higher GPU utilization when baking lightmaps, leading to faster bake times.
+		</member>
 		<member name="rendering/lightmapping/bake_performance/region_size" type="int" setter="" getter="" default="512">
 			The region size to use when baking lightmaps with [LightmapGI].
 		</member>

modules/lightmapper_rd/lightmapper_rd.cpp

@@ -341,6 +341,7 @@ Lightmapper::BakeError LightmapperRD::_blit_meshes_into_atlas(int p_max_texture_
 }
 
 void LightmapperRD::_create_acceleration_structures(RenderingDevice *rd, Size2i atlas_size, int atlas_slices, AABB &bounds, int grid_size, uint32_t p_cluster_size, Vector<Probe> &p_probe_positions, GenerateProbes p_generate_probes, Vector<int> &slice_triangle_count, Vector<int> &slice_seam_count, RID &vertex_buffer, RID &triangle_buffer, RID &lights_buffer, RID &r_triangle_indices_buffer, RID &r_cluster_indices_buffer, RID &r_cluster_aabbs_buffer, RID &probe_positions_buffer, RID &grid_texture, RID &seams_buffer, BakeStepFunc p_step_function, void *p_bake_userdata) {
+	RendererRD::MaterialStorage *material_storage = RendererRD::MaterialStorage::get_singleton();
 	HashMap<Vertex, uint32_t, VertexHash> vertex_map;
 
 	//fill triangles array and vertex array
@@ -458,7 +459,15 @@ void LightmapperRD::_create_acceleration_structures(RenderingDevice *rd, Size2i
 			t.max_bounds[0] = taabb.position.x + MAX(taabb.size.x, 0.0001);
 			t.max_bounds[1] = taabb.position.y + MAX(taabb.size.y, 0.0001);
 			t.max_bounds[2] = taabb.position.z + MAX(taabb.size.z, 0.0001);
-			t.pad0 = t.pad1 = 0; //make valgrind not complain
+
+			t.cull_mode = RendererRD::MaterialStorage::ShaderData::CULL_BACK;
+
+			RID material = mi.data.material[i];
+			if (material.is_valid()) {
+				RendererRD::MaterialStorage::ShaderData *shader_data = material_storage->material_get_shader_data(mi.data.material[i]);
+				t.cull_mode = shader_data->get_cull_mode();
+			}
+			t.pad1 = 0; //make valgrind not complain
 			triangles.push_back(t);
 			slice_triangle_count.write[t.slice]++;
 		}
@@ -1207,6 +1216,8 @@ LightmapperRD::BakeError LightmapperRD::bake(BakeQuality p_quality, bool p_use_d
 	bake_parameters.exposure_normalization = p_exposure_normalization;
 	bake_parameters.bounces = p_bounces;
 	bake_parameters.bounce_indirect_energy = p_bounce_indirect_energy;
+	// Same number of rays for transparency regardless of quality (it's more of a retry rather than shooting new ones).
+	bake_parameters.transparency_rays = GLOBAL_GET("rendering/lightmapping/bake_performance/max_transparency_rays");
 
 	bake_parameters_buffer = rd->uniform_buffer_create(sizeof(BakeParameters));
 	rd->buffer_update(bake_parameters_buffer, 0, sizeof(BakeParameters), &bake_parameters);

modules/lightmapper_rd/lightmapper_rd.h

@@ -36,6 +36,8 @@
 #include "scene/resources/mesh.h"
 #include "servers/rendering/rendering_device.h"
 
+#include "servers/rendering/renderer_rd/storage_rd/material_storage.h"
+
 class RDShaderFile;
 class LightmapperRD : public Lightmapper {
 	GDCLASS(LightmapperRD, Lightmapper)
@@ -57,7 +59,8 @@ class LightmapperRD : public Lightmapper {
 		uint32_t bounces = 0;
 
 		float bounce_indirect_energy = 0.0f;
-		uint32_t pad[3] = {};
+		uint32_t transparency_rays = 0;
+		uint32_t pad[2] = {};
 	};
 
 	struct MeshInstance {
@@ -184,7 +187,7 @@ class LightmapperRD : public Lightmapper {
 		uint32_t indices[3] = {};
 		uint32_t slice = 0;
 		float min_bounds[3] = {};
-		float pad0 = 0.0;
+		uint32_t cull_mode = 0;
 		float max_bounds[3] = {};
 		float pad1 = 0.0;
 		bool operator<(const Triangle &p_triangle) const {

modules/lightmapper_rd/lm_common_inc.glsl

@@ -18,6 +18,7 @@ layout(set = 0, binding = 0) uniform BakeParameters {
 	uint bounces;
 
 	float bounce_indirect_energy;
+	uint transparency_rays;
 }
 bake_params;
 
@@ -33,11 +34,15 @@ layout(set = 0, binding = 1, std430) restrict readonly buffer Vertices {
 }
 vertices;
 
+#define CULL_DISABLED 0
+#define CULL_FRONT 1
+#define CULL_BACK 2
+
 struct Triangle {
 	uvec3 indices;
 	uint slice;
 	vec3 min_bounds;
-	uint pad0;
+	uint cull_mode;
 	vec3 max_bounds;
 	uint pad1;
 };

modules/lightmapper_rd/lm_compute.glsl

@@ -238,6 +238,17 @@ uint trace_ray(vec3 p_from, vec3 p_to, bool p_any_hit, out float r_distance, out
 								}
 
 								if (distance < best_distance) {
+									switch (triangle.cull_mode) {
+										case CULL_DISABLED:
+											backface = false;
+											break;
+										case CULL_FRONT:
+											backface = !backface;
+											break;
+										case CULL_BACK: // default behavior
+											break;
+									}
+
 									hit = backface ? RAY_BACK : RAY_FRONT;
 									best_distance = distance;
 									r_distance = distance;
@@ -290,6 +301,27 @@ uint trace_ray_closest_hit_triangle(vec3 p_from, vec3 p_to, out uint r_triangle,
 	return trace_ray(p_from, p_to, false, distance, normal, r_triangle, r_barycentric);
 }
 
+uint trace_ray_closest_hit_triangle_albedo_alpha(vec3 p_from, vec3 p_to, out vec4 albedo_alpha, out vec3 hit_position) {
+	float distance;
+	vec3 normal;
+	uint tidx;
+	vec3 barycentric;
+
+	uint ret = trace_ray(p_from, p_to, false, distance, normal, tidx, barycentric);
+	if (ret != RAY_MISS) {
+		Vertex vert0 = vertices.data[triangles.data[tidx].indices.x];
+		Vertex vert1 = vertices.data[triangles.data[tidx].indices.y];
+		Vertex vert2 = vertices.data[triangles.data[tidx].indices.z];
+
+		vec3 uvw = vec3(barycentric.x * vert0.uv + barycentric.y * vert1.uv + barycentric.z * vert2.uv, float(triangles.data[tidx].slice));
+
+		albedo_alpha = textureLod(sampler2DArray(albedo_tex, linear_sampler), uvw, 0);
+		hit_position = barycentric.x * vert0.position + barycentric.y * vert1.position + barycentric.z * vert2.position;
+	}
+
+	return ret;
+}
+
 uint trace_ray_closest_hit_distance(vec3 p_from, vec3 p_to, out float r_distance, out vec3 r_normal) {
 	uint triangle;
 	vec3 barycentric;
@@ -359,6 +391,8 @@ float get_omni_attenuation(float distance, float inv_range, float decay) {
 }
 
 void trace_direct_light(vec3 p_position, vec3 p_normal, uint p_light_index, bool p_soft_shadowing, out vec3 r_light, out vec3 r_light_dir, inout uint r_noise) {
+	const float EPSILON = 0.00001;
+
 	r_light = vec3(0.0f);
 
 	vec3 light_pos;
@@ -417,7 +451,13 @@ void trace_direct_light(vec3 p_position, vec3 p_normal, uint p_light_index, bool
 
 		uint hits = 0;
 		vec3 light_disk_to_point = light_to_point;
+
+		float power = 0.0;
+		int power_accm = 0;
+		vec3 prev_pos = p_position;
 		for (uint j = 0; j < shadowing_ray_count; j++) {
+			p_position = prev_pos;
+
 			// Optimization:
 			// Once already traced an important proportion of rays, if all are hits or misses,
 			// assume we're not in the penumbra so we can infer the rest would have the same result
@@ -440,16 +480,68 @@ void trace_direct_light(vec3 p_position, vec3 p_normal, uint p_light_index, bool
 			vec2 disk_sample = (r * vec2(cos(a), sin(a))) * soft_shadowing_disk_size * light_data.shadow_blur;
 			light_disk_to_point = normalize(light_to_point + disk_sample.x * light_to_point_tan + disk_sample.y * light_to_point_bitan);
 
-			if (trace_ray_any_hit(p_position - light_disk_to_point * bake_params.bias, p_position - light_disk_to_point * dist) == RAY_MISS) {
-				hits++;
+			float sample_penumbra = 0.0;
+			bool sample_did_hit = false;
+			for (uint iter = 0; iter < bake_params.transparency_rays; iter++) {
+				vec4 hit_albedo = vec4(1.0);
+				vec3 hit_position;
+				uint ret = trace_ray_closest_hit_triangle_albedo_alpha(p_position - light_disk_to_point * bake_params.bias, p_position - light_disk_to_point * dist, hit_albedo, hit_position);
+				if (ret == RAY_MISS) {
+					if (!sample_did_hit)
+						sample_penumbra = 1.0;
+					hits += 1;
+					break;
+				} else if (ret == RAY_FRONT || ret == RAY_BACK) {
+					bool contribute = ret == RAY_FRONT || !sample_did_hit;
+					if (!sample_did_hit) {
+						sample_penumbra = 1.0;
+						sample_did_hit = true;
+					}
+
+					hits += 1;
+
+					if (contribute)
+						sample_penumbra = max(sample_penumbra - hit_albedo.a - EPSILON, 0.0);
+					p_position = hit_position + r_light_dir * bake_params.bias;
+
+					if (sample_penumbra - EPSILON <= 0)
+						break;
+				}
 			}
+
+			power += sample_penumbra;
+			power_accm += 1;
 		}
 
-		penumbra = float(hits) / float(shadowing_ray_count);
+		penumbra = power / float(power_accm);
 	} else {
-		if (trace_ray_any_hit(p_position + r_light_dir * bake_params.bias, light_pos) == RAY_MISS) {
-			penumbra = 1.0;
+		bool did_hit = false;
+		penumbra = 0.0;
+		for (uint iter = 0; iter < bake_params.transparency_rays; iter++) {
+			vec4 hit_albedo = vec4(1.0);
+			vec3 hit_position;
+			uint ret = trace_ray_closest_hit_triangle_albedo_alpha(p_position + r_light_dir * bake_params.bias, light_pos, hit_albedo, hit_position);
+			if (ret == RAY_MISS) {
+				if (!did_hit)
+					penumbra = 1.0;
+				break;
+			} else if (ret == RAY_FRONT || ret == RAY_BACK) {
+				bool contribute = (ret == RAY_FRONT || !did_hit);
+				if (!did_hit) {
+					penumbra = 1.0;
+					did_hit = true;
+				}
+
+				if (contribute)
+					penumbra = max(penumbra - hit_albedo.a - EPSILON, 0.0);
+				p_position = hit_position + r_light_dir * bake_params.bias;
+
+				if (penumbra - EPSILON <= 0)
+					break;
+			}
 		}
+
+		penumbra = clamp(penumbra, 0.0, 1.0);
 	}
 
 	r_light = light_data.color * light_data.energy * attenuation * penumbra;
@@ -474,6 +566,7 @@ vec3 trace_indirect_light(vec3 p_position, vec3 p_ray_dir, inout uint r_noise) {
 	vec3 position = p_position;
 	vec3 ray_dir = p_ray_dir;
 	uint max_depth = max(bake_params.bounces, 1);
+	uint transparency_rays_left = bake_params.transparency_rays;
 	vec3 throughput = vec3(1.0);
 	vec3 light = vec3(0.0);
 	for (uint depth = 0; depth < max_depth; depth++) {
@@ -487,6 +580,8 @@ vec3 trace_indirect_light(vec3 p_position, vec3 p_ray_dir, inout uint r_noise) {
 			vec3 uvw = vec3(barycentric.x * vert0.uv + barycentric.y * vert1.uv + barycentric.z * vert2.uv, float(triangles.data[tidx].slice));
 			position = barycentric.x * vert0.position + barycentric.y * vert1.position + barycentric.z * vert2.position;
 
+			vec3 prev_normal = ray_dir;
+
 			vec3 norm0 = vec3(vert0.normal_xy, vert0.normal_z);
 			vec3 norm1 = vec3(vert1.normal_xy, vert1.normal_z);
 			vec3 norm2 = vec3(vert2.normal_xy, vert2.normal_z);
@@ -508,13 +603,29 @@ vec3 trace_indirect_light(vec3 p_position, vec3 p_ray_dir, inout uint r_noise) {
 			direct_light *= bake_params.exposure_normalization;
 #endif
 
-			vec3 albedo = textureLod(sampler2DArray(albedo_tex, linear_sampler), uvw, 0).rgb;
+			vec4 albedo_alpha = textureLod(sampler2DArray(albedo_tex, linear_sampler), uvw, 0).rgba;
 			vec3 emissive = textureLod(sampler2DArray(emission_tex, linear_sampler), uvw, 0).rgb;
 			emissive *= bake_params.exposure_normalization;
 
-			light += throughput * emissive;
-			throughput *= albedo;
-			light += throughput * direct_light * bake_params.bounce_indirect_energy;
+			light += throughput * emissive * albedo_alpha.a;
+			throughput = mix(throughput, throughput * albedo_alpha.rgb, albedo_alpha.a);
+			light += throughput * direct_light * bake_params.bounce_indirect_energy * albedo_alpha.a;
+
+			if (albedo_alpha.a < 1.0) {
+				transparency_rays_left -= 1;
+				depth -= 1;
+				if (transparency_rays_left <= 0) {
+					break;
+				}
+
+				// Either bounce off the transparent surface or keep going forward
+				float pa = albedo_alpha.a * albedo_alpha.a;
+				if (randomize(r_noise) > pa) {
+					normal = prev_normal;
+				}
+
+				position += normal * bake_params.bias;
+			}
 
 			// Use Russian Roulette to determine a probability to terminate the bounce earlier as an optimization.
 			// <https://computergraphics.stackexchange.com/questions/2316/is-russian-roulette-really-the-answer>
@@ -532,9 +643,53 @@ vec3 trace_indirect_light(vec3 p_position, vec3 p_ray_dir, inout uint r_noise) {
 			// Look for the environment color and stop bouncing.
 			light += throughput * trace_environment_color(ray_dir);
 			break;
-		} else {
-			// Ignore any other trace results.
-			break;
+		} else if (trace_result == RAY_BACK) {
+			Vertex vert0 = vertices.data[triangles.data[tidx].indices.x];
+			Vertex vert1 = vertices.data[triangles.data[tidx].indices.y];
+			Vertex vert2 = vertices.data[triangles.data[tidx].indices.z];
+			vec3 uvw = vec3(barycentric.x * vert0.uv + barycentric.y * vert1.uv + barycentric.z * vert2.uv, float(triangles.data[tidx].slice));
+			position = barycentric.x * vert0.position + barycentric.y * vert1.position + barycentric.z * vert2.position;
+
+			vec4 albedo_alpha = textureLod(sampler2DArray(albedo_tex, linear_sampler), uvw, 0).rgba;
+
+			if (albedo_alpha.a > 1.0)
+				break;
+
+			transparency_rays_left -= 1;
+			depth -= 1;
+			if (transparency_rays_left <= 0) {
+				break;
+			}
+
+			vec3 norm0 = vec3(vert0.normal_xy, vert0.normal_z);
+			vec3 norm1 = vec3(vert1.normal_xy, vert1.normal_z);
+			vec3 norm2 = vec3(vert2.normal_xy, vert2.normal_z);
+			vec3 normal = barycentric.x * norm0 + barycentric.y * norm1 + barycentric.z * norm2;
+
+			vec3 direct_light = vec3(0.0f);
+#ifdef USE_LIGHT_TEXTURE_FOR_BOUNCES
+			direct_light += textureLod(sampler2DArray(source_light, linear_sampler), uvw, 0.0).rgb;
+#else
+			// Trace the lights directly. Significantly more expensive but more accurate in scenarios
+			// where the lightmap texture isn't reliable.
+			for (uint i = 0; i < bake_params.light_count; i++) {
+				vec3 light;
+				vec3 light_dir;
+				trace_direct_light(position, normal, i, false, light, light_dir, r_noise);
+				direct_light += light * lights.data[i].indirect_energy;
+			}
+
+			direct_light *= bake_params.exposure_normalization;
+#endif
+
+			vec3 emissive = textureLod(sampler2DArray(emission_tex, linear_sampler), uvw, 0).rgb;
+			emissive *= bake_params.exposure_normalization;
+
+			light += throughput * emissive * albedo_alpha.a;
+			throughput = mix(mix(throughput, throughput * albedo_alpha.rgb, albedo_alpha.a), vec3(0.0), albedo_alpha.a);
+			light += throughput * direct_light * bake_params.bounce_indirect_energy * albedo_alpha.a;
+
+			position += ray_dir * bake_params.bias;
 		}
 	}
 

modules/lightmapper_rd/register_types.cpp

@@ -52,6 +52,7 @@ void initialize_lightmapper_rd_module(ModuleInitializationLevel p_level) {
 	GLOBAL_DEF(PropertyInfo(Variant::INT, "rendering/lightmapping/bake_quality/ultra_quality_ray_count", PROPERTY_HINT_RANGE, "1,4096,1,or_greater"), 2048);
 	GLOBAL_DEF(PropertyInfo(Variant::INT, "rendering/lightmapping/bake_performance/max_rays_per_pass", PROPERTY_HINT_RANGE, "1,256,1,or_greater"), 32);
 	GLOBAL_DEF(PropertyInfo(Variant::INT, "rendering/lightmapping/bake_performance/region_size", PROPERTY_HINT_RANGE, "1,4096,1,or_greater"), 512);
+	GLOBAL_DEF(PropertyInfo(Variant::INT, "rendering/lightmapping/bake_performance/max_transparency_rays", PROPERTY_HINT_RANGE, "1,256,1,or_greater"), 64);
 
 	GLOBAL_DEF(PropertyInfo(Variant::INT, "rendering/lightmapping/bake_quality/low_quality_probe_ray_count", PROPERTY_HINT_RANGE, "1,4096,1,or_greater"), 64);
 	GLOBAL_DEF(PropertyInfo(Variant::INT, "rendering/lightmapping/bake_quality/medium_quality_probe_ray_count", PROPERTY_HINT_RANGE, "1,4096,1,or_greater"), 256);

scene/3d/lightmap_gi.cpp

@@ -791,7 +791,7 @@ LightmapGI::BakeError LightmapGI::bake(Node *p_from_node, String p_image_data_pa
 					w_albedo[i + 0] = uint8_t(CLAMP(float(r_aa[i + 0]) * (1.0 - float(r_orm[i + 2] / 255.0)), 0, 255));
 					w_albedo[i + 1] = uint8_t(CLAMP(float(r_aa[i + 1]) * (1.0 - float(r_orm[i + 2] / 255.0)), 0, 255));
 					w_albedo[i + 2] = uint8_t(CLAMP(float(r_aa[i + 2]) * (1.0 - float(r_orm[i + 2] / 255.0)), 0, 255));
-					w_albedo[i + 3] = 255;
+					w_albedo[i + 3] = r_aa[i + 3];
 				}
 
 				md.albedo_on_uv2.instantiate();
@@ -812,6 +812,11 @@ LightmapGI::BakeError LightmapGI::bake(Node *p_from_node, String p_image_data_pa
 					continue;
 				}
 				Array a = mf.mesh->surface_get_arrays(i);
+				Ref<Material> mat = mf.mesh->surface_get_material(i);
+				RID mat_rid = RID();
+				if (mat.is_valid()) {
+					mat_rid = mat->get_rid();
+				}
 
 				Vector<Vector3> vertices = a[Mesh::ARRAY_VERTEX];
 				const Vector3 *vr = vertices.ptr();
@@ -861,6 +866,7 @@ LightmapGI::BakeError LightmapGI::bake(Node *p_from_node, String p_image_data_pa
 
 						md.uv2.push_back(uvr[vidx[k]]);
 						md.normal.push_back(normal_xform.xform(nr[vidx[k]]).normalized());
+						md.material.push_back(mat_rid);
 					}
 				}
 			}

scene/3d/lightmapper.h

@@ -170,6 +170,7 @@ class Lightmapper : public RefCounted {
 		Vector<Vector3> points;
 		Vector<Vector2> uv2;
 		Vector<Vector3> normal;
+		Vector<RID> material;
 		Ref<Image> albedo_on_uv2;
 		Ref<Image> emission_on_uv2;
 		Variant userdata;