feat(Gsplat): IBL shadows support for Gaussian Splats

packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts

@@ -7,6 +7,7 @@ import { type Matrix } from "../../Maths/math.vector";
 import { type GaussianSplattingMesh } from "../../Meshes/GaussianSplatting/gaussianSplattingMesh";
 import { type AbstractEngine } from "../../Engines/abstractEngine";
 import { SerializationHelper } from "../../Misc/decorators.serialization";
+import { Logger } from "../../Misc/logger";
 import { VertexBuffer } from "../../Buffers/buffer";
 import { MaterialDefines } from "../../Materials/materialDefines";
 import { PushMaterial } from "../../Materials/pushMaterial";
@@ -26,6 +27,8 @@ import "../../Shaders/gaussianSplattingDepth.fragment";
 import "../../Shaders/gaussianSplattingDepth.vertex";
 import "../../ShadersWGSL/gaussianSplattingDepth.fragment";
 import "../../ShadersWGSL/gaussianSplattingDepth.vertex";
+import "../../Shaders/gaussianSplattingVoxel.fragment";
+import "../../Shaders/gaussianSplattingVoxel.vertex";
 import {
     BindFogParameters,
     BindLogDepth,
@@ -36,6 +39,7 @@ import {
     PrepareUniformsAndSamplersList,
 } from "../materialHelper.functions";
 import { ShaderLanguage } from "../shaderLanguage";
+import { Engine } from "../../Engines/engine";
 
 /**
  * Computes the maximum number of Gaussian Splatting compound parts supported by the given engine.
@@ -191,6 +195,20 @@ export class GaussianSplattingMaterial extends PushMaterial {
         "partIndicesTexture",
     ];
     protected static _UniformBuffers = ["Scene", "Mesh"];
+    protected static _VoxelUniforms = [
+        "world",
+        "dataTextureSize",
+        "alpha",
+        "invWorldScale",
+        "viewMatrix",
+        "nearPlane",
+        "farPlane",
+        "stepSize",
+        "partWorld",
+        "partVisibility",
+        "axis",
+    ];
+    protected static _VoxelSamplers = ["rotationsATexture", "rotationsBTexture", "rotationScaleTexture", "centersTexture", "colorsTexture", "partIndicesTexture"];
     protected static _Uniforms = [
         "world",
         "view",
@@ -571,6 +589,113 @@ export class GaussianSplattingMaterial extends PushMaterial {
         }
     }
 
+    private _voxelMissingTextureWarned = new Set<number>();
+
+    protected _bindVoxelEffectUniforms(gsMesh: GaussianSplattingMesh, gsMaterial: GaussianSplattingMaterial, shaderMaterial: ShaderMaterial): boolean {
+        if (!gsMesh.rotationsATexture) {
+            // Attempt to enable rotation/scale texture generation (no-op if already enabled).
+            // If the mesh is already loaded and splat data is in RAM this triggers updateData() synchronously.
+            // If the mesh was loaded without keepInRam=true the setter will log an error.
+            gsMesh.needsRotationScaleTextures = true;
+
+            if (!gsMesh.rotationsATexture) {
+                // Mesh data was already processed but rotation textures couldn't be generated — warn once.
+                if (gsMesh.centersTexture && !gsMesh.splatsData && !this._voxelMissingTextureWarned.has(gsMesh.uniqueId)) {
+                    this._voxelMissingTextureWarned.add(gsMesh.uniqueId);
+                    Logger.Error(
+                        "IBL voxelization: GaussianSplattingMesh rotation/scale textures are not available. " +
+                            "Create the mesh either with keepInRam=true or needsRotationScaleTextures=true to enable IBL shadow voxelization."
+                    );
+                }
+                return false;
+            }
+        }
+        this._voxelMissingTextureWarned.delete(gsMesh.uniqueId);
+
+        const effect = shaderMaterial.getEffect()!;
+        gsMesh.getMeshUniformBuffer().bindToEffect(effect, "Mesh");
+
+        const textureSize = gsMesh.rotationsATexture.getSize();
+        effect.setFloat("alpha", gsMaterial.alpha);
+        effect.setFloat2("dataTextureSize", textureSize.width, textureSize.height);
+        effect.setTexture("rotationsATexture", gsMesh.rotationsATexture);
+        effect.setTexture("rotationsBTexture", gsMesh.rotationsBTexture);
+        effect.setTexture("rotationScaleTexture", gsMesh.rotationScaleTexture);
+        effect.setTexture("centersTexture", gsMesh.centersTexture);
+        effect.setTexture("colorsTexture", gsMesh.colorsTexture);
+
+        if (gsMesh.partIndicesTexture) {
+            effect.setTexture("partIndicesTexture", gsMesh.partIndicesTexture);
+            const partWorldData = new Float32Array(gsMesh.partCount * 16);
+            for (let i = 0; i < gsMesh.partCount; i++) {
+                gsMesh.getWorldMatrixForPart(i).toArray(partWorldData, i * 16);
+            }
+            effect.setMatrices("partWorld", partWorldData);
+            const partVisibilityData: number[] = [];
+            for (let i = 0; i < gsMesh.partCount; i++) {
+                partVisibilityData.push(gsMesh.partVisibility[i] ?? 1.0);
+            }
+            effect.setArray("partVisibility", partVisibilityData);
+        }
+        return true;
+    }
+
+    /**
+     * Create a voxel rendering material for a Gaussian Splatting mesh, for use with IBL shadow voxelization.
+     * The returned ShaderMaterial's onBindObservable binds the GS mesh-side uniforms (textures, alpha, dataTextureSize, part data).
+     * The caller (e.g. iblShadowsVoxelRenderer) is responsible for setting the per-slab uniforms on the returned material:
+     * viewMatrix, invWorldScale, nearPlane, farPlane, stepSize.
+     * @param scene scene it belongs to
+     * @param shaderLanguage GLSL or WGSL
+     * @param maxDrawBuffers number of draw buffers (MRT outputs) per voxelization slab
+     * @param compoundMesh whether the mesh is a compound mesh
+     * @returns voxel rendering shader material
+     */
+    public makeVoxelRenderingMaterial(scene: Scene, shaderLanguage: ShaderLanguage, maxDrawBuffers: number, compoundMesh: boolean = false): ShaderMaterial {
+        const defines = [`#define MAX_DRAW_BUFFERS ${maxDrawBuffers}`];
+
+        defines.push("#define IS_FOR_VOXELIZATION");
+
+        if (compoundMesh) {
+            defines.push("#define IS_COMPOUND 1");
+            defines.push(`#define MAX_PART_COUNT ${GetGaussianSplattingMaxPartCount(scene.getEngine())}`);
+        }
+
+        const shaderMaterial = new ShaderMaterial(
+            "gaussianSplattingVoxelRender",
+            scene,
+            {
+                vertex: "gaussianSplattingVoxel",
+                fragment: "gaussianSplattingVoxel",
+            },
+            {
+                attributes: GaussianSplattingMaterial._Attribs,
+                uniforms: GaussianSplattingMaterial._VoxelUniforms,
+                samplers: GaussianSplattingMaterial._VoxelSamplers,
+                uniformBuffers: GaussianSplattingMaterial._UniformBuffers,
+                shaderLanguage: shaderLanguage,
+                defines: defines,
+                needAlphaBlending: false,
+                extraInitializationsAsync: async () => {
+                    if (shaderLanguage === ShaderLanguage.WGSL) {
+                        await Promise.all([import("../../ShadersWGSL/gaussianSplattingVoxel.vertex"), import("../../ShadersWGSL/gaussianSplattingVoxel.fragment")]);
+                    } else {
+                        await Promise.all([import("../../Shaders/gaussianSplattingVoxel.vertex"), import("../../Shaders/gaussianSplattingVoxel.fragment")]);
+                    }
+                },
+            }
+        );
+        shaderMaterial.cullBackFaces = false;
+        shaderMaterial.backFaceCulling = false;
+        shaderMaterial.depthFunction = Engine.ALWAYS;
+        shaderMaterial.onBindObservable.add((mesh: AbstractMesh) => {
+            const gsMaterial = mesh.material as GaussianSplattingMaterial;
+            const gsMesh = mesh as GaussianSplattingMesh;
+            this._bindVoxelEffectUniforms(gsMesh, gsMaterial, shaderMaterial);
+        });
+        return shaderMaterial;
+    }
+
     /**
      * Create a depth rendering material for a Gaussian Splatting mesh
      * @param scene scene it belongs to
@@ -587,7 +712,7 @@ export class GaussianSplattingMaterial extends PushMaterial {
         }
 
         if (compoundMesh) {
-            defines.push("#define IS_COMPOUND");
+            defines.push("#define IS_COMPOUND 1");
             defines.push(`#define MAX_PART_COUNT ${GetGaussianSplattingMaxPartCount(scene.getEngine())}`);
         }
 

packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMesh.ts

@@ -106,19 +106,26 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase {
     private _partIndicesTexture: Nullable<BaseTexture> = null;
     private _partIndices: Nullable<Uint8Array> = null;
 
+    /** Gets the part indices texture used for compound rendering */
+    public get partIndicesTexture() {
+        return this._partIndicesTexture;
+    }
+
     /**
      * Creates a new GaussianSplattingMesh
      * @param name the name of the mesh
      * @param url optional URL to load a Gaussian Splatting file from
      * @param scene the hosting scene
      * @param keepInRam whether to keep the raw splat data in RAM after uploading to GPU
+     * @param needsRotationScaleTextures generate rotation and scale matrix textures required for voxel-based IBL shadows
      */
-    constructor(name: string, url: Nullable<string> = null, scene: Nullable<Scene> = null, keepInRam: boolean = false) {
+    constructor(name: string, url: Nullable<string> = null, scene: Nullable<Scene> = null, keepInRam: boolean = false, needsRotationScaleTextures: boolean = false) {
         super(name, url, scene, keepInRam);
         // Ensure _splatsData is retained once compound mode is entered — addPart/addParts need
         // the source data for full-texture rebuilds. Set after super() so it is visible to
         // _updateData when the async load completes.
         this._alwaysRetainSplatsData = true;
+        this._needsRotationScaleTextures = needsRotationScaleTextures;
     }
 
     /**
@@ -137,9 +144,7 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase {
         for (const proxy of this._partProxies) {
             proxy.dispose();
         }
-        if (this._partIndicesTexture) {
-            this._partIndicesTexture.dispose();
-        }
+        this._partIndicesTexture?.dispose();
         this._partProxies = [];
         this._partMatrices = [];
         this._partVisibility = [];
@@ -932,10 +937,16 @@ export class GaussianSplattingMesh extends GaussianSplattingMeshBase {
         this._covariancesBTexture?.dispose();
         this._centersTexture?.dispose();
         this._colorsTexture?.dispose();
+        this._rotationsATexture?.dispose();
+        this._rotationsBTexture?.dispose();
+        this._rotationScaleTexture?.dispose();
         this._covariancesATexture = null;
         this._covariancesBTexture = null;
         this._centersTexture = null;
         this._colorsTexture = null;
+        this._rotationsATexture = null;
+        this._rotationsBTexture = null;
+        this._rotationScaleTexture = null;
         if (this._shTextures) {
             for (const t of this._shTextures) {
                 t.dispose();