IBL Shadows - Init passes only after voxelization (#16527)

Okay, I've been investigating the issue where the shadow accumulation
isn't working initially after loading the [IBL Shadow Demo
playground](https://playground.babylonjs.com/#8R5SSE#618). On some
loads, the shadows will appear noisy and not accumulate over multiple
frames. Note that this issue never happens if you have Spector.js
enabled, which is why I had such a hard time reproducing it originally.
The actual issue is that the uniform, `accumulationParameters` fails to
be set with
`WebGL: INVALID_OPERATION: uniform4f: location is not from the
associated program`

Even though this fails, the uniform value is still cached and Babylon
doesn't set the uniform again until the value changes. That is why
accumulation appears to be disabled until you move the camera (which
causes the uniform to change and update).
I have not been able to figure out _why_ the uniform set fails in the
first place. It seems to be consistently the 3rd render that fails.
Before that, the uniform is set successfully and, as far as I can tell,
the shader program isn't updated after that point. And after the
failure, the uniform can be set again.

The one thing that I was able to change to make the issue seem to go
away was to put enough of a delay on the rendering of the accumulation
pass. I've added logic to only start rendering the accumulation pass
once the first voxelization is complete. This is fine since we can't
render shadows until this is done anyway but I don't know why it should
fix the problem.

Author: MiiBond

packages/dev/core/src/Rendering/IBLShadows/iblShadowsAccumulationPass.ts

@@ -10,6 +10,8 @@ import { ProceduralTexture } from "core/Materials/Textures/Procedurals/procedura
 import type { IProceduralTextureCreationOptions } from "core/Materials/Textures/Procedurals/proceduralTexture";
 import type { IblShadowsRenderPipeline } from "./iblShadowsRenderPipeline";
 import { Observable } from "../../Misc/observable";
+import type { EventState } from "../../Misc/observable";
+import type { Nullable } from "../../types";
 
 /**
  * This should not be instanciated directly, as it is part of a scene component
@@ -207,17 +209,10 @@ export class _IblShadowsAccumulationPass {
         // Need to set all the textures first so that the effect gets created with the proper uniforms.
         this._setOutputTextureBindings();
 
-        let counter = 0;
-        this._scene.onBeforeRenderObservable.add(() => {
-            counter = 0;
-        });
-        this._scene.onAfterRenderTargetsRenderObservable.add(() => {
-            if (++counter == 2) {
-                if (this.enabled && this._outputTexture.isReady()) {
-                    this._setOutputTextureBindings();
-                    this._outputTexture.render();
-                }
-            }
+        this._renderWhenGBufferReady = this._render.bind(this);
+        // Don't start rendering until the first vozelization is done.
+        this._renderPipeline.onVoxelizationCompleteObservable.addOnce(() => {
+            this._scene.geometryBufferRenderer!.getGBuffer().onAfterRenderObservable.add(this._renderWhenGBufferReady);
         });
 
         // Create the accumulation texture for the previous frame.
@@ -284,7 +279,7 @@ export class _IblShadowsAccumulationPass {
         this._oldPositionCopy.onBeforeGenerationObservable.add(this._updatePositionCopy.bind(this));
     }
 
-    private _setOutputTextureBindings() {
+    private _setOutputTextureBindings(): boolean {
         const remanence = this._isMoving ? this.remanence : 0.99;
         this._accumulationParams.set(remanence, this.reset ? 1.0 : 0.0, this._renderPipeline.voxelGridSize, 0.0);
         this._outputTexture.setTexture("spatialBlurSampler", this._renderPipeline._getSpatialBlurTexture());
@@ -294,7 +289,7 @@ export class _IblShadowsAccumulationPass {
 
         const geometryBufferRenderer = this._scene.geometryBufferRenderer;
         if (!geometryBufferRenderer) {
-            return;
+            return false;
         }
         const velocityIndex = geometryBufferRenderer.getTextureIndex(GeometryBufferRenderer.VELOCITY_LINEAR_TEXTURE_TYPE);
         this._outputTexture.setTexture("motionSampler", geometryBufferRenderer.getGBuffer().textures[velocityIndex]);
@@ -303,6 +298,7 @@ export class _IblShadowsAccumulationPass {
 
         this.reset = false;
         this._isMoving = false;
+        return true;
     }
 
     private _updatePositionCopy() {
@@ -315,6 +311,16 @@ export class _IblShadowsAccumulationPass {
         this._oldAccumulationCopy.setTexture("textureSampler", this._outputTexture);
     }
 
+    private _render() {
+        if (this.enabled && this._outputTexture.isReady() && this._outputTexture.getEffect()?.isReady()) {
+            if (this._setOutputTextureBindings()) {
+                this._outputTexture.render();
+            }
+        }
+    }
+
+    private _renderWhenGBufferReady: Nullable<(eventData: number, eventState: EventState) => void> = null;
+
     /**
      * Called by render pipeline when canvas resized.
      * @param scaleFactor The factor by which to scale the canvas size.
@@ -324,6 +330,10 @@ export class _IblShadowsAccumulationPass {
             width: Math.max(1.0, Math.floor(this._engine.getRenderWidth() * scaleFactor)),
             height: Math.max(1.0, Math.floor(this._engine.getRenderHeight() * scaleFactor)),
         };
+        // Don't resize if the size is the same as the current size.
+        if (this._outputTexture.getSize().width === newSize.width && this._outputTexture.getSize().height === newSize.height) {
+            return;
+        }
         this._outputTexture.resize(newSize, false);
         this._oldAccumulationCopy.resize(newSize, false);
         this._oldPositionCopy.resize({ width: this._engine.getRenderWidth(), height: this._engine.getRenderHeight() }, false);
@@ -355,6 +365,10 @@ export class _IblShadowsAccumulationPass {
      * Disposes the associated resources
      */
     public dispose() {
+        if (this._scene.geometryBufferRenderer && this._renderWhenGBufferReady) {
+            const gBuffer = this._scene.geometryBufferRenderer.getGBuffer();
+            gBuffer.onAfterRenderObservable.removeCallback(this._renderWhenGBufferReady);
+        }
         this._disposeTextures();
         if (this._debugPassPP) {
             this._debugPassPP.dispose();

packages/dev/core/src/Rendering/IBLShadows/iblShadowsRenderPipeline.ts

@@ -1125,6 +1125,7 @@ export class IblShadowsRenderPipeline extends PostProcessRenderPipeline {
         this._dummyTexture3d.dispose();
         this.onNewIblReadyObservable.clear();
         this.onShadowTextureReadyObservable.clear();
+        this.onVoxelizationCompleteObservable.clear();
         super.dispose();
     }
 }

packages/dev/core/src/Rendering/IBLShadows/iblShadowsSpatialBlurPass.ts

@@ -9,6 +9,8 @@ import { GeometryBufferRenderer } from "../../Rendering/geometryBufferRenderer";
 import { ProceduralTexture } from "core/Materials/Textures/Procedurals/proceduralTexture";
 import type { IProceduralTextureCreationOptions } from "core/Materials/Textures/Procedurals/proceduralTexture";
 import type { IblShadowsRenderPipeline } from "./iblShadowsRenderPipeline";
+import type { EventState } from "../../Misc/observable";
+import type { Nullable } from "../../types";
 
 /**
  * This should not be instanciated directly, as it is part of a scene component
@@ -161,35 +163,39 @@ export class _IblShadowsSpatialBlurPass {
         // Need to set all the textures first so that the effect gets created with the proper uniforms.
         this._setBindings();
 
-        let counter = 0;
-        this._scene.onBeforeRenderObservable.add(() => {
-            counter = 0;
-        });
-        this._scene.onAfterRenderTargetsRenderObservable.add(() => {
-            if (++counter == 2) {
-                if (this.enabled && this._outputTexture.isReady()) {
-                    this._setBindings();
-                    this._outputTexture.render();
-                }
-            }
+        this._renderWhenGBufferReady = this._render.bind(this);
+        // Don't start rendering until the first vozelization is done.
+        this._renderPipeline.onVoxelizationCompleteObservable.addOnce(() => {
+            this._scene.geometryBufferRenderer!.getGBuffer().onAfterRenderObservable.add(this._renderWhenGBufferReady);
         });
     }
 
-    private _setBindings() {
+    private _setBindings(): boolean {
         this._outputTexture.setTexture("voxelTracingSampler", this._renderPipeline._getVoxelTracingTexture());
         const iterationCount = 1;
         this._blurParameters.set(iterationCount, this._worldScale, 0.0, 0.0);
         this._outputTexture.setVector4("blurParameters", this._blurParameters);
         const geometryBufferRenderer = this._scene.geometryBufferRenderer;
         if (!geometryBufferRenderer) {
-            return;
+            return false;
         }
         const depthIndex = geometryBufferRenderer.getTextureIndex(GeometryBufferRenderer.SCREENSPACE_DEPTH_TEXTURE_TYPE);
         this._outputTexture.setTexture("depthSampler", geometryBufferRenderer.getGBuffer().textures[depthIndex]);
         const wnormalIndex = geometryBufferRenderer.getTextureIndex(GeometryBufferRenderer.NORMAL_TEXTURE_TYPE);
         this._outputTexture.setTexture("worldNormalSampler", geometryBufferRenderer.getGBuffer().textures[wnormalIndex]);
+        return true;
+    }
+
+    private _render() {
+        if (this.enabled && this._outputTexture.isReady() && this._outputTexture.getEffect()?.isReady()) {
+            if (this._setBindings()) {
+                this._outputTexture.render();
+            }
+        }
     }
 
+    private _renderWhenGBufferReady: Nullable<(eventData: number, eventState: EventState) => void> = null;
+
     /**
      * Called by render pipeline when canvas resized.
      * @param scaleFactor The factor by which to scale the canvas size.
@@ -199,6 +205,10 @@ export class _IblShadowsSpatialBlurPass {
             width: Math.max(1.0, Math.floor(this._engine.getRenderWidth() * scaleFactor)),
             height: Math.max(1.0, Math.floor(this._engine.getRenderHeight() * scaleFactor)),
         };
+        // Don't resize if the size is the same as the current size.
+        if (this._outputTexture.getSize().width === newSize.width && this._outputTexture.getSize().height === newSize.height) {
+            return;
+        }
         this._outputTexture.resize(newSize, false);
     }
 
@@ -214,6 +224,10 @@ export class _IblShadowsSpatialBlurPass {
      * Disposes the associated resources
      */
     public dispose() {
+        if (this._scene.geometryBufferRenderer && this._renderWhenGBufferReady) {
+            const gBuffer = this._scene.geometryBufferRenderer.getGBuffer();
+            gBuffer.onAfterRenderObservable.removeCallback(this._renderWhenGBufferReady);
+        }
         this._outputTexture.dispose();
         if (this._debugPassPP) {
             this._debugPassPP.dispose();

packages/dev/core/src/Rendering/IBLShadows/iblShadowsVoxelTracingPass.ts

@@ -11,6 +11,9 @@ import { GeometryBufferRenderer } from "../../Rendering/geometryBufferRenderer";
 import { ProceduralTexture } from "core/Materials/Textures/Procedurals/proceduralTexture";
 import type { IProceduralTextureCreationOptions } from "core/Materials/Textures/Procedurals/proceduralTexture";
 import type { CubeTexture } from "../../Materials/Textures/cubeTexture";
+import { Logger } from "../../Misc/logger";
+import type { EventState } from "../../Misc/observable";
+import type { Nullable } from "../../types";
 
 /**
  * Build cdf maps for IBL importance sampling during IBL shadow computation.
@@ -322,18 +325,10 @@ export class _IblShadowsVoxelTracingPass {
         this._outputTexture.defines = defines;
         // Need to set all the textures first so that the effect gets created with the proper uniforms.
         this._setBindings(this._scene.activeCamera!);
-
-        let counter = 0;
-        this._scene.onBeforeRenderObservable.add(() => {
-            counter = 0;
-        });
-        this._scene.onAfterRenderTargetsRenderObservable.add(() => {
-            if (++counter == 2) {
-                if (this.enabled && this._outputTexture.isReady()) {
-                    this._setBindings(this._scene.activeCamera!);
-                    this._outputTexture.render();
-                }
-            }
+        this._renderWhenGBufferReady = this._render.bind(this);
+        // Don't start rendering until the first vozelization is done.
+        this._renderPipeline.onVoxelizationCompleteObservable.addOnce(() => {
+            this._scene.geometryBufferRenderer!.getGBuffer().onAfterRenderObservable.add(this._renderWhenGBufferReady);
         });
     }
 
@@ -351,7 +346,7 @@ export class _IblShadowsVoxelTracingPass {
         return defines;
     }
 
-    private _setBindings(camera: Camera) {
+    private _setBindings(camera: Camera): boolean {
         this._outputTexture.defines = this._createDefines();
         this._outputTexture.setMatrix("viewMtx", camera.getViewMatrix());
         this._outputTexture.setMatrix("projMtx", camera.getProjectionMatrix());
@@ -384,23 +379,37 @@ export class _IblShadowsVoxelTracingPass {
         this._outputTexture.setTexture("voxelGridSampler", voxelGrid);
         this._outputTexture.setTexture("blueNoiseSampler", this._renderPipeline!._getNoiseTexture());
         const cdfGenerator = this._scene.iblCdfGenerator;
-        if (cdfGenerator) {
-            this._outputTexture.setTexture("icdfSampler", cdfGenerator.getIcdfTexture());
+        if (!cdfGenerator) {
+            Logger.Warn("IBLShadowsVoxelTracingPass: Can't bind for render because iblCdfGenerator is not enabled.");
+            return false;
         }
+        this._outputTexture.setTexture("icdfSampler", cdfGenerator.getIcdfTexture());
         if (this._coloredShadows && this._scene.environmentTexture) {
             this._outputTexture.setTexture("iblSampler", this._scene.environmentTexture);
         }
 
         const geometryBufferRenderer = this._scene.geometryBufferRenderer;
         if (!geometryBufferRenderer) {
-            return;
+            Logger.Warn("IBLShadowsVoxelTracingPass: Can't bind for render because GeometryBufferRenderer is not enabled.");
+            return false;
         }
         const depthIndex = geometryBufferRenderer.getTextureIndex(GeometryBufferRenderer.SCREENSPACE_DEPTH_TEXTURE_TYPE);
         this._outputTexture.setTexture("depthSampler", geometryBufferRenderer.getGBuffer().textures[depthIndex]);
         const wnormalIndex = geometryBufferRenderer.getTextureIndex(GeometryBufferRenderer.NORMAL_TEXTURE_TYPE);
         this._outputTexture.setTexture("worldNormalSampler", geometryBufferRenderer.getGBuffer().textures[wnormalIndex]);
+        return true;
     }
 
+    private _render() {
+        if (this.enabled && this._outputTexture.isReady() && this._outputTexture.getEffect()?.isReady()) {
+            if (this._setBindings(this._scene.activeCamera!)) {
+                this._outputTexture.render();
+            }
+        }
+    }
+
+    private _renderWhenGBufferReady: Nullable<(eventData: number, eventState: EventState) => void> = null;
+
     /**
      * Called by render pipeline when canvas resized.
      * @param scaleFactor The factor by which to scale the canvas size.
@@ -410,6 +419,10 @@ export class _IblShadowsVoxelTracingPass {
             width: Math.max(1.0, Math.floor(this._engine.getRenderWidth() * scaleFactor)),
             height: Math.max(1.0, Math.floor(this._engine.getRenderHeight() * scaleFactor)),
         };
+        // Don't resize if the size is the same as the current size.
+        if (this._outputTexture.getSize().width === newSize.width && this._outputTexture.getSize().height === newSize.height) {
+            return;
+        }
         this._outputTexture.resize(newSize, false);
     }
 
@@ -431,6 +444,10 @@ export class _IblShadowsVoxelTracingPass {
      * Disposes the associated resources
      */
     public dispose() {
+        if (this._scene.geometryBufferRenderer && this._renderWhenGBufferReady) {
+            const gBuffer = this._scene.geometryBufferRenderer.getGBuffer();
+            gBuffer.onAfterRenderObservable.removeCallback(this._renderWhenGBufferReady);
+        }
         this._outputTexture.dispose();
         if (this._debugPassPP) {
             this._debugPassPP.dispose();