FrameGraph: add support for utility layer renderers (#16185)

* Add suport for utility layer renderers to frame graph

* Fix inspector gizmos when using a frame graph at scene level

packages/dev/core/src/FrameGraph/Node/Blocks/Rendering/utilityLayerRendererBlock.ts

@@ -0,0 +1,102 @@
+// eslint-disable-next-line import/no-internal-modules
+import type { NodeRenderGraphConnectionPoint, Scene, FrameGraph, NodeRenderGraphBuildState, FrameGraphTextureHandle, Camera } from "core/index";
+import { RegisterClass } from "../../../../Misc/typeStore";
+import { editableInPropertyPage, PropertyTypeForEdition } from "../../../../Decorators/nodeDecorator";
+import { NodeRenderGraphBlockConnectionPointTypes } from "../../Types/nodeRenderGraphTypes";
+import { NodeRenderGraphBlock } from "../../nodeRenderGraphBlock";
+import { FrameGraphUtilityLayerRendererTask } from "../../../Tasks/Rendering/utilityLayerRendererTask";
+
+/**
+ * Block used to render an utility layer in the frame graph
+ */
+export class NodeRenderGraphUtilityLayerRendererBlock extends NodeRenderGraphBlock {
+    protected override _frameGraphTask: FrameGraphUtilityLayerRendererTask;
+
+    /**
+     * Gets the frame graph task associated with this block
+     */
+    public override get task() {
+        return this._frameGraphTask;
+    }
+
+    /**
+     * Creates a new NodeRenderGraphUtilityLayerRendererBlock
+     * @param name defines the block name
+     * @param frameGraph defines the hosting frame graph
+     * @param scene defines the hosting scene
+     * @param handleEvents If the utility layer should handle events.
+     */
+    public constructor(name: string, frameGraph: FrameGraph, scene: Scene, handleEvents = true) {
+        super(name, frameGraph, scene);
+
+        this._additionalConstructionParameters = [handleEvents];
+
+        this.registerInput("destination", NodeRenderGraphBlockConnectionPointTypes.Texture);
+        this.registerInput("camera", NodeRenderGraphBlockConnectionPointTypes.Camera);
+        this._addDependenciesInput();
+        this.registerOutput("output", NodeRenderGraphBlockConnectionPointTypes.BasedOnInput);
+
+        this.destination.addAcceptedConnectionPointTypes(NodeRenderGraphBlockConnectionPointTypes.TextureAll);
+        this.output._typeConnectionSource = this.destination;
+
+        this._frameGraphTask = new FrameGraphUtilityLayerRendererTask(name, frameGraph, scene, handleEvents);
+    }
+
+    private _createTask(handleEvents: boolean) {
+        this._frameGraphTask.dispose();
+
+        this._frameGraphTask = new FrameGraphUtilityLayerRendererTask(this.name, this._frameGraph, this._scene, handleEvents);
+
+        this._additionalConstructionParameters = [handleEvents];
+    }
+
+    /** If the utility layer should handle events */
+    @editableInPropertyPage("Handle events", PropertyTypeForEdition.Boolean, "PROPERTIES")
+    public get handleEvents() {
+        return this._frameGraphTask.layer.handleEvents;
+    }
+
+    public set handleEvents(value: boolean) {
+        this._createTask(value);
+    }
+
+    /**
+     * Gets the current class name
+     * @returns the class name
+     */
+    public override getClassName() {
+        return "NodeRenderGraphUtilityLayerRendererBlock";
+    }
+
+    /**
+     * Gets the destination input component
+     */
+    public get destination(): NodeRenderGraphConnectionPoint {
+        return this._inputs[0];
+    }
+
+    /**
+     * Gets the camera input component
+     */
+    public get camera(): NodeRenderGraphConnectionPoint {
+        return this._inputs[1];
+    }
+
+    /**
+     * Gets the output component
+     */
+    public get output(): NodeRenderGraphConnectionPoint {
+        return this._outputs[0];
+    }
+
+    protected override _buildBlock(state: NodeRenderGraphBuildState) {
+        super._buildBlock(state);
+
+        this.output.value = this._frameGraphTask.outputTexture;
+
+        this._frameGraphTask.destinationTexture = this.destination.connectedPoint?.value as FrameGraphTextureHandle;
+        this._frameGraphTask.camera = this.camera.connectedPoint?.value as Camera;
+    }
+}
+
+RegisterClass("BABYLON.NodeRenderGraphUtilityLayerRendererBlock", NodeRenderGraphUtilityLayerRendererBlock);

packages/dev/core/src/FrameGraph/Node/Blocks/index.ts

@@ -21,6 +21,7 @@ export * from "./Rendering/geometryRendererBlock";
 export * from "./Rendering/objectRendererBlock";
 export * from "./Rendering/shadowGeneratorBlock";
 export * from "./Rendering/taaObjectRendererBlock";
+export * from "./Rendering/utilityLayerRendererBlock";
 
 export * from "./Teleport/teleportInBlock";
 export * from "./Teleport/teleportOutBlock";

packages/dev/core/src/FrameGraph/Tasks/Rendering/utilityLayerRendererTask.ts

@@ -0,0 +1,73 @@
+// eslint-disable-next-line import/no-internal-modules
+import type { Camera, FrameGraph, FrameGraphTextureHandle, Scene } from "core/index";
+import { FrameGraphTask } from "../../frameGraphTask";
+import { UtilityLayerRenderer } from "core/Rendering/utilityLayerRenderer";
+
+/**
+ * Task used to render an utility layer.
+ */
+export class FrameGraphUtilityLayerRendererTask extends FrameGraphTask {
+    /**
+     * The destination texture of the task.
+     */
+    public destinationTexture: FrameGraphTextureHandle;
+
+    /**
+     * The camera used to render the utility layer.
+     */
+    public camera: Camera;
+
+    /**
+     * The output texture of the task.
+     * This is the same texture as the destination texture, but the handles are different!
+     */
+    public readonly outputTexture: FrameGraphTextureHandle;
+
+    /**
+     * The utility layer renderer.
+     */
+    public readonly layer: UtilityLayerRenderer;
+
+    /**
+     * Creates a new utility layer renderer task.
+     * @param name The name of the task.
+     * @param frameGraph The frame graph the task belongs to.
+     * @param scene The scene the task belongs to.
+     * @param handleEvents If the utility layer should handle events.
+     */
+    constructor(name: string, frameGraph: FrameGraph, scene: Scene, handleEvents = true) {
+        super(name, frameGraph);
+
+        this.layer = new UtilityLayerRenderer(scene, handleEvents, true);
+        this.layer.utilityLayerScene._useCurrentFrameBuffer = true;
+
+        this.outputTexture = this._frameGraph.textureManager.createDanglingHandle();
+    }
+
+    public record(): void {
+        if (!this.destinationTexture || !this.camera) {
+            throw new Error("FrameGraphUtilityLayerRendererTask: destinationTexture and camera are required");
+        }
+
+        this._frameGraph.textureManager.resolveDanglingHandle(this.outputTexture, this.destinationTexture);
+
+        const pass = this._frameGraph.addRenderPass(this.name);
+
+        pass.setRenderTarget(this.outputTexture);
+        pass.setExecuteFunc((context) => {
+            this.layer.setRenderCamera(this.camera);
+
+            context.render(this.layer);
+        });
+
+        const passDisabled = this._frameGraph.addRenderPass(this.name + "_disabled", true);
+
+        passDisabled.setRenderTarget(this.outputTexture);
+        passDisabled.setExecuteFunc((_context) => {});
+    }
+
+    public override dispose(): void {
+        this.layer.dispose();
+        super.dispose();
+    }
+}

packages/dev/core/src/FrameGraph/frameGraph.ts

@@ -50,6 +50,13 @@ export class FrameGraph {
         return this._engine;
     }
 
+    /**
+     * Gets the list of tasks in the frame graph
+     */
+    public get tasks() {
+        return this._tasks;
+    }
+
     /**
      * Constructs the frame graph
      * @param engine defines the hosting engine

packages/dev/core/src/FrameGraph/frameGraphRenderContext.ts

@@ -11,6 +11,7 @@ import type {
     Scene,
     FrameGraphRenderTarget,
     InternalTexture,
+    UtilityLayerRenderer,
     // eslint-disable-next-line import/no-internal-modules
 } from "core/index";
 import { Constants } from "../Engines/constants";
@@ -32,7 +33,7 @@ export class FrameGraphRenderContext extends FrameGraphContext {
     private _depthTest: boolean;
     private _depthWrite: boolean;
 
-    private static _IsObjectRenderer(value: Layer | ObjectRenderer): value is ObjectRenderer {
+    private static _IsObjectRenderer(value: Layer | ObjectRenderer | UtilityLayerRenderer): value is ObjectRenderer {
         return (value as ObjectRenderer).initRender !== undefined;
     }
 
@@ -262,7 +263,7 @@ export class FrameGraphRenderContext extends FrameGraphContext {
      * @param viewportWidth The width of the viewport (optional for Layer, but mandatory for ObjectRenderer)
      * @param viewportHeight The height of the viewport (optional for Layer, but mandatory for ObjectRenderer)
      */
-    public render(object: Layer | ObjectRenderer, viewportWidth?: number, viewportHeight?: number): void {
+    public render(object: Layer | ObjectRenderer | UtilityLayerRenderer, viewportWidth?: number, viewportHeight?: number): void {
         if (FrameGraphRenderContext._IsObjectRenderer(object)) {
             if (object.shouldRender()) {
                 this._scene.incrementRenderId();

packages/dev/core/src/FrameGraph/index.ts

@@ -35,6 +35,7 @@ export * from "./Tasks/Rendering/geometryRendererTask";
 export * from "./Tasks/Rendering/objectRendererTask";
 export * from "./Tasks/Rendering/shadowGeneratorTask";
 export * from "./Tasks/Rendering/taaObjectRendererTask";
+export * from "./Tasks/Rendering/utilityLayerRendererTask";
 
 export * from "./frameGraph";
 export * from "./frameGraphContext";

packages/dev/core/src/Rendering/utilityLayerRenderer.ts

@@ -152,11 +152,13 @@ export class UtilityLayerRenderer implements IDisposable {
      * Instantiates a UtilityLayerRenderer
      * @param originalScene the original scene that will be rendered on top of
      * @param handleEvents boolean indicating if the utility layer should handle events
+     * @param manualRender boolean indicating if the utility layer should render manually.
      */
     constructor(
         /** the original scene that will be rendered on top of */
         public originalScene: Scene,
-        handleEvents: boolean = true
+        public readonly handleEvents: boolean = true,
+        manualRender = false
     ) {
         // Create scene which will be rendered in the foreground and remove it from being referenced by engine to avoid interfering with existing app
         this.utilityLayerScene = new Scene(originalScene.getEngine(), { virtual: true });
@@ -327,12 +329,14 @@ export class UtilityLayerRenderer implements IDisposable {
         // Render directly on top of existing scene without clearing
         this.utilityLayerScene.autoClear = false;
 
-        this._afterRenderObserver = this.originalScene.onAfterRenderCameraObservable.add((camera) => {
-            // Only render when the render camera finishes rendering
-            if (this.shouldRender && camera == this.getRenderCamera()) {
-                this.render();
-            }
-        });
+        if (!manualRender) {
+            this._afterRenderObserver = this.originalScene.onAfterRenderCameraObservable.add((camera) => {
+                // Only render when the render camera finishes rendering
+                if (this.shouldRender && camera == this.getRenderCamera()) {
+                    this.render();
+                }
+            });
+        }
 
         this._sceneDisposeObserver = this.originalScene.onDisposeObservable.add(() => {
             this.dispose();

packages/dev/core/src/scene.ts

@@ -4412,14 +4412,19 @@ export class Scene implements IAnimatable, IClipPlanesHolder, IAssetContainer {
         }
     }
 
+    /** @internal */
+    public _useCurrentFrameBuffer = false;
+
     private _bindFrameBuffer(camera: Nullable<Camera>, clear = true) {
-        if (camera && camera._multiviewTexture) {
-            camera._multiviewTexture._bindFrameBuffer();
-        } else if (camera && camera.outputRenderTarget) {
-            camera.outputRenderTarget._bindFrameBuffer();
-        } else {
-            if (!this._engine._currentFrameBufferIsDefaultFrameBuffer()) {
-                this._engine.restoreDefaultFramebuffer();
+        if (!this._useCurrentFrameBuffer) {
+            if (camera && camera._multiviewTexture) {
+                camera._multiviewTexture._bindFrameBuffer();
+            } else if (camera && camera.outputRenderTarget) {
+                camera.outputRenderTarget._bindFrameBuffer();
+            } else {
+                if (!this._engine._currentFrameBufferIsDefaultFrameBuffer()) {
+                    this._engine.restoreDefaultFramebuffer();
+                }
             }
         }
         if (clear) {

packages/dev/gui/src/2D/FrameGraph/renderGraphGUIBlock.ts

@@ -77,14 +77,9 @@ export class NodeRenderGraphGUIBlock extends NodeRenderGraphBlock {
     protected override _buildBlock(state: NodeRenderGraphBuildState) {
         super._buildBlock(state);
 
-        this._frameGraphTask.name = this.name;
-
         this.output.value = this._frameGraphTask.outputTexture; // the value of the output connection point is the "output" texture of the task
 
-        const destinationConnectedPoint = this.destination.connectedPoint;
-        if (destinationConnectedPoint) {
-            this._frameGraphTask.destinationTexture = destinationConnectedPoint.value as FrameGraphTextureHandle;
-        }
+        this._frameGraphTask.destinationTexture = this.destination.connectedPoint?.value as FrameGraphTextureHandle;
     }
 }