RFC/POC: support float16 postprocess buffers

modules/core/src/index.ts

@@ -108,6 +108,7 @@ export type {LayersList, LayerContext} from './lib/layer-manager';
 export type {ViewStateMap} from './lib/view-manager';
 export type {UpdateParameters} from './lib/layer';
 export type {DeckProps} from './lib/deck';
+export type {PostProcessColorFormat} from './lib/deck-renderer';
 export type {
   LayerProps,
   CompositeLayerProps,

modules/core/src/lib/deck-renderer.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {CanvasContext, Device, PresentationContext} from '@luma.gl/core';
+import type {CanvasContext, Device, PresentationContext, TextureFormatColor} from '@luma.gl/core';
 import {Framebuffer} from '@luma.gl/core';
 import debug from '../debug/index';
 import DrawLayersPass from '../passes/draw-layers-pass';
@@ -19,6 +19,8 @@ const TRACE_RENDER_LAYERS = 'deckRenderer.renderLayers';
 
 type LayerFilter = ((context: FilterContext) => boolean) | null;
 
+export type PostProcessColorFormat = Extract<TextureFormatColor, 'rgba8unorm' | 'rgba16float'>;
+
 export default class DeckRenderer {
   device: Device;
   layerFilter: LayerFilter;
@@ -31,6 +33,7 @@ export default class DeckRenderer {
   private _needsRedraw: string | false;
   private renderBuffers: Framebuffer[];
   private lastPostProcessEffect: string | null;
+  private postProcessColorFormat: PostProcessColorFormat;
 
   constructor(device: Device, opts: {stats?: Stats} = {}) {
     this.device = device;
@@ -43,9 +46,14 @@ export default class DeckRenderer {
     this._needsRedraw = 'Initial render';
     this.renderBuffers = [];
     this.lastPostProcessEffect = null;
+    this.postProcessColorFormat = 'rgba8unorm';
   }
 
-  setProps(props: {layerFilter: LayerFilter; drawPickingColors: boolean}) {
+  setProps(props: {
+    layerFilter: LayerFilter;
+    drawPickingColors: boolean;
+    postProcessColorFormat?: PostProcessColorFormat;
+  }) {
     if (this.layerFilter !== props.layerFilter) {
       this.layerFilter = props.layerFilter;
       this._needsRedraw = 'layerFilter changed';
@@ -55,6 +63,13 @@ export default class DeckRenderer {
       this.drawPickingColors = props.drawPickingColors;
       this._needsRedraw = 'drawPickingColors changed';
     }
+
+    const postProcessColorFormat = props.postProcessColorFormat || 'rgba8unorm';
+    if (this.postProcessColorFormat !== postProcessColorFormat) {
+      this.postProcessColorFormat = postProcessColorFormat;
+      this._deleteRenderBuffers();
+      this._needsRedraw = 'postProcessColorFormat changed';
+    }
   }
 
   renderLayers(opts: {
@@ -119,6 +134,10 @@ export default class DeckRenderer {
   }
 
   finalize() {
+    this._deleteRenderBuffers();
+  }
+
+  private _deleteRenderBuffers() {
     const {renderBuffers} = this;
     for (const buffer of renderBuffers) {
       buffer.delete();
@@ -156,11 +175,19 @@ export default class DeckRenderer {
     const size = canvasContext.getDrawingBufferSize();
     const [width, height] = size;
     if (renderBuffers.length === 0) {
+      const {postProcessColorFormat} = this;
+      const capabilities = this.device.getTextureFormatCapabilities(postProcessColorFormat);
+      if (!capabilities.render || !capabilities.filter) {
+        throw new Error(
+          `Postprocess color format ${postProcessColorFormat} must be renderable and filterable on this device`
+        );
+      }
       [0, 1].map(i => {
         const texture = this.device.createTexture({
           sampler: {minFilter: 'linear', magFilter: 'linear'},
           width,
-          height
+          height,
+          format: postProcessColorFormat
         });
         renderBuffers.push(
           this.device.createFramebuffer({

modules/core/src/lib/deck.ts

@@ -6,7 +6,7 @@ import LayerManager from './layer-manager';
 import ViewManager, {DEFAULT_CANVAS_ID} from './view-manager';
 import MapView from '../views/map-view';
 import EffectManager from './effect-manager';
-import DeckRenderer from './deck-renderer';
+import DeckRenderer, {type PostProcessColorFormat} from './deck-renderer';
 import DeckPicker from './deck-picker';
 import {Widget} from './widget';
 import {WidgetManager} from './widget-manager';
@@ -123,6 +123,12 @@ export type DeckProps<ViewsT extends ViewOrViews = null> = {
 
   /** WebGL parameters to be set before each frame is rendered. */
   parameters?: Parameters;
+  /**
+   * Color format of deck.gl's intermediate framebuffers when postprocessing effects are enabled.
+   * Set to `'rgba16float'` to preserve HDR highlights for effects such as bloom.
+   * @default `'rgba8unorm'`
+   */
+  postProcessColorFormat?: PostProcessColorFormat;
   /** If supplied, will be called before a layer is drawn to determine whether it should be rendered. */
   layerFilter?: ((context: FilterContext) => boolean) | null;
 
@@ -253,6 +259,7 @@ const defaultProps: DeckProps = {
   pickAsync: 'auto',
   layerFilter: null,
   parameters: {},
+  postProcessColorFormat: 'rgba8unorm',
   parent: null,
   device: null,
   deviceProps: {} as DeviceProps,

test/modules/core/lib/deck-renderer.spec.ts

@@ -50,3 +50,73 @@ test('DeckRenderer#post processing renders to the supplied canvas framebuffer',
   deckRenderer.finalize();
   framebuffer.destroy();
 });
+
+test('DeckRenderer#uses the configured postprocess color format', ({skip}) => {
+  const capabilities = device.getTextureFormatCapabilities('rgba16float');
+  if (!capabilities.render || !capabilities.filter) {
+    skip();
+  }
+
+  const framebuffer = device.createFramebuffer({
+    width: 60,
+    height: 40,
+    colorAttachments: ['rgba8unorm']
+  });
+  const canvasContext = {
+    getCurrentFramebuffer: () => framebuffer,
+    getDrawingBufferSize: () => [60, 40],
+    cssToDeviceRatio: () => 1
+  } as CanvasContext;
+  const effect: Effect = {
+    id: 'postprocess-effect',
+    props: {},
+    setup() {},
+    cleanup() {},
+    preRender() {},
+    postRender({target, swapBuffer}: PostRenderOptions) {
+      return target || swapBuffer;
+    }
+  };
+  const deckRenderer = new DeckRenderer(device);
+
+  deckRenderer.setProps({
+    layerFilter: null,
+    drawPickingColors: false,
+    postProcessColorFormat: 'rgba16float'
+  });
+  deckRenderer.renderLayers({
+    pass: 'screen',
+    layers: [],
+    viewports: [new Viewport({id: 'float-postprocess-view', width: 60, height: 40})],
+    views: {},
+    onViewportActive: () => {},
+    effects: [effect],
+    target: null,
+    canvasContext
+  });
+
+  const getRenderBufferFormats = () =>
+    // @ts-expect-error Access private state to verify the allocated attachment formats.
+    deckRenderer.renderBuffers.map(buffer => buffer.colorAttachments[0].texture.format);
+  expect(getRenderBufferFormats()).toEqual(['rgba16float', 'rgba16float']);
+
+  deckRenderer.setProps({
+    layerFilter: null,
+    drawPickingColors: false,
+    postProcessColorFormat: 'rgba8unorm'
+  });
+  deckRenderer.renderLayers({
+    pass: 'screen',
+    layers: [],
+    viewports: [new Viewport({id: 'unorm-postprocess-view', width: 60, height: 40})],
+    views: {},
+    onViewportActive: () => {},
+    effects: [effect],
+    target: null,
+    canvasContext
+  });
+  expect(getRenderBufferFormats()).toEqual(['rgba8unorm', 'rgba8unorm']);
+
+  deckRenderer.finalize();
+  framebuffer.destroy();
+});