Aggregator and AggregationLayer

modules/aggregation-layers/src/aggregation-layer-v9/aggregation-layer.ts

@@ -0,0 +1,102 @@
+import {
+  CompositeLayer,
+  LayerDataSource,
+  LayerContext,
+  UpdateParameters,
+  CompositeLayerProps,
+  Attribute
+} from '@deck.gl/core';
+import {Aggregator} from './aggregator';
+
+// TODO
+type GPUAggregator = Aggregator & {destroy(): void};
+// TODO
+type CPUAggregator = Aggregator;
+
+export type AggregationLayerProps<DataT> = CompositeLayerProps & {
+  data: LayerDataSource<DataT>;
+};
+
+export default abstract class AggregationLayer<
+  DataT,
+  ExtraPropsT extends {} = {}
+> extends CompositeLayer<Required<AggregationLayer<DataT>> & ExtraPropsT> {
+  static layerName = 'AggregationLayer';
+
+  state!: {
+    gpuAggregator: GPUAggregator | null;
+    cpuAggregator: CPUAggregator | null;
+  };
+
+  /** Allow this layer to have an AttributeManager and participates in the draw cycle */
+  get isDrawable() {
+    return true;
+  }
+
+  /** Called to create a GPUAggregator instance */
+  abstract getGPUAggregator(): GPUAggregator | null;
+  /** Called to create a CPUAggregator instance if getGPUAggregator() returns null */
+  abstract getCPUAggregator(): CPUAggregator | null;
+  /** Called when some attributes change, a chance to mark Aggregator as dirty */
+  abstract onAttributeChange(id: string): void;
+
+  initializeState(): void {
+    this.getAttributeManager()!.remove(['instancePickingColors']);
+  }
+
+  // Override Layer.updateState to update the GPUAggregator instance
+  updateState(params: UpdateParameters<this>) {
+    super.updateState(params);
+
+    if (params.changeFlags.extensionsChanged) {
+      this.state.gpuAggregator?.destroy();
+      this.state.gpuAggregator = this.getGPUAggregator();
+      if (this.state.gpuAggregator) {
+        this.getAttributeManager()!.invalidateAll();
+      } else if (!this.state.cpuAggregator) {
+        this.state.cpuAggregator = this.getCPUAggregator();
+      }
+    }
+  }
+
+  // Override Layer.finalizeState to dispose the GPUAggregator instance
+  finalizeState(context: LayerContext) {
+    super.finalizeState(context);
+    this.state.gpuAggregator?.destroy();
+  }
+
+  // Override Layer.updateAttributes to update the aggregator
+  protected updateAttributes(changedAttributes: {[id: string]: Attribute}) {
+    this.getAggregator()?.setProps({
+      attributes: changedAttributes
+    });
+
+    for (const id in changedAttributes) {
+      this.onAttributeChange(id);
+    }
+  }
+
+  draw({moduleParameters}) {
+    // GPU aggregation needs `moduleSettings` for projection/filter uniforms which are only accessible at draw time
+    // GPUAggregator's Buffers are allocated during `updateState`/`GPUAggregator.setProps`
+    // and passed down to the sublayer attributes in renderLayers()
+    // Although the Buffers have been bound to the sublayer's Model, their content are not populated yet
+    // GPUAggregator.update() is called in the draw cycle here right before Buffers are used by sublayer.draw()
+    this.state.gpuAggregator?.update({moduleSettings: moduleParameters});
+  }
+
+  protected getAggregator(): Aggregator | null {
+    return this.state.gpuAggregator || this.state.cpuAggregator;
+  }
+
+  // Override CompositeLayer._postUpdate to update attributes and the CPUAggregator
+  protected _postUpdate(updateParams: UpdateParameters<this>, forceUpdate: boolean) {
+    this._updateAttributes();
+    // CPUAggregator.update() must be called before renderLayers()
+    // CPUAggregator's outputs are Float32Array whose content is applied during the `updateState` lifecycle
+    // The typed arrays are passed to the sublayer's attributes and uploaded to GPU Buffers during the sublayer's update
+    // therefore they must be up to date before renderLayers()
+    this.state.cpuAggregator?.update();
+    super._postUpdate(updateParams, forceUpdate);
+  }
+}

modules/aggregation-layers/src/aggregation-layer-v9/aggregator.ts

@@ -0,0 +1,65 @@
+import type {Attribute, BinaryAttribute} from '@deck.gl/core';
+
+export type AggregationOperation = 'SUM' | 'MEAN' | 'MIN' | 'MAX';
+
+export type AggregationProps = {
+  /** Number of data points */
+  pointCount: number;
+  /** The input data */
+  attributes: {[id: string]: Attribute};
+  /** How to aggregate getWeights, defined for each channel */
+  operations: AggregationOperation[];
+  /** Additional options to control bin sorting, e.g. bin size */
+  binOptions: Record<string, number | number[]>;
+};
+
+/**
+ * _Aggregation_ is a 2-step process:
+ * 1. Sort: Group a collection of _data points_ by some property into _bins_.
+ * 2. Aggregate: for each _bin_, calculate one or more metrics (_channels_) from all its members.
+ *
+ * An implementation of the _Aggregator_ interface takes the following inputs:
+ * - The number of data points
+ * - The group that each data point belongs to, by mapping each data point to a _binId_ (integer or array of integers)
+ * - The value(s) to aggregate, by mapping each data point in each channel to one _weight_
+ * - The method (_aggregationOperation_) to reduce a list of _weights_ to one number, such as SUM
+ *
+ * And yields the following outputs:
+ * - The aggregated values (_result_) as a list of numbers for each channel, comprised of one number per bin
+ * - The [min, max] among all aggregated values (_domain_) for each channel
+ *
+ */
+export interface Aggregator {
+  /** Update aggregation props */
+  setProps(props: Partial<AggregationProps>): void;
+
+  /** Flags a channel to need update
+   * @param {number} channel - mark the given channel as dirty. If not provided, all channels will be updated.
+   */
+  setNeedsUpdate(channel?: number): void;
+
+  /** Run aggregation */
+  update(params?: unknown): void;
+
+  /** Get the number of bins */
+  get numBins(): number;
+
+  /** Returns an accessor to the bins. */
+  getBins(): BinaryAttribute | null;
+
+  /** Returns an accessor to the output for a given channel. */
+  getResult(channel: number): BinaryAttribute | null;
+
+  /** Returns the [min, max] of aggregated values for a given channel. */
+  getResultDomain(channel: number): [min: number, max: number];
+
+  /** Returns the information for a given bin. */
+  getBin(index: number): {
+    /** The original id */
+    id: number | number[];
+    /** Aggregated values by channel */
+    value: number[];
+    /** Count of data points in this bin */
+    count: number;
+  } | null;
+}

modules/core/src/lib/composite-layer.ts

@@ -21,7 +21,6 @@ import Layer, {UpdateParameters} from './layer';
 import debug from '../debug/index';
 import {flatten} from '../utils/flatten';
 
-import type AttributeManager from './attribute/attribute-manager';
 import type {PickingInfo, GetPickingInfoParams} from './picking/pick-info';
 import type {FilterContext} from '../passes/layers-pass';
 import type {LayersList, LayerContext} from './layer-manager';
@@ -41,6 +40,11 @@ export default abstract class CompositeLayer<PropsT extends {} = {}> extends Lay
     return true;
   }
 
+  /** `true` if the layer renders to screen */
+  get isDrawable(): boolean {
+    return false;
+  }
+
   /** Returns true if all async resources are loaded */
   get isLoaded(): boolean {
     return super.isLoaded && this.getSubLayers().every(layer => layer.isLoaded);
@@ -253,11 +257,6 @@ export default abstract class CompositeLayer<PropsT extends {} = {}> extends Lay
     }
   }
 
-  /** Override base Layer method */
-  protected _getAttributeManager(): AttributeManager | null {
-    return null;
-  }
-
   /** (Internal) Called after an update to rerender sub layers */
   protected _postUpdate(updateParams: UpdateParameters<this>, forceUpdate: boolean) {
     // @ts-ignore (TS2531) this method is only called internally when internalState is defined

modules/core/src/lib/layer.ts

@@ -285,6 +285,11 @@ export default abstract class Layer<PropsT extends {} = {}> extends Component<
     return false;
   }
 
+  /** `true` if the layer renders to screen */
+  get isDrawable(): boolean {
+    return true;
+  }
+
   /** Updates selected state members and marks the layer for redraw */
   setState(partialState: any): void {
     this.setChangeFlags({stateChanged: true});
@@ -879,7 +884,7 @@ export default abstract class Layer<PropsT extends {} = {}> extends Component<
 
     debug(TRACE_INITIALIZE, this);
 
-    const attributeManager = this._getAttributeManager();
+    const attributeManager = this.isDrawable ? this._getAttributeManager() : null;
 
     if (attributeManager) {
       // All instanced layers get instancePickingColors attribute by default

modules/core/src/passes/layers-pass.ts

@@ -251,7 +251,8 @@ export default class LayersPass extends Pass {
       }
       if (layer.isComposite) {
         renderStatus.compositeCount++;
-      } else if (shouldDrawLayer) {
+      }
+      if (layer.isDrawable && shouldDrawLayer) {
         // Draw the layer
         renderStatus.visibleCount++;