Implement 'images in labels' feature (#8904)

* Set 'isSDF' flag per icon quad

* Simplify formatted expression parsing
  Allow omission of empty section options objects. This enables
  following syntax ["format", "first section\n",
                              "second section\n",
                              ["image", "smiley"],
                              "third section"]

* Add Image to Format expression
  Image could be part of Format expression and evaluated ResolvedImage
  is a optional member of evaluated Formatted type.

* Add formatted text's image section image to deps

* Add images to TaggedString's section

* Pass images to text shaping

* Use {text|icon}-size vertex attribute field to encode 'isSDF' flag
  This change uses higher bit to indicate whether quad's vertices
  belong to a quad whose texture should be rasterized using SDF function.
  Size scaling factor is changed from 256 to 128, which reduces resolution
  a bit, however, as the scaling result is now being rounded, final result
  should be the same.

* Shape images in labels

* Draw color icons and sdf text with one program / draw call

* Align images and scaled glyphs in vertical layout

* Allow image breaking

* Update render test expectations for tests using font-scale
  Fixes misaligned text boxes

* Update format expression tests to account for image sections

* Update quads unit test

* Update shaping unit test

* Update unit test for format expression API

* Fix linter errors

* Don't scale images in labels with text size

* Basic render test

* Multiline test

* Vertical layout with images

* Line placement test

* Expression test for image section

* Format expression unit and API test

* Shaping unit test

* Add benchmarks for symbol layer with icons

* Use explicit 'symbol_text_and_icon' shader name instead of 'symbol'
  Add is_text back, in follow-up pr shader can be renamed to sdf and
  raster, and can be used to draw sdf and raster icons in one pass.

* Don't use mod for unpacking flag

* Use linear interpolation for images in label for zoom dependent text size

* Add back glyphMap to shaping
  Glyph map is used as a fallback whenever glyph is missing in glyph
  positions, e.g., space character that may not be backed by a texture,
  yet, has a glyph metrics.

* Render test for constant text-size

* Render test for image justification

* Take into account image padding and image pixel ratio

* Add render test to verify that linear filtering is used
  Test that linear filtering is used for icons in text when
  text-size is zoom-dependent.

* Fix size of images in text at integer zoom levels

* Update shaping unit test and remove unused parameter from shapeLines

bench/benchmarks/layers.js

@@ -220,3 +220,22 @@ export class LayerSymbol extends LayerBenchmark {
         });
     }
 }
+
+export class LayerSymbolWithIcons extends LayerBenchmark {
+    constructor() {
+        super();
+
+        this.layerStyle = Object.assign({}, style, {
+            layers: generateLayers({
+                'id': 'symbollayer',
+                'type': 'symbol',
+                'source': 'composite',
+                'source-layer': 'poi_label',
+                'layout': {
+                    'icon-image': 'dot-11',
+                    'text-field': ['format', ['get', 'name_en'], ['image', 'dot-11']]
+                }
+            })
+        });
+    }
+}

bench/versions/benchmarks.js

@@ -9,7 +9,7 @@ import WorkerTransfer from '../benchmarks/worker_transfer';
 import Paint from '../benchmarks/paint';
 import PaintStates from '../benchmarks/paint_states';
 import {PropertyLevelRemove, FeatureLevelRemove, SourceLevelRemove} from '../benchmarks/remove_paint_state';
-import {LayerBackground, LayerCircle, LayerFill, LayerFillExtrusion, LayerHeatmap, LayerHillshade, LayerLine, LayerRaster, LayerSymbol} from '../benchmarks/layers';
+import {LayerBackground, LayerCircle, LayerFill, LayerFillExtrusion, LayerHeatmap, LayerHillshade, LayerLine, LayerRaster, LayerSymbol, LayerSymbolWithIcons} from '../benchmarks/layers';
 import Load from '../benchmarks/map_load';
 import Validate from '../benchmarks/style_validate';
 import StyleLayerCreate from '../benchmarks/style_layer_create';
@@ -63,6 +63,7 @@ register('LayerHillshade', new LayerHillshade());
 register('LayerLine', new LayerLine());
 register('LayerRaster', new LayerRaster());
 register('LayerSymbol', new LayerSymbol());
+register('LayerSymbolWithIcons', new LayerSymbolWithIcons());
 register('Load', new Load());
 register('LayoutDDS', new LayoutDDS());
 register('SymbolLayout', new SymbolLayout(style, styleLocations.map(location => location.tileID[0])));

src/data/bucket/symbol_bucket.js

@@ -33,6 +33,7 @@ const vectorTileFeatureTypes = mvt.VectorTileFeature.types;
 import {verticalizedCharacterMap} from '../../util/verticalize_punctuation';
 import Anchor from '../../symbol/anchor';
 import {getSizeData} from '../../symbol/symbol_size';
+import {MAX_PACKED_SIZE} from '../../symbol/symbol_layout';
 import {register} from '../../util/web_worker_transfer';
 import EvaluationParameters from '../../style/evaluation_parameters';
 import Formatted from '../../style-spec/expression/types/formatted';
@@ -101,7 +102,9 @@ const shaderOpacityAttributes = [
     {name: 'a_fade_opacity', components: 1, type: 'Uint8', offset: 0}
 ];
 
-function addVertex(array, anchorX, anchorY, ox, oy, tx, ty, sizeVertex) {
+function addVertex(array, anchorX, anchorY, ox, oy, tx, ty, sizeVertex, isSDF: boolean) {
+    const aSizeX = sizeVertex ? Math.min(MAX_PACKED_SIZE, Math.round(sizeVertex[0])) : 0;
+    const aSizeY = sizeVertex ? Math.min(MAX_PACKED_SIZE, Math.round(sizeVertex[1])) : 0;
     array.emplaceBack(
         // a_pos_offset
         anchorX,
@@ -112,8 +115,8 @@ function addVertex(array, anchorX, anchorY, ox, oy, tx, ty, sizeVertex) {
         // a_data
         tx, // x coordinate of symbol on glyph atlas texture
         ty, // y coordinate of symbol on glyph atlas texture
-        sizeVertex ? sizeVertex[0] : 0,
-        sizeVertex ? sizeVertex[1] : 0
+        (aSizeX << 1) + (isSDF ? 1 : 0),
+        aSizeY
     );
 }
 
@@ -271,6 +274,7 @@ class SymbolBucket implements Bucket {
 
     index: number;
     sdfIcons: boolean;
+    iconsInText: boolean;
     iconsNeedLinear: boolean;
     bucketInstanceId: number;
     justReloaded: boolean;
@@ -381,7 +385,9 @@ class SymbolBucket implements Bucket {
         const textField = layout.get('text-field');
         const iconImage = layout.get('icon-image');
         const hasText =
-            (textField.value.kind !== 'constant' || textField.value.value.toString().length > 0) &&
+            (textField.value.kind !== 'constant' ||
+                (textField.value.value instanceof Formatted && !textField.value.value.isEmpty()) ||
+                textField.value.value.toString().length > 0) &&
             (textFont.value.kind !== 'constant' || textFont.value.value.length > 0);
         // we should always resolve the icon-image value if the property was defined in the style
         // this allows us to fire the styleimagemissing event if image evaluation returns null
@@ -470,10 +476,15 @@ class SymbolBucket implements Bucket {
                 const textAlongLine = layout.get('text-rotation-alignment') === 'map' && layout.get('symbol-placement') !== 'point';
                 this.allowVerticalPlacement = this.writingModes && this.writingModes.indexOf(WritingMode.vertical) >= 0;
                 for (const section of text.sections) {
-                    const doesAllowVerticalWritingMode = allowsVerticalWritingMode(text.toString());
-                    const sectionFont = section.fontStack || fontStack;
-                    const sectionStack = stacks[sectionFont] = stacks[sectionFont] || {};
-                    this.calculateGlyphDependencies(section.text, sectionStack, textAlongLine, this.allowVerticalPlacement, doesAllowVerticalWritingMode);
+                    if (!section.image) {
+                        const doesAllowVerticalWritingMode = allowsVerticalWritingMode(text.toString());
+                        const sectionFont = section.fontStack || fontStack;
+                        const sectionStack = stacks[sectionFont] = stacks[sectionFont] || {};
+                        this.calculateGlyphDependencies(section.text, sectionStack, textAlongLine, this.allowVerticalPlacement, doesAllowVerticalWritingMode);
+                    } else {
+                        // Add section image to the list of dependencies.
+                        icons[section.image.name] = true;
+                    }
                 }
             }
         }
@@ -587,10 +598,10 @@ class SymbolBucket implements Bucket {
             const index = segment.vertexLength;
 
             const y = symbol.glyphOffset[1];
-            addVertex(layoutVertexArray, labelAnchor.x, labelAnchor.y, tl.x, y + tl.y, tex.x, tex.y, sizeVertex);
-            addVertex(layoutVertexArray, labelAnchor.x, labelAnchor.y, tr.x, y + tr.y, tex.x + tex.w, tex.y, sizeVertex);
-            addVertex(layoutVertexArray, labelAnchor.x, labelAnchor.y, bl.x, y + bl.y, tex.x, tex.y + tex.h, sizeVertex);
-            addVertex(layoutVertexArray, labelAnchor.x, labelAnchor.y, br.x, y + br.y, tex.x + tex.w, tex.y + tex.h, sizeVertex);
+            addVertex(layoutVertexArray, labelAnchor.x, labelAnchor.y, tl.x, y + tl.y, tex.x, tex.y, sizeVertex, symbol.isSDF);
+            addVertex(layoutVertexArray, labelAnchor.x, labelAnchor.y, tr.x, y + tr.y, tex.x + tex.w, tex.y, sizeVertex, symbol.isSDF);
+            addVertex(layoutVertexArray, labelAnchor.x, labelAnchor.y, bl.x, y + bl.y, tex.x, tex.y + tex.h, sizeVertex, symbol.isSDF);
+            addVertex(layoutVertexArray, labelAnchor.x, labelAnchor.y, br.x, y + br.y, tex.x + tex.w, tex.y + tex.h, sizeVertex, symbol.isSDF);
 
             addDynamicAttributes(dynamicLayoutVertexArray, labelAnchor, angle);
 

src/render/draw_symbol.js

@@ -20,7 +20,8 @@ import {evaluateVariableOffset} from '../symbol/symbol_layout';
 
 import {
     symbolIconUniformValues,
-    symbolSDFUniformValues
+    symbolSDFUniformValues,
+    symbolTextAndIconUniformValues
 } from './program/symbol_program';
 
 import type Painter from './painter';
@@ -43,7 +44,9 @@ type SymbolTileRenderState = {
         buffers: SymbolBuffers,
         uniformValues: any,
         atlasTexture: Texture,
+        atlasTextureIcon: Texture | null,
         atlasInterpolation: any,
+        atlasInterpolationIcon: any,
         isSDF: boolean,
         hasHalo: boolean
     }
@@ -208,6 +211,16 @@ function updateVariableAnchorsForBucket(bucket, rotateWithMap, pitchWithMap, var
     bucket.text.dynamicLayoutVertexBuffer.updateData(dynamicTextLayoutVertexArray);
 }
 
+function getSymbolProgramName(isSDF: boolean, isText: boolean, bucket: SymbolBucket) {
+    if (bucket.iconsInText && isText) {
+        return 'symbolTextAndIcon';
+    } else if (isSDF) {
+        return 'symbolSDF';
+    } else {
+        return 'symbolIcon';
+    }
+}
+
 function drawLayerSymbols(painter, sourceCache, layer, coords, isText, translate, translateAnchor,
                           rotationAlignment, pitchAlignment, keepUpright, stencilMode, colorMode) {
 
@@ -244,28 +257,33 @@ function drawLayerSymbols(painter, sourceCache, layer, coords, isText, translate
         const isSDF = isText || bucket.sdfIcons;
 
         const sizeData = isText ? bucket.textSizeData : bucket.iconSizeData;
+        const transformed = pitchWithMap || tr.pitch !== 0;
 
         if (!program) {
-            program = painter.useProgram(isSDF ? 'symbolSDF' : 'symbolIcon', programConfiguration);
+            program = painter.useProgram(getSymbolProgramName(isSDF, isText, bucket), programConfiguration);
             size = symbolSize.evaluateSizeForZoom(sizeData, tr.zoom);
         }
 
-        context.activeTexture.set(gl.TEXTURE0);
-
         let texSize: [number, number];
+        let texSizeIcon: [number, number] = [0, 0];
         let atlasTexture;
         let atlasInterpolation;
+        let atlasTextureIcon = null;
+        let atlasInterpolationIcon;
         if (isText) {
             atlasTexture = tile.glyphAtlasTexture;
             atlasInterpolation = gl.LINEAR;
             texSize = tile.glyphAtlasTexture.size;
-
+            if (bucket.iconsInText) {
+                texSizeIcon = tile.imageAtlasTexture.size;
+                atlasTextureIcon = tile.imageAtlasTexture;
+                const zoomDependentSize = sizeData.kind === 'composite' || sizeData.kind === 'camera';
+                atlasInterpolationIcon = transformed || painter.options.rotating || painter.options.zooming || zoomDependentSize ? gl.LINEAR : gl.NEAREST;
+            }
         } else {
             const iconScaled = layer.layout.get('icon-size').constantOr(0) !== 1 || bucket.iconsNeedLinear;
-            const iconTransformed = pitchWithMap || tr.pitch !== 0;
-
             atlasTexture = tile.imageAtlasTexture;
-            atlasInterpolation = isSDF || painter.options.rotating || painter.options.zooming || iconScaled || iconTransformed ?
+            atlasInterpolation = isSDF || painter.options.rotating || painter.options.zooming || iconScaled || transformed ?
                 gl.LINEAR :
                 gl.NEAREST;
             texSize = tile.imageAtlasTexture.size;
@@ -292,10 +310,15 @@ function drawLayerSymbols(painter, sourceCache, layer, coords, isText, translate
 
         let uniformValues;
         if (isSDF) {
-            uniformValues = symbolSDFUniformValues(sizeData.kind,
+            if (!bucket.iconsInText) {
+                uniformValues = symbolSDFUniformValues(sizeData.kind,
                 size, rotateInShader, pitchWithMap, painter, matrix,
                 uLabelPlaneMatrix, uglCoordMatrix, isText, texSize, true);
-
+            } else {
+                uniformValues = symbolTextAndIconUniformValues(sizeData.kind,
+                size, rotateInShader, pitchWithMap, painter, matrix,
+                uLabelPlaneMatrix, uglCoordMatrix, texSize, texSizeIcon);
+            }
         } else {
             uniformValues = symbolIconUniformValues(sizeData.kind,
                 size, rotateInShader, pitchWithMap, painter, matrix,
@@ -307,7 +330,9 @@ function drawLayerSymbols(painter, sourceCache, layer, coords, isText, translate
             buffers,
             uniformValues,
             atlasTexture,
+            atlasTextureIcon,
             atlasInterpolation,
+            atlasInterpolationIcon,
             isSDF,
             hasHalo
         };
@@ -337,7 +362,14 @@ function drawLayerSymbols(painter, sourceCache, layer, coords, isText, translate
     for (const segmentState of tileRenderState) {
         const state = segmentState.state;
 
+        context.activeTexture.set(gl.TEXTURE0);
         state.atlasTexture.bind(state.atlasInterpolation, gl.CLAMP_TO_EDGE);
+        if (state.atlasTextureIcon) {
+            context.activeTexture.set(gl.TEXTURE1);
+            if (state.atlasTextureIcon) {
+                state.atlasTextureIcon.bind(state.atlasInterpolationIcon, gl.CLAMP_TO_EDGE);
+            }
+        }
 
         if (state.isSDF) {
             const uniformValues = ((state.uniformValues: any): UniformValues<SymbolSDFUniformsType>);

src/render/glyph_atlas.js

@@ -8,7 +8,7 @@ import type {GlyphMetrics, StyleGlyph} from '../style/style_glyph';
 
 const padding = 1;
 
-type Rect = {
+export type Rect = {
     x: number,
     y: number,
     w: number,

src/render/image_atlas.js

@@ -8,7 +8,8 @@ import type {StyleImage} from '../style/style_image';
 import type ImageManager from './image_manager';
 import type Texture from './texture';
 
-const padding = 1;
+const IMAGE_PADDING = 1;
+export {IMAGE_PADDING};
 
 type Rect = {
     x: number,
@@ -30,15 +31,15 @@ export class ImagePosition {
 
     get tl(): [number, number] {
         return [
-            this.paddedRect.x + padding,
-            this.paddedRect.y + padding
+            this.paddedRect.x + IMAGE_PADDING,
+            this.paddedRect.y + IMAGE_PADDING
         ];
     }
 
     get br(): [number, number] {
         return [
-            this.paddedRect.x + this.paddedRect.w - padding,
-            this.paddedRect.y + this.paddedRect.h - padding
+            this.paddedRect.x + this.paddedRect.w - IMAGE_PADDING,
+            this.paddedRect.y + this.paddedRect.h - IMAGE_PADDING
         ];
     }
 
@@ -48,8 +49,8 @@ export class ImagePosition {
 
     get displaySize(): [number, number] {
         return [
-            (this.paddedRect.w - padding * 2) / this.pixelRatio,
-            (this.paddedRect.h - padding * 2) / this.pixelRatio
+            (this.paddedRect.w - IMAGE_PADDING * 2) / this.pixelRatio,
+            (this.paddedRect.h - IMAGE_PADDING * 2) / this.pixelRatio
         ];
     }
 }
@@ -76,14 +77,14 @@ export default class ImageAtlas {
         for (const id in icons) {
             const src = icons[id];
             const bin = iconPositions[id].paddedRect;
-            RGBAImage.copy(src.data, image, {x: 0, y: 0}, {x: bin.x + padding, y: bin.y + padding}, src.data);
+            RGBAImage.copy(src.data, image, {x: 0, y: 0}, {x: bin.x + IMAGE_PADDING, y: bin.y + IMAGE_PADDING}, src.data);
         }
 
         for (const id in patterns) {
             const src = patterns[id];
             const bin = patternPositions[id].paddedRect;
-            const x = bin.x + padding,
-                y = bin.y + padding,
+            const x = bin.x + IMAGE_PADDING,
+                y = bin.y + IMAGE_PADDING,
                 w = src.data.width,
                 h = src.data.height;
 
@@ -106,8 +107,8 @@ export default class ImageAtlas {
             const bin = {
                 x: 0,
                 y: 0,
-                w: src.data.width + 2 * padding,
-                h: src.data.height + 2 * padding,
+                w: src.data.width + 2 * IMAGE_PADDING,
+                h: src.data.height + 2 * IMAGE_PADDING,
             };
             bins.push(bin);
             positions[id] = new ImagePosition(bin, src);

src/render/program/program_uniforms.js

@@ -10,7 +10,7 @@ import {heatmapUniforms, heatmapTextureUniforms} from './heatmap_program';
 import {hillshadeUniforms, hillshadePrepareUniforms} from './hillshade_program';
 import {lineUniforms, lineGradientUniforms, linePatternUniforms, lineSDFUniforms} from './line_program';
 import {rasterUniforms} from './raster_program';
-import {symbolIconUniforms, symbolSDFUniforms} from './symbol_program';
+import {symbolIconUniforms, symbolSDFUniforms, symbolTextAndIconUniforms} from './symbol_program';
 import {backgroundUniforms, backgroundPatternUniforms} from './background_program';
 
 export const programUniforms = {
@@ -36,6 +36,7 @@ export const programUniforms = {
     raster: rasterUniforms,
     symbolIcon: symbolIconUniforms,
     symbolSDF: symbolSDFUniforms,
+    symbolTextAndIcon: symbolTextAndIconUniforms,
     background: backgroundUniforms,
     backgroundPattern: backgroundPatternUniforms
 };