Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added support of parentUrl property in layer.json #5864

Merged
merged 7 commits into from
Oct 13, 2017
Merged

Added support of parentUrl property in layer.json #5864

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b00498d
Changed Cesium terrain provider to be able to handle a parentUrl in l…
Sep 28, 2017
17f9d4b
Tweaks to availability.
Sep 29, 2017
645fdd8
Fixed tests.
Sep 29, 2017
323cf28
Added test for parentUrl
Sep 29, 2017
ef33387
Tweaks from PR comments.
Oct 10, 2017
d62c765
Merge remote-tracking branch 'origin/master' into terrain-ancestor-la…
Oct 10, 2017
e6e45b5
Update CHANGES.md
Oct 10, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 139 additions & 41 deletions Source/Core/CesiumTerrainProvider.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,15 @@ define([
TileProviderError) {
'use strict';

function LayerInformation(layer) {
this.isHeightmap = layer.isHeightmap;
this.tileUrlTemplates = layer.tileUrlTemplates;
this.availability = layer.availability;
this.hasVertexNormals = layer.hasVertexNormals;
this.hasWaterMask = layer.hasWaterMask;
this.littleEndianExtensionSize = layer.littleEndianExtensionSize;
}

/**
* A {@link TerrainProvider} that accesses terrain data in a Cesium terrain format.
* The format is described on the
Expand Down Expand Up @@ -114,22 +123,16 @@ define([

this._heightmapStructure = undefined;
this._hasWaterMask = false;

/**
* Boolean flag that indicates if the Terrain Server can provide vertex normals.
* @type {Boolean}
* @default false
* @private
*/
this._hasVertexNormals = false;

/**
* Boolean flag that indicates if the client should request vertex normals from the server.
* @type {Boolean}
* @default false
* @private
*/
this._requestVertexNormals = defaultValue(options.requestVertexNormals, false);
this._littleEndianExtensionSize = true;

/**
* Boolean flag that indicates if the client should request tile watermasks from the server.
* @type {Boolean}
Expand All @@ -139,17 +142,19 @@ define([
this._requestWaterMask = defaultValue(options.requestWaterMask, false);

this._errorEvent = new Event();
this._availability = undefined;

var credit = options.credit;
if (typeof credit === 'string') {
credit = new Credit(credit);
}
this._credit = credit;

this._availability = undefined;

this._ready = false;
this._readyPromise = when.defer();

var lastUrl = this._url;
var metadataUrl = joinUrls(this._url, 'layer.json');
if (defined(this._proxy)) {
metadataUrl = this._proxy.getURL(metadataUrl);
Expand All @@ -158,7 +163,11 @@ define([
var that = this;
var metadataError;

function metadataSuccess(data) {
var layers = this._layers = [];
var attribution = '';
var overallAvailability = [];

function parseMetadataSuccess(data) {
var message;

if (!data.format) {
Expand All @@ -173,8 +182,14 @@ define([
return;
}

var hasVertexNormals = false;
var hasWaterMask = false;
var littleEndianExtensionSize = true;
var isHeightmap = false;
if (data.format === 'heightmap-1.0') {
that._heightmapStructure = {
isHeightmap = true;
if (!defined(that._heightmapStructure)) {
that._heightmapStructure = {
heightScale : 1.0 / 5.0,
heightOffset : -1000.0,
elementsPerHeight : 1,
Expand All @@ -184,63 +199,131 @@ define([
lowestEncodedHeight : 0,
highestEncodedHeight : 256 * 256 - 1
};
that._hasWaterMask = true;
}
hasWaterMask = true;
that._requestWaterMask = true;
} else if (data.format.indexOf('quantized-mesh-1.') !== 0) {
message = 'The tile format "' + data.format + '" is invalid or not supported.';
metadataError = TileProviderError.handleError(metadataError, that, that._errorEvent, message, undefined, undefined, undefined, requestMetadata);
return;
}

that._tileUrlTemplates = data.tiles;
for (var i = 0; i < that._tileUrlTemplates.length; ++i) {
var template = new Uri(that._tileUrlTemplates[i]);
var baseUri = new Uri(that._url);
var tileUrlTemplates = data.tiles;
for (var i = 0; i < tileUrlTemplates.length; ++i) {
var template = new Uri(tileUrlTemplates[i]);
var baseUri = new Uri(lastUrl);
if (template.authority && !baseUri.authority) {
baseUri.authority = template.authority;
baseUri.scheme = template.scheme;
}
that._tileUrlTemplates[i] = joinUrls(baseUri, template).toString().replace('{version}', data.version);
tileUrlTemplates[i] = joinUrls(baseUri, template).toString().replace('{version}', data.version);
}

var availableTiles = data.available;

var availability;
if (defined(availableTiles)) {
that._availability = new TileAvailability(that._tilingScheme, availableTiles.length);
availability = new TileAvailability(that._tilingScheme, availableTiles.length);

for (var level = 0; level < availableTiles.length; ++level) {
var rangesAtLevel = availableTiles[level];
var yTiles = that._tilingScheme.getNumberOfYTilesAtLevel(level);
if (!defined(overallAvailability[level])) {
overallAvailability[level] = [];
}

for (var rangeIndex = 0; rangeIndex < rangesAtLevel.length; ++rangeIndex) {
var range = rangesAtLevel[rangeIndex];
that._availability.addAvailableTileRange(level, range.startX, yTiles - range.endY - 1, range.endX, yTiles - range.startY - 1);
var yStart = yTiles - range.endY - 1;
var yEnd = yTiles - range.startY - 1;
overallAvailability[level].push([range.startX, yStart, range.endX, yEnd]);
availability.addAvailableTileRange(level, range.startX, yStart, range.endX, yEnd);
}
}
}

if (!defined(that._credit) && defined(data.attribution) && data.attribution !== null) {
that._credit = new Credit(data.attribution);
}

// The vertex normals defined in the 'octvertexnormals' extension is identical to the original
// contents of the original 'vertexnormals' extension. 'vertexnormals' extension is now
// deprecated, as the extensionLength for this extension was incorrectly using big endian.
// We maintain backwards compatibility with the legacy 'vertexnormal' implementation
// by setting the _littleEndianExtensionSize to false. Always prefer 'octvertexnormals'
// over 'vertexnormals' if both extensions are supported by the server.
if (defined(data.extensions) && data.extensions.indexOf('octvertexnormals') !== -1) {
that._hasVertexNormals = true;
hasVertexNormals = true;
} else if (defined(data.extensions) && data.extensions.indexOf('vertexnormals') !== -1) {
that._hasVertexNormals = true;
that._littleEndianExtensionSize = false;
hasVertexNormals = true;
littleEndianExtensionSize = false;
}
if (defined(data.extensions) && data.extensions.indexOf('watermask') !== -1) {
that._hasWaterMask = true;
hasWaterMask = true;
}

that._hasWaterMask = that._hasWaterMask || hasWaterMask;
that._hasVertexNormals = that._hasVertexNormals || hasVertexNormals;
if (defined(data.attribution)) {
if (attribution.length > 0) {
attribution += ' ';
}
attribution += data.attribution;
}

layers.push(new LayerInformation({
isHeightmap: isHeightmap,
tileUrlTemplates: tileUrlTemplates,
availability: availability,
hasVertexNormals: hasVertexNormals,
hasWaterMask: hasWaterMask,
littleEndianExtensionSize: littleEndianExtensionSize
}));

var parentUrl = data.parentUrl;
if (defined(parentUrl)) {
if (!defined(availability)) {
console.log('A layer.json can\'t have a parentUrl if it does\'t have an available array.');
return when.resolve();
}
lastUrl = joinUrls(lastUrl, parentUrl);
metadataUrl = joinUrls(lastUrl, 'layer.json');
if (defined(that._proxy)) {
metadataUrl = that._proxy.getURL(metadataUrl);
}
var parentMetadata = loadJson(metadataUrl);
return when(parentMetadata, parseMetadataSuccess, parseMetadataFailure);
}

that._ready = true;
that._readyPromise.resolve(true);
return when.resolve();
}

function parseMetadataFailure(data) {
var message = 'An error occurred while accessing ' + metadataUrl + '.';
metadataError = TileProviderError.handleError(metadataError, that, that._errorEvent, message, undefined, undefined, undefined, requestMetadata);
}

function metadataSuccess(data) {
parseMetadataSuccess(data)
.then(function() {
if (defined(metadataError)) {
return;
}

var length = overallAvailability.length;
if (length > 0) {
var availability = that._availability = new TileAvailability(that._tilingScheme, length);
for (var level = 0; level < length; ++level) {
var levelRanges = overallAvailability[level];
for (var i = 0; i < levelRanges.length; ++i) {
var range = levelRanges[i];
availability.addAvailableTileRange(level, range[0], range[1], range[2], range[3]);
}
}
}

if (!defined(that._credit) && attribution.length > 0) {
that._credit = new Credit(attribution);
}

that._ready = true;
that._readyPromise.resolve(true);
});
}

function metadataFailure(data) {
Expand All @@ -257,8 +340,7 @@ define([
});
return;
}
var message = 'An error occurred while accessing ' + metadataUrl + '.';
metadataError = TileProviderError.handleError(metadataError, that, that._errorEvent, message, undefined, undefined, undefined, requestMetadata);
parseMetadataFailure(data);
}

function requestMetadata() {
Expand Down Expand Up @@ -320,7 +402,7 @@ define([
});
}

function createQuantizedMeshTerrainData(provider, buffer, level, x, y, tmsY) {
function createQuantizedMeshTerrainData(provider, buffer, level, x, y, tmsY, littleEndianExtensionSize) {
var pos = 0;
var cartesian3Elements = 3;
var boundingSphereElements = cartesian3Elements + 1;
Expand Down Expand Up @@ -431,7 +513,7 @@ define([
while (pos < view.byteLength) {
var extensionId = view.getUint8(pos, true);
pos += Uint8Array.BYTES_PER_ELEMENT;
var extensionLength = view.getUint32(pos, provider._littleEndianExtensionSize);
var extensionLength = view.getUint32(pos, littleEndianExtensionSize);
pos += Uint32Array.BYTES_PER_ELEMENT;

if (extensionId === QuantizedMeshExtensionIds.OCT_VERTEX_NORMALS && provider._requestVertexNormals) {
Expand Down Expand Up @@ -505,7 +587,22 @@ define([
}
//>>includeEnd('debug');

var urlTemplates = this._tileUrlTemplates;
var layers = this._layers;
var layerToUse = layers[0];
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This assignment is not necessary.

var layerCount = layers.length;
for (var i=0;i<layerCount;++i) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

whitepace

var layer = layers[i];
if (!defined(layer.availability) || layer.availability.isTileAvailable(level, x, y)) {
layerToUse = layer;
break;
}
}

if (!defined(layerToUse)) {
return undefined;
}

var urlTemplates = layerToUse.tileUrlTemplates;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't measured or anything, but there's the potential for a somewhat high performance cost here, especially on slow systems with slow internet connections. On these systems, Cesium tends to spend a lot of time traversing the quadtree to select the tiles it wishes it has, then calling requestTileGeometry on them, only to see it return undefined because too many tile requests are already in progress.

Before this pull request, Cesium only had to do some string manipulations to build up a URL, and then it would hand it off to loadArrayBuffer which would check for too many requests in flight and potentially return undefined.

But after this pull request, Cesium now has to do a lot of tile availability checking before handing off to loadArrayBuffer. I'd be surprised if this doesn't show up pretty clearly in a profiler.

So two suggestions, one easy and one harder:

  1. If the terrain provider has only one layer, don't bother checking availability at all. (easy)
  2. Check if we need to throttle (and do so) before checking tile availability.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know if 2 is possible. Even if we check the throttle status, we still need to know whether that layer contains the tile so we can return a Promise/undefined or if we should go to the next layer, which may be from a different domain and have a different throttle status.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah true you could only optimize some special cases. i.e. if all the layers are from the same host, and that host already has too many requests in flight, you can skip availability checking. Or another less useful special case: if all layers already have too many tile requests in flight, just return undefined without checking availability. Anyway, the important thing is to avoid the performance regression for the single layer case (as you've already done) and we can worry about optimizing the multi layer case if it ever really becomes an issue.

if (urlTemplates.length === 0) {
return undefined;
}
Expand All @@ -522,10 +619,10 @@ define([
}

var extensionList = [];
if (this._requestVertexNormals && this._hasVertexNormals) {
extensionList.push(this._littleEndianExtensionSize ? 'octvertexnormals' : 'vertexnormals');
if (this._requestVertexNormals && layerToUse.hasVertexNormals) {
extensionList.push(layerToUse.littleEndianExtensionSize ? 'octvertexnormals' : 'vertexnormals');
}
if (this._requestWaterMask && this._hasWaterMask) {
if (this._requestWaterMask && layerToUse.hasWaterMask) {
extensionList.push('watermask');
}

Expand All @@ -540,7 +637,7 @@ define([
if (defined(that._heightmapStructure)) {
return createHeightmapTerrainData(that, buffer, level, x, y, tmsY);
}
return createQuantizedMeshTerrainData(that, buffer, level, x, y, tmsY);
return createQuantizedMeshTerrainData(that, buffer, level, x, y, tmsY, layerToUse.littleEndianExtensionSize);
});
};

Expand Down Expand Up @@ -723,10 +820,11 @@ define([
* @returns {Boolean} Undefined if not supported, otherwise true or false.
*/
CesiumTerrainProvider.prototype.getTileDataAvailable = function(x, y, level) {
if (!defined(this.availability)) {
if (!defined(this._availability)) {
return undefined;
}
return this.availability.isTileAvailable(level, x, y);

return this._availability.isTileAvailable(level, x, y);
};

return CesiumTerrainProvider;
Expand Down
41 changes: 41 additions & 0 deletions Specs/Core/CesiumTerrainProviderSpec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,20 @@ defineSuite([
return returnTileJson('Data/CesiumTerrainTileJson/PartialAvailability.tile.json');
}

function returnParentUrlTileJson() {
var paths = ['Data/CesiumTerrainTileJson/ParentUrl.tile.json',
'Data/CesiumTerrainTileJson/Parent.tile.json'];
var i = 0;
var oldLoad = loadWithXhr.load;
loadWithXhr.load = function(url, responseType, method, data, headers, deferred, overrideMimeType) {
if (url.indexOf('layer.json') >= 0) {
loadWithXhr.defaultLoad(paths[i++], responseType, method, data, headers, deferred);
} else {
return oldLoad(url, responseType, method, data, headers, deferred, overrideMimeType);
}
};
}

function waitForTile(level, x, y, requestNormals, requestWaterMask, f) {
var terrainProvider = new CesiumTerrainProvider({
url : 'made/up/url',
Expand Down Expand Up @@ -247,6 +261,33 @@ defineSuite([
});
});

it('requests parent layer.json', function() {
returnParentUrlTileJson();

var provider = new CesiumTerrainProvider({
url : 'made/up/url',
requestVertexNormals : true,
requestWaterMask : true
});

return pollToPromise(function() {
return provider.ready;
}).then(function() {
expect(provider.credit.text).toBe('This is a child tileset! This amazing data is courtesy The Amazing Data Source!');
expect(provider.requestVertexNormals).toBe(true);
expect(provider.requestWaterMask).toBe(true);
expect(provider.hasVertexNormals).toBe(false); // Neither tileset has them
expect(provider.hasWaterMask).toBe(true); // The top level tileset has them

var layers = provider._layers;
expect(layers.length).toBe(2);
expect(layers[0].hasVertexNormals).toBe(false);
expect(layers[0].hasWaterMask).toBe(true);
expect(layers[1].hasVertexNormals).toBe(false);
expect(layers[1].hasWaterMask).toBe(false);
});
});

it('raises an error if layer.json does not specify a format', function() {
returnTileJson('Data/CesiumTerrainTileJson/NoFormat.tile.json');

Expand Down
Loading