Compress textures stage

.travis.yml

@@ -4,7 +4,7 @@ node_js:
   - "6"
 script:
   - npm run jsHint -- --failTaskOnError
-  - npm run test -- --failTaskOnError --suppressPassed
+  - npm run test -- --failTaskOnError --suppressPassed --excludeCompressedTextures
 after_success:
 ## We only need to run coveralls for one node version (doesn't matter which one).
 ## We also ignore publishing failures, since restarting an existing travis build would otherwise break.

CHANGES.md

@@ -2,6 +2,7 @@ Change Log
 ==========
 
 ### 0.1.0-alpha11 - ??
+* Added `compressTextures` stage to compress input textures to a variety of compressed texture formats.
 * Optimized `mergeBuffers` to avoid repeated copies, drastically improving performance when there are lots of buffers to merge.
 * Fixed a bug in `addPipelineExtras` that made it try to add extras to null objects.
 * Expose `triangleAxisAlignedBoundingBoxOverlap`, an implementation of Tomas Akenine-Möller algorithm for determining if a triangle overlaps an axis aligned bounding box.

README.md

@@ -26,6 +26,11 @@ Command-Line Example:
 node ./bin/gltf-pipeline.js -i ./specs/data/boxTexturedUnoptimized/CesiumTexturedBoxTest.gltf -o output.gltf
 ```
 
+Compress textures to the dxt1 and etc1 formats
+```
+node ./bin/gltf-pipeline.js -i ./specs/data/boxTexturedUnoptimized/CesiumTexturedBoxTest.gltf -o output.gltf --texcomp.dxt1.enable --texcomp.dxt1.quality=10 --texcomp.etc1.enable
+```
+
 ### Command-Line Flags
 
 |Flag|Description|Required|
@@ -35,15 +40,15 @@ node ./bin/gltf-pipeline.js -i ./specs/data/boxTexturedUnoptimized/CesiumTexture
 |`--output`, `-o`|output=PATH, Write optimized glTF to the specified file.|No|
 |`--binary`, `-b`|Write binary glTF file using KHR_binary_glTF extension.|No, default `false`|
 |`--separate`, `-s`|Write separate geometry/animation data files, shader files, and textures instead of embedding them in the glTF asset.|No, default `false`|
-|`--separateTexture`, `-t`|Write out separate textures, but embeds geometry/animation data files and shader files in the glTF asset.|No, default `false`|
+|`--separateImage`, `-t`|Write out separate textures, but embeds geometry/animation data files and shader files in the glTF asset.|No, default `false`|
 |`--quantize`, `-q`|Quantize the attributes of this glTF asset using the WEB3D_quantized_attributes extension.|No, default `false`|
 |`--encodeNormals`, `-n`|Oct-encode the normals of this glTF asset.|No, default `false`|
 |`--compressTextureCoordinates`, `-c`|Compress the testure coordinates of this glTF asset.|No, default `false`|
 |`--removeNormals`, `-r`|Strips off existing normals, allowing them to be regenerated.|No, default `false`|
 |`--faceNormals`, `-f`|If normals are missing, they should be generated using the face normal.|No, default `false`|
+|`--cesium`|Optimize the glTF for Cesium by using the sun as a default light source.|No, default `false`|
 |`--tangentsBitangents`|If normals and texture coordinates are given, generate tangents and bitangents.|No, default `false`|
 |`--stats`|Print statistics to console for input and output glTF files.|No, default `false`|
-|`--cesium`, `-c`|Optimize the glTF for Cesium by using the sun as a default light source.|No, default `false`|
 |`--kmc.enable`|Materials should be expressed using the KHR_materials_common extension. If other `kmc` flags are enabled, this is implicitly true.|No, default `false`|
 |`--kmc.doubleSided`|Declares whether backface culling should be disabled.|No, default `false`|
 |`--kmc.technique`|The lighting model to use.|No, default `PHONG`|
@@ -52,6 +57,11 @@ node ./bin/gltf-pipeline.js -i ./specs/data/boxTexturedUnoptimized/CesiumTexture
 |`--ao.groundPlane`|Simulate a groundplane at the lowest point of the model when baking AO.|No, default `false`|
 |`--ao.ambientShadowContribution`|Amount of AO to show when blending between shader computed lighting and AO. 1.0 is full AO, 0.5 is a 50/50 blend.|No, default `0.5`|
 |`--ao.quality`|Quality to use when baking AO. Valid settings are high, medium, and low.|No, default `low`|
+|`--texcomp.<format>.enable`|Whether to compress textures with the given compressed texture format. If other `texcomp.<format>` flags are enabled, this is implicitly true. Multiple formats may be supplied by repeating this flag. <format> Must be one of the following: `pvrtc1`, `pvrtc2`, `etc1`, `etc2`, `astc`, `dxt1`, `dxt3`, `dxt5`, `crunch-dxt1`, `crunch-dxt5`. Compressed textures are saved as Cesium and 3D Tiles specific metadata inside `image.extras.compressedImage3DTiles`. More details about texture compression in glTF here: https://github.com/KhronosGroup/glTF/issues/739|No, unless other texcomp options are set.|
+|`--texcomp.<format>.quality`|The compressed texture quality from 0 to 10.|No, default `5`|
+|`--texcomp.<format>.bitrate`|The bitrate when using the pvrtc or astc formats. For pvrtc formats this value must be `2.0` or `4.0`.|No, default `2.0`|
+|`--texcomp.<format>.blockSize`|The block size for astc compression. Smaller block sizes result in higher bitrates. This value is ignored if `options.bitrate` is also set. Must be one of the following: `4x4`, `5x4`, `5x5`, `6x5`, `6x6`, `8x5`, `8x6`, `8x8`, `10x5`, `10x6`, `10x8`, `10x10`, `12x10`, `12x12`|No, default `8x8`|
+|`--texcomp.<format>.alphaBit`|Store a single bit for alpha. Only supported for etc2.|No, default `false`|
 
 ## Build Instructions
 
@@ -77,6 +87,19 @@ npm run build-cesium
 
 This will output a portion of the gltf-pipeline code into the `dist/cesium` folder, reformatted into AMD style for use with RequireJS and Cesium in the browser.
 
+### Third-party tools
+
+This project bundles several native binaries for texture compression.
+
+* [PVRTexTool] (https://community.imgtec.com/developers/powervr/tools/pvrtextool/) - Windows, macOS, and Linux binaries are provided by the PowerVR SDK.
+* [etc2comp](https://github.com/google/etc2comp) - Windows, macOS, and Linux binaries built from source.
+* [crunch](https://github.com/BinomialLLC/crunch) - Windows binary built from [crunch](https://github.com/BinomialLLC/crunch), macOS binary built from [BKcore/crunch-osx](https://github.com/BKcore/crunch-osx), Linux binary built from [tomerb/linux_fixes](https://github.com/tomerb/crunch/tree/linux_fixes).
+* [astc-encoder](https://github.com/ARM-software/astc-encoder) - Windows, macOS, and Linux binaries built from source.
+
+On Linux you may need to install g++ 4.8 as some of the tools link to the libgcc and libstdc++ runtimes.
+
+Run the help command for each tool for a more detailed description of the supported command-line options.
+
 ### Building for other integration
 
 Some functionality of gltf-pipeline is used by other projects along with Cesium as a third party library. The necessary files can be generated using:
@@ -146,6 +169,11 @@ Contact [@lilleyse](https://github.com/lilleyse) if you need access to publish.
 
 Pull requests are appreciated!  Please use the same [Contributor License Agreement (CLA)](https://github.com/AnalyticalGraphicsInc/cesium/blob/master/CONTRIBUTING.md) and [Coding Guide](https://github.com/AnalyticalGraphicsInc/cesium/blob/master/Documentation/Contributors/CodingGuide/README.md) used for [Cesium](http://cesiumjs.org/).
 
+## Attribution
+
+This product includes components of the PowerVR Tools Software from Imagination Technologies Limited.
+https://community.imgtec.com/developers/powervr/powervr-tools-software-eula/
+
 ---
 
 <p align="center">

bin/gltf-pipeline.js

@@ -1,14 +1,20 @@
 #!/usr/bin/env node
 'use strict';
+var Cesium = require('cesium');
 var Pipeline = require('../lib/Pipeline');
 var parseArguments = require('../lib/parseArguments');
 
+var defined = Cesium.defined;
 var processFileToDisk = Pipeline.processFileToDisk;
 
 var args = process.argv;
 args = args.slice(2, args.length);
 var options = parseArguments(args);
 
+if (!defined(options)) {
+    return;
+}
+
 console.time('optimize');
 // Node automatically waits for all promises to terminate
 processFileToDisk(options.inputPath, options.outputPath, options)

gulpfile.js

@@ -44,9 +44,29 @@ gulp.task('jsHint-watch', function () {
     gulp.watch(jsHintFiles, ['jsHint']);
 });
 
+function excludeCompressedTextures(jasmine) {
+    var excludedSpecs = ['compressTexturesSpec.js', 'compressTexturesMultipleFormatsSpec.js'];
+    var specFiles = jasmine.specFiles;
+    var specsLength = specFiles.length;
+    var excludedLength = excludedSpecs.length;
+    for (var i = specsLength - 1; i >= 0; --i) {
+        for (var j = 0; j < excludedLength; ++j) {
+            if (specFiles[i].indexOf(excludedSpecs[j]) > -1) {
+                specFiles.splice(i, 1);
+                break;
+            }
+        }
+    }
+}
+
 gulp.task('test', function (done) {
     var jasmine = new Jasmine();
     jasmine.loadConfigFile('specs/jasmine.json');
+    if (defined(argv.excludeCompressedTextures)) {
+        // Exclude compressTexturesSpec for Travis builds
+        // Travis runs Ubuntu 12.04.5 which has glibc 2.15, while crunch requires glibc 2.22 or higher
+        excludeCompressedTextures(jasmine);
+    }
     jasmine.addReporter(new JasmineSpecReporter({
         displaySuccessfulSpec: !defined(argv.suppressPassed) || !argv.suppressPassed
     }));
@@ -72,11 +92,20 @@ gulp.task('test-watch', function () {
 
 gulp.task('coverage', function () {
     fsExtra.removeSync('coverage/server');
+
+    // Exclude compressTexturesSpec from coverage for Travis builds
+    // Travis runs Ubuntu 12.04.5 which has glibc 2.15, while crunch requires glibc 2.22 or higher
+    var additionalExcludes = '';
+    if (defined(argv.excludeCompressedTextures)) {
+        additionalExcludes += 'specs/lib/compressTexturesSpec.js';
+        additionalExcludes += 'specs/lib/compressTexturesMultipleFormatsSpec.js';
+    }
+
     child_process.execSync('istanbul' +
         ' cover' +
         ' --include-all-sources' +
         ' --dir coverage' +
-        ' -x "specs/** coverage/** dist/** index.js gulpfile.js"' +
+        ' -x "specs/** coverage/** dist/** index.js gulpfile.js"' + additionalExcludes +
         ' node_modules/jasmine/bin/jasmine.js' +
         ' JASMINE_CONFIG_PATH=specs/jasmine.json', {
         stdio: [process.stdin, process.stdout, process.stderr]

lib/Pipeline.js

@@ -13,6 +13,7 @@ var combineNodes = require('./combineNodes');
 var combinePrimitives = require('./combinePrimitives');
 var compressIntegerAccessors = require('./compressIntegerAccessors');
 var compressTextureCoordinates = require('./compressTextureCoordinates');
+var compressTextures = require('./compressTextures');
 var convertDagToTree = require('./convertDagToTree');
 var encodeImages = require('./encodeImages');
 var generateModelMaterialsCommon = require('./generateModelMaterialsCommon');
@@ -80,7 +81,9 @@ Pipeline.processJSON = function(gltf, options) {
  * @param {Object} [options.aoOptions=undefined] Options to pass to the bakeAmbientOcclusion stage, if undefined, stage is not run.
  * @param {Object} [options.encodeNormals=false] Flag to run octEncodeNormals stage.
  * @param {Object} [options.compressTextureCoordinates=false] Flag to run compressTextureCoordinates stage.
+ * @param {Object} [options.kmcOptions=undefined] Options to pass to the generateModelMaterialsCommon stage, if undefined, stage is not run.
  * @param {Object} [options.quantize] Flag to run quantizeAttributes stage.
+ * @param {Object|Object[]} [options.textureCompressionOptions=undefined] Options to pass to the compressTextures stage. If an array of options is given, the textures will be compressed in multiple formats. If undefined, stage is not run.
  * @returns {Promise} A promise that resolves to the processed glTF asset.
  */
 Pipeline.processJSONWithExtras  = function(gltfWithExtras, options) {
@@ -127,7 +130,7 @@ Pipeline.processJSONWithExtras  = function(gltfWithExtras, options) {
     }
 
     var aoOptions = options.aoOptions;
-    if (defined(aoOptions) && aoOptions.enabled) {
+    if (defined(aoOptions)) {
         bakeAmbientOcclusion(gltfWithExtras, aoOptions);
     }
 
@@ -156,11 +159,16 @@ Pipeline.processJSONWithExtras  = function(gltfWithExtras, options) {
                 }
                 quantizeAttributes(gltfWithExtras, quantizedOptions);
             }
-            return encodeImages(gltfWithExtras);
+            var textureCompressionOptions = options.textureCompressionOptions;
+            if (defined(textureCompressionOptions)) {
+                return compressTextures(gltfWithExtras, textureCompressionOptions);
+            } else {
+                return encodeImages(gltfWithExtras);
+            }
         })
         .then(function() {
             var kmcOptions = options.kmcOptions;
-            if (defined(kmcOptions) && kmcOptions.enable) {
+            if (defined(kmcOptions)) {
                 generateModelMaterialsCommon(gltfWithExtras, options.kmcOptions);
             }
 
@@ -254,10 +262,21 @@ function writeSources(gltf) {
     var embed = true;
     var embedImage = true;
     var writeSourcePromises = [
-        writeSource(gltf, undefined, 'buffers', embed, embedImage),
-        writeSource(gltf, undefined, 'images', embed, embedImage),
-        writeSource(gltf, undefined, 'shaders', embed, embedImage)
+        writeSource(gltf.buffers, 'buffers', undefined, embed, embedImage),
+        writeSource(gltf.images, 'images', undefined, embed, embedImage),
+        writeSource(gltf.shaders, 'shaders', undefined, embed, embedImage)
     ];
+    // Write sources for compressed textures
+    var images = gltf.images;
+    for (var imageId in images) {
+        if (images.hasOwnProperty(imageId)) {
+            var image = images[imageId];
+            if (defined(image.extras) && defined(image.extras.compressedImage3DTiles)) {
+                var compressedImages = image.extras.compressedImage3DTiles;
+                writeSourcePromises.push(writeSource(compressedImages, 'images', undefined, embed, embedImage));
+            }
+        }
+    }
     return Promise.all(writeSourcePromises)
         .then(function() {
             return gltf;

lib/addPipelineExtras.js

@@ -38,6 +38,21 @@ function addPipelineExtras(gltf) {
     if (defined(gltf.asset)) {
         objectStack.push(gltf.asset);
     }
+    // Add pipeline extras to compressed textures
+    var images = gltf.images;
+    for (var imageId in images) {
+        if (images.hasOwnProperty(imageId)) {
+            var image = images[imageId];
+            if (defined(image.extras) && defined(image.extras.compressedImage3DTiles)) {
+                var compressedImages = image.extras.compressedImage3DTiles;
+                for (var compressedImageId in compressedImages) {
+                    if (compressedImages.hasOwnProperty(compressedImageId)) {
+                        objectStack.push(compressedImages[compressedImageId]);
+                    }
+                }
+            }
+        }
+    }
     while (objectStack.length > 0) {
         var object = objectStack.pop();
         if (Array.isArray(object)) {