feat(rosetta): tablets can be compressed (#3652)

Usage: `yarn rosetta extract --compress-tablet`

This should reduce the `.jsii.tabl.json` size by about 90%.

---

By submitting this pull request, I confirm that my contribution is made under the terms of the [Apache 2.0 license].

[Apache 2.0 license]: https://www.apache.org/licenses/LICENSE-2.0

packages/jsii-rosetta/bin/jsii-rosetta.ts

@@ -228,6 +228,12 @@ function main() {
             describe: 'Ignore missing fixtures and literate markdown files instead of failing',
             type: 'boolean',
           })
+          .options('compress-tablet', {
+            alias: 'z',
+            type: 'boolean',
+            describe: 'Compress the resulting tablet file',
+            default: false,
+          })
           .conflicts('loose', 'strict')
           .conflicts('loose', 'fail'),
       wrapHandler(async (args) => {
@@ -252,6 +258,7 @@ function main() {
           cacheToFile: absCacheTo,
           trimCache: args['trim-cache'],
           loose: args.loose,
+          compressTablet: args['compress-tablet'],
         };
 
         const result = args.infuse

packages/jsii-rosetta/lib/commands/extract.ts

@@ -5,7 +5,7 @@ import * as logging from '../logging';
 import { RosettaTranslator, RosettaTranslatorOptions } from '../rosetta-translator';
 import { TypeScriptSnippet, SnippetParameters } from '../snippet';
 import { snippetKey } from '../tablets/key';
-import { LanguageTablet, DEFAULT_TABLET_NAME } from '../tablets/tablets';
+import { LanguageTablet, DEFAULT_TABLET_NAME, DEFAULT_TABLET_NAME_COMPRESSED } from '../tablets/tablets';
 import { RosettaDiagnostic } from '../translate';
 import { groupBy, isDefined } from '../util';
 import { infuse } from './infuse';
@@ -73,6 +73,13 @@ export interface ExtractOptions {
    * @default false
    */
   readonly allowDirtyTranslations?: boolean;
+
+  /**
+   * Compress the resulting tablet file
+   *
+   * @default false
+   */
+  readonly compressTablet?: boolean;
 }
 
 export async function extractAndInfuse(assemblyLocations: string[], options: ExtractOptions): Promise<ExtractResult> {
@@ -154,13 +161,16 @@ export async function extractSnippets(
   if (options.writeToImplicitTablets ?? true) {
     await Promise.all(
       Object.entries(snippetsPerAssembly).map(async ([location, snips]) => {
-        const asmTabletFile = path.join(location, DEFAULT_TABLET_NAME);
+        const asmTabletFile = path.join(
+          location,
+          options.compressTablet ? DEFAULT_TABLET_NAME_COMPRESSED : DEFAULT_TABLET_NAME,
+        );
         logging.debug(`Writing ${snips.length} translations to ${asmTabletFile}`);
         const translations = snips.map(({ key }) => translator.tablet.tryGetSnippet(key)).filter(isDefined);
 
         const asmTablet = new LanguageTablet();
         asmTablet.addSnippets(...translations);
-        await asmTablet.save(asmTabletFile);
+        await asmTablet.save(asmTabletFile, options.compressTablet);
       }),
     );
   }

packages/jsii-rosetta/lib/tablets/tablets.ts

@@ -1,5 +1,6 @@
 import * as fs from 'fs-extra';
 import * as path from 'path';
+import * as zlib from 'zlib';
 
 import { TargetLanguage } from '../languages';
 import * as logging from '../logging';
@@ -11,8 +12,16 @@ import { TabletSchema, TranslatedSnippetSchema, ORIGINAL_SNIPPET_KEY } from './s
 // eslint-disable-next-line @typescript-eslint/no-require-imports,@typescript-eslint/no-var-requires
 const TOOL_VERSION = require('../../package.json').version;
 
+/**
+ * The default name of the tablet file
+ */
 export const DEFAULT_TABLET_NAME = '.jsii.tabl.json';
 
+/**
+ * The default name of the compressed tablet file
+ */
+export const DEFAULT_TABLET_NAME_COMPRESSED = '.jsii.tabl.json.gz';
+
 export const CURRENT_SCHEMA_VERSION = '2';
 
 /**
@@ -121,8 +130,19 @@ export class LanguageTablet {
     return this.snippets[snippetKey(typeScriptSource)];
   }
 
+  /**
+   * Load the tablet from a file. Will automatically detect if the file is
+   * compressed and decompress accordingly.
+   */
   public async load(filename: string) {
-    const obj = (await fs.readJson(filename, { encoding: 'utf-8' })) as TabletSchema;
+    let data = await fs.readFile(filename);
+    // Gzip objects start with 1f 8b 08
+    if (data[0] === 0x1f && data[1] === 0x8b && data[2] === 0x08) {
+      // This is a gz object, so we decompress it now...
+      data = zlib.gunzipSync(data);
+    }
+
+    const obj: TabletSchema = JSON.parse(data.toString('utf-8'));
 
     if (!obj.toolVersion || !obj.snippets) {
       throw new Error(`File '${filename}' does not seem to be a Tablet file`);
@@ -147,12 +167,19 @@ export class LanguageTablet {
     return Object.values(this.snippets);
   }
 
-  public async save(filename: string) {
+  /**
+   * Saves the tablet schema to a file. If the compress option is passed, then
+   * the schema will be gzipped before writing to the file.
+   */
+  public async save(filename: string, compress = false) {
     await fs.mkdirp(path.dirname(filename));
-    await fs.writeJson(filename, this.toSchema(), {
-      encoding: 'utf-8',
-      spaces: 2,
-    });
+
+    let schema = Buffer.from(JSON.stringify(this.toSchema(), null, 2));
+    if (compress) {
+      schema = zlib.gzipSync(schema);
+    }
+
+    await fs.writeFile(filename, schema);
   }
 
   private toSchema(): TabletSchema {

packages/jsii-rosetta/test/commands/extract.test.ts

@@ -73,6 +73,19 @@ test('extract samples from test assembly', async () => {
   expect(tablet.snippetKeys.length).toEqual(1);
 });
 
+test('extract can save/load compressed tablets', async () => {
+  const compressedCacheFile = path.join(assembly.moduleDirectory, 'test.tabl.gz');
+  await extract.extractSnippets([assembly.moduleDirectory], {
+    cacheToFile: compressedCacheFile,
+    ...defaultExtractOptions,
+  });
+
+  const tablet = new LanguageTablet();
+  await tablet.load(compressedCacheFile);
+
+  expect(tablet.snippetKeys.length).toEqual(1);
+});
+
 test('extract works from compressed test assembly', async () => {
   const compressedAssembly = TestJsiiModule.fromSource(
     {