From 9d791e14e35298dc6f714bf41adb0d974e2c363f Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Thu, 7 Jul 2022 19:34:15 -0400 Subject: [PATCH 1/5] feat(rosetta): tablets can be compressed --- packages/jsii-rosetta/bin/jsii-rosetta.ts | 7 +++++ packages/jsii-rosetta/lib/commands/extract.ts | 14 +++++++-- packages/jsii-rosetta/lib/tablets/tablets.ts | 30 +++++++++++++++---- .../test/commands/extract.test.ts | 13 ++++++++ 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/packages/jsii-rosetta/bin/jsii-rosetta.ts b/packages/jsii-rosetta/bin/jsii-rosetta.ts index c2027c4d5c..77541f1f0c 100644 --- a/packages/jsii-rosetta/bin/jsii-rosetta.ts +++ b/packages/jsii-rosetta/bin/jsii-rosetta.ts @@ -228,6 +228,12 @@ function main() { describe: 'Ignore missing fixtures and literate markdown files instead of failing', type: 'boolean', }) + .options('compress-tablet', { + alias: 'z', + type: 'boolean', + describe: 'Compress the resulting tablet file', + default: false, + }) .conflicts('loose', 'strict') .conflicts('loose', 'fail'), wrapHandler(async (args) => { @@ -252,6 +258,7 @@ function main() { cacheToFile: absCacheTo, trimCache: args['trim-cache'], loose: args.loose, + compressTablet: args['compress-tablet'], }; const result = args.infuse diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts index b0ec4d03aa..ec036dcb41 100644 --- a/packages/jsii-rosetta/lib/commands/extract.ts +++ b/packages/jsii-rosetta/lib/commands/extract.ts @@ -5,7 +5,7 @@ import * as logging from '../logging'; import { RosettaTranslator, RosettaTranslatorOptions } from '../rosetta-translator'; import { TypeScriptSnippet, SnippetParameters } from '../snippet'; import { snippetKey } from '../tablets/key'; -import { LanguageTablet, DEFAULT_TABLET_NAME } from '../tablets/tablets'; +import { LanguageTablet, DEFAULT_TABLET_NAME, DEFAULT_TABLET_NAME_COMPRESSED } from '../tablets/tablets'; import { RosettaDiagnostic } from '../translate'; import { groupBy, isDefined } from '../util'; import { infuse } from './infuse'; @@ -73,6 +73,13 @@ export interface ExtractOptions { * @default false */ readonly allowDirtyTranslations?: boolean; + + /** + * Compress the resulting tablet file + * + * @default false + */ + readonly compressTablet?: boolean; } export async function extractAndInfuse(assemblyLocations: string[], options: ExtractOptions): Promise { @@ -154,7 +161,10 @@ export async function extractSnippets( if (options.writeToImplicitTablets ?? true) { await Promise.all( Object.entries(snippetsPerAssembly).map(async ([location, snips]) => { - const asmTabletFile = path.join(location, DEFAULT_TABLET_NAME); + const asmTabletFile = path.join( + location, + options.compressTablet ? DEFAULT_TABLET_NAME_COMPRESSED : DEFAULT_TABLET_NAME, + ); logging.debug(`Writing ${snips.length} translations to ${asmTabletFile}`); const translations = snips.map(({ key }) => translator.tablet.tryGetSnippet(key)).filter(isDefined); diff --git a/packages/jsii-rosetta/lib/tablets/tablets.ts b/packages/jsii-rosetta/lib/tablets/tablets.ts index 556e099edf..f1dc3408c4 100644 --- a/packages/jsii-rosetta/lib/tablets/tablets.ts +++ b/packages/jsii-rosetta/lib/tablets/tablets.ts @@ -1,5 +1,6 @@ import * as fs from 'fs-extra'; import * as path from 'node:path'; +import * as zlib from 'node:zlib'; import { TargetLanguage } from '../languages'; import * as logging from '../logging'; @@ -12,6 +13,7 @@ import { TabletSchema, TranslatedSnippetSchema, ORIGINAL_SNIPPET_KEY } from './s const TOOL_VERSION = require('../../package.json').version; export const DEFAULT_TABLET_NAME = '.jsii.tabl.json'; +export const DEFAULT_TABLET_NAME_COMPRESSED = '.jsii.tabl.json.gz'; export const CURRENT_SCHEMA_VERSION = '2'; @@ -121,8 +123,17 @@ export class LanguageTablet { return this.snippets[snippetKey(typeScriptSource)]; } + /** + * Load the tablet from a file. Will automatically detect if the file is in + * compression format ('.gz') and gunzip if it is. + */ public async load(filename: string) { - const obj = (await fs.readJson(filename, { encoding: 'utf-8' })) as TabletSchema; + let obj: TabletSchema | any = {}; + if (path.extname(filename) === '.gz') { + obj = JSON.parse(zlib.gunzipSync(await fs.readFile(filename)).toString()) as TabletSchema; + } else { + obj = (await fs.readJson(filename, { encoding: 'utf-8' })) as TabletSchema; + } if (!obj.toolVersion || !obj.snippets) { throw new Error(`File '${filename}' does not seem to be a Tablet file`); @@ -147,12 +158,21 @@ export class LanguageTablet { return Object.values(this.snippets); } + /** + * Saves the tablet schema to a file. If the filename has a '.gz' extension the tablet + * schema will be compressed, otherwise the tablet schema will be written in raw json. + */ public async save(filename: string) { await fs.mkdirp(path.dirname(filename)); - await fs.writeJson(filename, this.toSchema(), { - encoding: 'utf-8', - spaces: 2, - }); + + if (path.extname(filename) === '.gz') { + await fs.writeFile(filename, zlib.gzipSync(JSON.stringify(this.toSchema()))); + } else { + await fs.writeJson(filename, this.toSchema(), { + encoding: 'utf-8', + spaces: 2, + }); + } } private toSchema(): TabletSchema { diff --git a/packages/jsii-rosetta/test/commands/extract.test.ts b/packages/jsii-rosetta/test/commands/extract.test.ts index a18140b385..bd15d11739 100644 --- a/packages/jsii-rosetta/test/commands/extract.test.ts +++ b/packages/jsii-rosetta/test/commands/extract.test.ts @@ -73,6 +73,19 @@ test('extract samples from test assembly', async () => { expect(tablet.snippetKeys.length).toEqual(1); }); +test('extract can save/load compressed tablets', async () => { + const compressedCacheFile = path.join(assembly.moduleDirectory, 'test.tabl.gz'); + await extract.extractSnippets([assembly.moduleDirectory], { + cacheToFile: compressedCacheFile, + ...defaultExtractOptions, + }); + + const tablet = new LanguageTablet(); + await tablet.load(compressedCacheFile); + + expect(tablet.snippetKeys.length).toEqual(1); +}); + test('extract works from compressed test assembly', async () => { const compressedAssembly = TestJsiiModule.fromSource( { From 30514c63e255ef38672c65dae138908c038696d1 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Thu, 7 Jul 2022 19:38:25 -0400 Subject: [PATCH 2/5] docs --- packages/jsii-rosetta/lib/tablets/tablets.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/jsii-rosetta/lib/tablets/tablets.ts b/packages/jsii-rosetta/lib/tablets/tablets.ts index f1dc3408c4..ca3c62afc6 100644 --- a/packages/jsii-rosetta/lib/tablets/tablets.ts +++ b/packages/jsii-rosetta/lib/tablets/tablets.ts @@ -12,7 +12,14 @@ import { TabletSchema, TranslatedSnippetSchema, ORIGINAL_SNIPPET_KEY } from './s // eslint-disable-next-line @typescript-eslint/no-require-imports,@typescript-eslint/no-var-requires const TOOL_VERSION = require('../../package.json').version; +/** + * The default name of the tablet file + */ export const DEFAULT_TABLET_NAME = '.jsii.tabl.json'; + +/** + * The default name of the compressed tablet file + */ export const DEFAULT_TABLET_NAME_COMPRESSED = '.jsii.tabl.json.gz'; export const CURRENT_SCHEMA_VERSION = '2'; From 54fa15828881c202b66399544d9fa4ca27663994 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Fri, 8 Jul 2022 13:47:37 -0400 Subject: [PATCH 3/5] pr comments --- packages/jsii-rosetta/lib/tablets/tablets.ts | 34 ++++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/jsii-rosetta/lib/tablets/tablets.ts b/packages/jsii-rosetta/lib/tablets/tablets.ts index a9b05f99ef..34ab6fcce7 100644 --- a/packages/jsii-rosetta/lib/tablets/tablets.ts +++ b/packages/jsii-rosetta/lib/tablets/tablets.ts @@ -131,17 +131,19 @@ export class LanguageTablet { } /** - * Load the tablet from a file. Will automatically detect if the file is in - * compression format ('.gz') and gunzip if it is. + * Load the tablet from a file. Will automatically detect if the file is + * compressed and decompress accordingly. */ public async load(filename: string) { - let obj: TabletSchema | any = {}; - if (path.extname(filename) === '.gz') { - obj = JSON.parse(zlib.gunzipSync(await fs.readFile(filename)).toString()) as TabletSchema; - } else { - obj = (await fs.readJson(filename, { encoding: 'utf-8' })) as TabletSchema; + let data = await fs.readFile(filename); + // Gzip objects start with 1f 8b 08 + if (data[0] === 0x1f && data[1] === 0x8b && data[2] === 0x08) { + // This is a gz object, so we decompress it now... + data = zlib.gunzipSync(data); } + const obj: TabletSchema = JSON.parse(data.toString('utf-8')); + if (!obj.toolVersion || !obj.snippets) { throw new Error(`File '${filename}' does not seem to be a Tablet file`); } @@ -166,20 +168,18 @@ export class LanguageTablet { } /** - * Saves the tablet schema to a file. If the filename has a '.gz' extension the tablet - * schema will be compressed, otherwise the tablet schema will be written in raw json. + * Saves the tablet schema to a file. If the compress option is passed, then + * the schema will be gzipped before writing to the file. */ - public async save(filename: string) { + public async save(filename: string, compress = false) { await fs.mkdirp(path.dirname(filename)); - if (path.extname(filename) === '.gz') { - await fs.writeFile(filename, zlib.gzipSync(JSON.stringify(this.toSchema()))); - } else { - await fs.writeJson(filename, this.toSchema(), { - encoding: 'utf-8', - spaces: 2, - }); + let schema = Buffer.from(JSON.stringify(this.toSchema())); + if (compress) { + schema = zlib.gzipSync(schema); } + + await fs.writeFile(filename, schema); } private toSchema(): TabletSchema { From 5bdf999005a84a63fe871c2aa1bda0a9b77397c8 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 11 Jul 2022 08:18:29 -0400 Subject: [PATCH 4/5] add compress parameter and pretty-print json --- packages/jsii-rosetta/lib/commands/extract.ts | 2 +- packages/jsii-rosetta/lib/tablets/tablets.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts index 3385c142c4..3ab7daf300 100644 --- a/packages/jsii-rosetta/lib/commands/extract.ts +++ b/packages/jsii-rosetta/lib/commands/extract.ts @@ -170,7 +170,7 @@ export async function extractSnippets( const asmTablet = new LanguageTablet(); asmTablet.addSnippets(...translations); - await asmTablet.save(asmTabletFile); + await asmTablet.save(asmTabletFile, options.compressTablet); }), ); } diff --git a/packages/jsii-rosetta/lib/tablets/tablets.ts b/packages/jsii-rosetta/lib/tablets/tablets.ts index 34ab6fcce7..d6bb6b55b6 100644 --- a/packages/jsii-rosetta/lib/tablets/tablets.ts +++ b/packages/jsii-rosetta/lib/tablets/tablets.ts @@ -174,7 +174,7 @@ export class LanguageTablet { public async save(filename: string, compress = false) { await fs.mkdirp(path.dirname(filename)); - let schema = Buffer.from(JSON.stringify(this.toSchema())); + let schema = Buffer.from(JSON.stringify(this.toSchema(), null, 2)); if (compress) { schema = zlib.gzipSync(schema); } From 3c90a50bf2414c84d13d93f5595001df76bec693 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 11 Jul 2022 11:58:12 -0400 Subject: [PATCH 5/5] empty commit