From 43d1c8265b9db5fc707079c7c270b1e9cdf1b346 Mon Sep 17 00:00:00 2001 From: Corentin Mors Date: Mon, 3 Jun 2024 11:35:00 +0200 Subject: [PATCH] Make a tsc plugin for esbuild (#252) This includes: - using Brotli to compress final builds - embed pem files from nsm attestation in code - make a tsc plugin for esbuild --- .github/workflows/manual-test-release.yml | 2 +- .github/workflows/release.yml | 2 +- package.json | 5 +- scripts/build.js | 99 ++++++++++++++--- tsconfig.build.json | 2 +- yarn.lock | 128 ++++++++++++++++++++-- 6 files changed, 212 insertions(+), 26 deletions(-) diff --git a/.github/workflows/manual-test-release.yml b/.github/workflows/manual-test-release.yml index b00abd4..bb436e9 100644 --- a/.github/workflows/manual-test-release.yml +++ b/.github/workflows/manual-test-release.yml @@ -35,7 +35,7 @@ jobs: - run: yarn workspaces focus --all --production # package final binaries - run: | - yarn dlx @yao-pkg/pkg@5.11.1 ./dist -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C GZip "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode" + yarn dlx @yao-pkg/pkg@5.11.1 ./dist -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C Brotli "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode" - name: Archive binary artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c351560..a4dca71 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,7 +38,7 @@ jobs: - run: yarn workspaces focus --all --production # package final binaries - run: | - yarn dlx @yao-pkg/pkg@5.11.1 ./dist -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C GZip "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode" + yarn dlx @yao-pkg/pkg@5.11.1 ./dist -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C Brotli "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode" - name: Archive binary artifact uses: actions/upload-artifact@v4 diff --git a/package.json b/package.json index 996c5bd..9458e67 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,6 @@ "contributors": [], "license": "Apache-2.0", "nativeDependencies": { - "@dashlane/nsm-attestation": "*", "better-sqlite3": "*", "@json2csv/plainjs": "*", "@json2csv/transforms": "*", @@ -52,6 +51,7 @@ "node-mac-auth": "*" }, "devDependencies": { + "@aivenio/tsc-output-parser": "^2.1.1", "@types/async": "^3.2.24", "@types/better-sqlite3": "^7.6.10", "@types/chai": "^4.3.16", @@ -67,6 +67,7 @@ "eslint": "^8.57.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-import": "^2.29.1", + "execa": "^9.1.0", "husky": "^9.0.11", "mocha": "^10.4.0", "prettier": "^3.2.5", @@ -74,7 +75,7 @@ "typescript": "^5.4.5" }, "dependencies": { - "@dashlane/nsm-attestation": "^1.0.1", + "@dashlane/nsm-attestation": "^1.0.2", "@inquirer/prompts": "^5.0.5", "@json2csv/plainjs": "^7.0.6", "@json2csv/transforms": "^7.0.6", diff --git a/scripts/build.js b/scripts/build.js index 369e317..3041212 100644 --- a/scripts/build.js +++ b/scripts/build.js @@ -4,33 +4,103 @@ import os from 'os'; import fs from 'fs'; import path from 'path'; import process from 'process'; -import childProcess from 'child_process'; import esbuild from 'esbuild'; import packageJSON from '../package.json' assert { type: 'json' }; import { fileURLToPath } from 'url'; +import { $ } from "execa"; +import tscOutputParser from '@aivenio/tsc-output-parser'; const platform = os.platform(); const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); +const pemReadFilePlugin = { + name: 'base64-plugin', + setup(build) { + build.onLoad({ filter: /\.js$/ }, async (args) => { + let contents = fs.readFileSync(args.path, 'utf8'); + + const regex = /await\s+fs\.promises\.readFile\(\s*path\.resolve\(__dirname,\s*['"`](.*\.pem)['"`]\s*\)\)/g; + let match; + + while ((match = regex.exec(contents)) !== null) { + const pemFilePath = path.resolve(path.dirname(args.path), match[1]); + const pemContents = fs.readFileSync(pemFilePath, 'utf8'); + const base64Contents = Buffer.from(pemContents).toString('base64'); + contents = contents.replace(match[0], `await Promise.resolve(Buffer.from("${base64Contents}", 'base64').toString())`); + } + + return { + contents, + loader: 'js', + }; + }); + }, +}; + +const tscDiagnosticToEsbuild = async ( + diagnostic, +) => { + const lineText = + await $`sed -n ${diagnostic.value.cursor.value.line}p ${diagnostic.value.path.value}`; + + const [firstLine, rest] = diagnostic.value.message.value.split("\n", 2); + + return { + location: { + column: diagnostic.value.cursor.value.col - 1, + line: diagnostic.value.cursor.value.line, + file: diagnostic.value.path.value, + lineText: lineText.stdout, + }, + notes: rest && rest.trim().length > 0 ? [{ text: rest }] : [], + text: `${firstLine} [${diagnostic.value.tsError.value.errorString}]`, + }; +}; + +const checkTypesPlugin = () => { + return { + name: 'check-types', + setup(build) { + build.onEnd(async (result) => { + if (result.errors.length > 0) { + return; + } + + const buildArgs = ['--noEmit', '-p', './tsconfig.build.json', '--pretty', 'false']; + try { + await $('tsc', buildArgs); + } catch (err) { + const tscOutput = tscOutputParser.parse(err.stdout); + const messages = await Promise.all(tscOutput.map(output => tscDiagnosticToEsbuild(output))); + const formatted = await esbuild.formatMessages( + messages, + { + kind: 'error', + color: true, + terminalWidth: 100, + } + ); + console.log(formatted.join('\n')); + process.exit(1); + } + }); + }, + }; +}; + async function main(argv = process.argv) { argv = argv.slice(2); const projectRoot = path.join(__dirname, '..'); - const buildPath = path.join(projectRoot, 'build'); + const srcPath = path.join(projectRoot, 'src'); const distPath = path.join(projectRoot, 'dist'); const gitPath = process.env.GIT_DIR ?? path.join(projectRoot, '.git'); await fs.promises.rm(distPath, { recursive: true, force: true, }); - const buildArgs = ['-p', './tsconfig.build.json', ...argv]; - console.error('Running tsc:'); - console.error(['tsc', ...buildArgs].join(' ')); - childProcess.execFileSync('tsc', buildArgs, { - stdio: ['inherit', 'inherit', 'inherit'], - windowsHide: true, - encoding: 'utf-8', - shell: platform === 'win32' ? true : false, + await fs.promises.mkdir(distPath, { + recursive: true, }); // This collects the build metadata and adds it to the build folder so that dynamic imports to it will resolve correctly. let gitHead = process.env.COMMIT_HASH; @@ -52,7 +122,7 @@ async function main(argv = process.argv) { console.error('Writing build metadata (build.json):'); console.error(buildJSON); await fs.promises.writeFile( - path.join(buildPath, 'build.json'), + path.join(distPath, 'build.json'), JSON.stringify(buildJSON, null, 2), ); // This specifies import paths that is left as an external require @@ -60,9 +130,9 @@ async function main(argv = process.argv) { const externalDependencies = Object.keys(packageJSON.nativeDependencies ?? {}); const esbuildOptions = { entryPoints: [ - path.join(buildPath, 'index.js'), + 'src/index.ts' ], - sourceRoot: buildPath, + sourceRoot: srcPath, bundle: true, platform: 'node', format: 'cjs', @@ -74,7 +144,8 @@ async function main(argv = process.argv) { minify: true, keepNames: true, outfile: path.join(distPath, 'index.cjs'), - metafile: true + metafile: true, + plugins: [checkTypesPlugin(), pemReadFilePlugin], }; console.error('Running esbuild:'); console.error(esbuildOptions); diff --git a/tsconfig.build.json b/tsconfig.build.json index 3e49c73..0bcdcf7 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -2,7 +2,7 @@ "extends": "./tsconfig.json", "compilerOptions": { "rootDir": "./src", - "noEmit": false, + "noEmit": true, "stripInternal": true }, "exclude": ["./documentation/**/*", "./scripts/**/*"] diff --git a/yarn.lock b/yarn.lock index 57f62f0..9525100 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12,6 +12,15 @@ __metadata: languageName: node linkType: hard +"@aivenio/tsc-output-parser@npm:^2.1.1": + version: 2.1.1 + resolution: "@aivenio/tsc-output-parser@npm:2.1.1" + bin: + tsc-output-parser: dist/cli.js + checksum: 10/87125352e7e85c77efa3c90083db072ae6a93a8c0d02e6b8174f5faaf87f1efedc1de082f8711969b4f089688a159edb117f6b063d1f43a5948f16fe53d45264 + languageName: node + linkType: hard + "@babel/generator@npm:7.23.0": version: 7.23.0 resolution: "@babel/generator@npm:7.23.0" @@ -132,7 +141,8 @@ __metadata: version: 0.0.0-use.local resolution: "@dashlane/cli@workspace:." dependencies: - "@dashlane/nsm-attestation": "npm:^1.0.1" + "@aivenio/tsc-output-parser": "npm:^2.1.1" + "@dashlane/nsm-attestation": "npm:^1.0.2" "@inquirer/prompts": "npm:^5.0.5" "@json2csv/plainjs": "npm:^7.0.6" "@json2csv/transforms": "npm:^7.0.6" @@ -158,6 +168,7 @@ __metadata: eslint: "npm:^8.57.0" eslint-config-prettier: "npm:^9.1.0" eslint-plugin-import: "npm:^2.29.1" + execa: "npm:^9.1.0" got: "npm:^14.3.0" husky: "npm:^9.0.11" jsonpath-plus: "npm:^9.0.0" @@ -177,14 +188,14 @@ __metadata: languageName: unknown linkType: soft -"@dashlane/nsm-attestation@npm:^1.0.1": - version: 1.0.1 - resolution: "@dashlane/nsm-attestation@npm:1.0.1" +"@dashlane/nsm-attestation@npm:^1.0.2": + version: 1.0.2 + resolution: "@dashlane/nsm-attestation@npm:1.0.2" dependencies: cbor: "npm:9.0.2" cose-js: "npm:0.9.0" pem: "npm:1.14.8" - checksum: 10/f490927a15c3c1ef1c3c23790b7908bb58bb56524af1154fb010ed73992e2055000c1b58ae825d1ca5b52d1cf917c0e3965971a85af2b2c9841c547cff4df826 + checksum: 10/1c64e592a081a50cd0e0e108145d8f7d1cf309cd19f86a05360b8b442557b6effb103812bb140393c377f603affd890bdfa01bcf3450daaa054c67a80df3bb56 languageName: node linkType: hard @@ -1205,6 +1216,13 @@ __metadata: languageName: node linkType: hard +"@sindresorhus/merge-streams@npm:^4.0.0": + version: 4.0.0 + resolution: "@sindresorhus/merge-streams@npm:4.0.0" + checksum: 10/16551c787f5328c8ef05fd9831ade64369ccc992df78deb635ec6c44af217d2f1b43f8728c348cdc4e00585ff2fad6e00d8155199cbf6b154acc45fe65cbf0aa + languageName: node + linkType: hard + "@streamparser/json@npm:^0.0.20": version: 0.0.20 resolution: "@streamparser/json@npm:0.0.20" @@ -2331,7 +2349,7 @@ __metadata: languageName: node linkType: hard -"cross-spawn@npm:^7.0.0, cross-spawn@npm:^7.0.2": +"cross-spawn@npm:^7.0.0, cross-spawn@npm:^7.0.2, cross-spawn@npm:^7.0.3": version: 7.0.3 resolution: "cross-spawn@npm:7.0.3" dependencies: @@ -2902,6 +2920,26 @@ __metadata: languageName: node linkType: hard +"execa@npm:^9.1.0": + version: 9.1.0 + resolution: "execa@npm:9.1.0" + dependencies: + "@sindresorhus/merge-streams": "npm:^4.0.0" + cross-spawn: "npm:^7.0.3" + figures: "npm:^6.1.0" + get-stream: "npm:^9.0.0" + human-signals: "npm:^7.0.0" + is-plain-obj: "npm:^4.1.0" + is-stream: "npm:^4.0.1" + npm-run-path: "npm:^5.2.0" + pretty-ms: "npm:^9.0.0" + signal-exit: "npm:^4.1.0" + strip-final-newline: "npm:^4.0.0" + yoctocolors: "npm:^2.0.0" + checksum: 10/a764df0bfa1c5d5df8cc54f8aaf2dd1757389e2c80f2aa3b77cc4d3bebe5b6fcade904469877149af8afba0d305a86e04102934bd7e25c4d07c96688ee77426c + languageName: node + linkType: hard + "expand-template@npm:^2.0.3": version: 2.0.3 resolution: "expand-template@npm:2.0.3" @@ -2984,6 +3022,15 @@ __metadata: languageName: node linkType: hard +"figures@npm:^6.1.0": + version: 6.1.0 + resolution: "figures@npm:6.1.0" + dependencies: + is-unicode-supported: "npm:^2.0.0" + checksum: 10/9822d13630bee8e6a9f2da866713adf13854b07e0bfde042defa8bba32d47a1c0b2afa627ce73837c674cf9a5e3edce7e879ea72cb9ea7960b2390432d8e1167 + languageName: node + linkType: hard + "file-entry-cache@npm:^6.0.1": version: 6.0.1 resolution: "file-entry-cache@npm:6.0.1" @@ -3234,7 +3281,7 @@ __metadata: languageName: node linkType: hard -"get-stream@npm:^9.0.1": +"get-stream@npm:^9.0.0, get-stream@npm:^9.0.1": version: 9.0.1 resolution: "get-stream@npm:9.0.1" dependencies: @@ -3534,6 +3581,13 @@ __metadata: languageName: node linkType: hard +"human-signals@npm:^7.0.0": + version: 7.0.0 + resolution: "human-signals@npm:7.0.0" + checksum: 10/f1356547f6553a90530527ba2325c2281e33fa3b9b372a647e377fd8f24e9eb8119082051ca6a9f8821e24f1faa13cd462c849a9476c86e7c190dbf0aa2ee8a4 + languageName: node + linkType: hard + "humanize-ms@npm:^1.2.1": version: 1.2.1 resolution: "humanize-ms@npm:1.2.1" @@ -3830,6 +3884,13 @@ __metadata: languageName: node linkType: hard +"is-plain-obj@npm:^4.1.0": + version: 4.1.0 + resolution: "is-plain-obj@npm:4.1.0" + checksum: 10/6dc45da70d04a81f35c9310971e78a6a3c7a63547ef782e3a07ee3674695081b6ca4e977fbb8efc48dae3375e0b34558d2bcd722aec9bddfa2d7db5b041be8ce + languageName: node + linkType: hard + "is-regex@npm:^1.1.4": version: 1.1.4 resolution: "is-regex@npm:1.1.4" @@ -3897,6 +3958,13 @@ __metadata: languageName: node linkType: hard +"is-unicode-supported@npm:^2.0.0": + version: 2.0.0 + resolution: "is-unicode-supported@npm:2.0.0" + checksum: 10/000b80639dedaf59a385f1c0a57f97a4d1435e0723716f24cc19ad94253a7a0a9f838bdc9ac49b10a29ac93b01f52ae9b2ed358a8876caf1eb74d73b4ede92b2 + languageName: node + linkType: hard + "is-weakref@npm:^1.0.2": version: 1.0.2 resolution: "is-weakref@npm:1.0.2" @@ -4589,6 +4657,15 @@ __metadata: languageName: node linkType: hard +"npm-run-path@npm:^5.2.0": + version: 5.3.0 + resolution: "npm-run-path@npm:5.3.0" + dependencies: + path-key: "npm:^4.0.0" + checksum: 10/ae8e7a89da9594fb9c308f6555c73f618152340dcaae423e5fb3620026fefbec463618a8b761920382d666fa7a2d8d240b6fe320e8a6cdd54dc3687e2b659d25 + languageName: node + linkType: hard + "npmlog@npm:^6.0.0": version: 6.0.2 resolution: "npmlog@npm:6.0.2" @@ -4761,6 +4838,13 @@ __metadata: languageName: node linkType: hard +"parse-ms@npm:^4.0.0": + version: 4.0.0 + resolution: "parse-ms@npm:4.0.0" + checksum: 10/673c801d9f957ff79962d71ed5a24850163f4181a90dd30c4e3666b3a804f53b77f1f0556792e8b2adbb5d58757907d1aa51d7d7dc75997c2a56d72937cbc8b7 + languageName: node + linkType: hard + "path-exists@npm:^4.0.0": version: 4.0.0 resolution: "path-exists@npm:4.0.0" @@ -4782,6 +4866,13 @@ __metadata: languageName: node linkType: hard +"path-key@npm:^4.0.0": + version: 4.0.0 + resolution: "path-key@npm:4.0.0" + checksum: 10/8e6c314ae6d16b83e93032c61020129f6f4484590a777eed709c4a01b50e498822b00f76ceaf94bc64dbd90b327df56ceadce27da3d83393790f1219e07721d7 + languageName: node + linkType: hard + "path-parse@npm:^1.0.7": version: 1.0.7 resolution: "path-parse@npm:1.0.7" @@ -4879,6 +4970,15 @@ __metadata: languageName: node linkType: hard +"pretty-ms@npm:^9.0.0": + version: 9.0.0 + resolution: "pretty-ms@npm:9.0.0" + dependencies: + parse-ms: "npm:^4.0.0" + checksum: 10/b11e1eda41a2efcc16aab218392c8e457a8ae5c8edf63aafba0477123426b1268136b9b532cbfd84625bcb826739120ec8490286dab66102b9f09e717bdb4e45 + languageName: node + linkType: hard + "process-nextick-args@npm:~2.0.0": version: 2.0.1 resolution: "process-nextick-args@npm:2.0.1" @@ -5440,6 +5540,13 @@ __metadata: languageName: node linkType: hard +"strip-final-newline@npm:^4.0.0": + version: 4.0.0 + resolution: "strip-final-newline@npm:4.0.0" + checksum: 10/b5fe48f695d74863153a3b3155220e6e9bf51f4447832998c8edec38e6559b3af87a9fe5ac0df95570a78a26f5fa91701358842eab3c15480e27980b154a145f + languageName: node + linkType: hard + "strip-json-comments@npm:3.1.1, strip-json-comments@npm:^3.1.1": version: 3.1.1 resolution: "strip-json-comments@npm:3.1.1" @@ -6041,6 +6148,13 @@ __metadata: languageName: node linkType: hard +"yoctocolors@npm:^2.0.0": + version: 2.0.2 + resolution: "yoctocolors@npm:2.0.2" + checksum: 10/cac20504b5fc954ff117e3a3fbde09db8ac0807bba59e68c5c08f3a43173ef46ccb1853b15b37bd96d0d8187bc444627f160fee7e5aede0b421382cf379d2438 + languageName: node + linkType: hard + "zlib@npm:^1.0.5": version: 1.0.5 resolution: "zlib@npm:1.0.5"