From b1bcc6c22eabb122954d9db89a627b6abb1e564c Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Sun, 25 Jun 2023 19:38:40 +1000 Subject: [PATCH 1/2] WIP --- package.json | 9 ++- scripts/prepublishOnly.js | 133 ++++++++++++++++++++++++++++++++++++++ scripts/version.js | 55 ++++++++++++++++ 3 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 scripts/prepublishOnly.js create mode 100644 scripts/version.js diff --git a/package.json b/package.json index 7206b085..ace0d391 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "install": "node-gyp-build", "prebuild": "node ./scripts/prebuild.js", "build": "shx rm -rf ./dist && tsc -p ./tsconfig.build.json", - "postversion": "npm install --package-lock-only --ignore-scripts --silent", + "version": "node ./scripts/version.js", + "prepublishOnly": "node ./scripts/prepublishOnly.js", "ts-node": "ts-node", "test": "jest", "lint": "eslint '{src,tests,scripts,benches}/**/*.{js,ts}'", @@ -37,6 +38,12 @@ "node-gyp-build": "4.4.0", "threads": "^1.6.5" }, + "optionalDependencies": { + "@matrixai/db-darwin-arm64": "5.1.0", + "@matrixai/db-darwin-x64": "5.1.0", + "@matrixai/db-linux-x64": "5.1.0", + "@matrixai/db-win32-x64": "5.1.0" + }, "devDependencies": { "@swc/core": "^1.3.62", "@swc/jest": "^0.2.26", diff --git a/scripts/prepublishOnly.js b/scripts/prepublishOnly.js new file mode 100644 index 00000000..1b3bea01 --- /dev/null +++ b/scripts/prepublishOnly.js @@ -0,0 +1,133 @@ +#!/usr/bin/env node + +/** + * This runs before `npm publish` command. + * This will take the native objects in `prebuild/` + * and create native packages under `prepublishOnly/`. + * + * For example: + * + * /prepublishOnly + * /@org + * /name-linux-x64 + * /package.json + * /node.napi.node + * /README.md + */ + +const os = require('os'); +const fs = require('fs'); +const path = require('path'); +const process = require('process'); +const childProcess = require('child_process'); +const packageJSON = require('../package.json'); + +const platform = os.platform(); + +/* eslint-disable no-console */ +async function main(argv = process.argv) { + argv = argv.slice(2); + let tag; + let dryRun = false; + const restArgs = []; + while (argv.length > 0) { + const option = argv.shift(); + let match; + if ((match = option.match(/--tag(?:=(.+)|$)/))) { + tag = match[1] ?? argv.shift(); + } else if ((match = option.match(/--dry-run$/))) { + dryRun = true; + } else { + restArgs.push(option); + } + } + if (tag == null) { + tag = process.env.npm_config_tag; + } + const projectRoot = path.join(__dirname, '..'); + const prebuildPath = path.join(projectRoot, 'prebuild'); + const prepublishOnlyPath = path.join(projectRoot, 'prepublishOnly'); + const buildNames = (await fs.promises.readdir(prebuildPath)).filter( + (filename) => /^(?:[^-]+)-(?:[^-]+)-(?:[^-]+)$/.test(filename), + ); + if (buildNames.length < 1) { + console.error( + 'You must prebuild at least 1 native object with the filename of `name-platform-arch` before prepublish', + ); + process.exitCode = 1; + return process.exitCode; + } + // Extract out the org name, this may be undefined + const orgName = packageJSON.name.match(/^@[^/]+/)?.[0]; + for (const buildName of buildNames) { + // This is `name-platform-arch` + const name = path.basename(buildName, '.node'); + // This is `@org/name-platform-arch`, uses `posix` to force usage of `/` + const packageName = path.posix.join(orgName ?? '', name); + const constraints = name.match( + /^(?:[^-]+)-(?[^-]+)-(?[^-]+)$/, + ); + // This will be `prebuild/name-platform-arch.node` + const buildPath = path.join(prebuildPath, buildName); + // This will be `prepublishOnly/@org/name-platform-arch` + const packagePath = path.join(prepublishOnlyPath, packageName); + console.error('Packaging:', packagePath); + try { + await fs.promises.rm(packagePath, { + recursive: true, + }); + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } + await fs.promises.mkdir(packagePath, { recursive: true }); + const nativePackageJSON = { + name: packageName, + version: packageJSON.version, + homepage: packageJSON.homepage, + author: packageJSON.author, + contributors: packageJSON.contributors, + description: packageJSON.description, + keywords: packageJSON.keywords, + license: packageJSON.license, + repository: packageJSON.repository, + main: 'node.napi.node', + os: [constraints.groups.platform], + cpu: [...constraints.groups.arch.split('+')], + }; + const packageJSONPath = path.join(packagePath, 'package.json'); + console.error(`Writing ${packageJSONPath}`); + const packageJSONString = JSON.stringify(nativePackageJSON, null, 2); + console.error(packageJSONString); + await fs.promises.writeFile(packageJSONPath, packageJSONString, { + encoding: 'utf-8', + }); + const packageReadmePath = path.join(packagePath, 'README.md'); + console.error(`Writing ${packageReadmePath}`); + const packageReadme = `# ${packageName}\n`; + console.error(packageReadme); + await fs.promises.writeFile(packageReadmePath, packageReadme, { + encoding: 'utf-8', + }); + const packageBuildPath = path.join(packagePath, 'node.napi.node'); + console.error(`Copying ${buildPath} to ${packageBuildPath}`); + await fs.promises.copyFile(buildPath, packageBuildPath); + const publishArgs = [ + 'publish', + packagePath, + ...(tag != null ? [`--tag=${tag}`] : []), + '--access=public', + ...(dryRun ? ['--dry-run'] : []), + ]; + console.error('Running npm publish:'); + console.error(['npm', ...publishArgs].join(' ')); + childProcess.execFileSync('npm', publishArgs, { + stdio: ['inherit', 'inherit', 'inherit'], + windowsHide: true, + encoding: 'utf-8', + shell: platform === 'win32' ? true : false, + }); + } +} +/* eslint-enable no-console */ + +void main(); diff --git a/scripts/version.js b/scripts/version.js new file mode 100644 index 00000000..cbe5af15 --- /dev/null +++ b/scripts/version.js @@ -0,0 +1,55 @@ +#!/usr/bin/env node + +/** + * This runs after `npm version` command updates the version but before changes are commited. + * This will also update the `package.json` optional native dependencies + * to match the same version as the version of this package. + * This maintains the same version between this master package + * and the optional native dependencies. + * At the same time, the `package-lock.json` is also regenerated. + * Note that at this point, the new optional native dependencies have + * not yet been published, so the `--package-lock-only` flag is used + * to prevent `npm` from attempting to download unpublished packages. + */ + +const path = require('path'); +const os = require('os'); +const childProcess = require('child_process'); +const packageJSON = require('../package.json'); + +const platform = os.platform(); + +/* eslint-disable no-console */ +async function main() { + console.error( + 'Updating the package.json with optional native dependencies and package-lock.json', + ); + const optionalDepsNative = []; + for (const key in packageJSON.optionalDependencies) { + if (key.startsWith(packageJSON.name)) { + optionalDepsNative.push(`${key}@${packageJSON.version}`); + } + } + if (optionalDepsNative.length > 0) { + const installArgs = [ + 'install', + '--ignore-scripts', + '--silent', + '--package-lock-only', + '--save-optional', + '--save-exact', + ...optionalDepsNative, + ]; + console.error('Running npm install:'); + console.error(['npm', ...installArgs].join(' ')); + childProcess.execFileSync('npm', installArgs, { + stdio: ['inherit', 'inherit', 'inherit'], + windowsHide: true, + encoding: 'utf-8', + shell: platform === 'win32' ? true : false, + }); + } +} +/* eslint-enable no-console */ + +void main(); From d0150235c2c09dc976890a60f59864af0a71021c Mon Sep 17 00:00:00 2001 From: Roger Qiu Date: Sun, 25 Jun 2023 20:26:10 +1000 Subject: [PATCH 2/2] WIP --- package.json | 2 - src/native/rocksdb.ts | 99 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 96 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index ace0d391..e4146c66 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,6 @@ "gypfile": true, "scripts": { "prepare": "tsc -p ./tsconfig.build.json", - "install": "node-gyp-build", "prebuild": "node ./scripts/prebuild.js", "build": "shx rm -rf ./dist && tsc -p ./tsconfig.build.json", "version": "node ./scripts/version.js", @@ -35,7 +34,6 @@ "@matrixai/logger": "^3.1.0", "@matrixai/resources": "^1.1.5", "@matrixai/workers": "^1.3.7", - "node-gyp-build": "4.4.0", "threads": "^1.6.5" }, "optionalDependencies": { diff --git a/src/native/rocksdb.ts b/src/native/rocksdb.ts index f97265c7..0d2ec1de 100644 --- a/src/native/rocksdb.ts +++ b/src/native/rocksdb.ts @@ -19,7 +19,6 @@ import type { RocksDBCountOptions, } from './types'; import path from 'path'; -import nodeGypBuild from 'node-gyp-build'; interface RocksDB { dbInit(): RocksDBDatabase; @@ -271,8 +270,102 @@ interface RocksDB { ): void; } -const rocksdb: RocksDB = nodeGypBuild(path.join(__dirname, '../../')); +const projectRoot = path.join(__dirname, '../../'); +const prebuildPath = path.join(projectRoot, 'prebuild'); -export default rocksdb; +/** + * Try require on all prebuild targets first, then + * try require on all npm targets second. + */ +function requireBinding(targets: Array): RocksDB { + const prebuildTargets = targets.map((target) => + path.join(prebuildPath, `db-${target}.node`), + ); + for (const prebuildTarget of prebuildTargets) { + try { + return require(prebuildTarget); + } catch (e) { + if (e.code !== 'MODULE_NOT_FOUND') throw e; + } + } + const npmTargets = targets.map((target) => `@matrixai/db-${target}`); + for (const npmTarget of npmTargets) { + try { + return require(npmTarget); + } catch (e) { + if (e.code !== 'MODULE_NOT_FOUND') throw e; + } + } + throw new Error( + `Failed requiring possible native bindings: ${prebuildTargets.concat( + npmTargets, + )}`, + ); +} + +let nativeBinding: RocksDB; + +/** + * For desktop we only support win32, darwin and linux. + * Mobile OS support is pending. + */ +switch (process.platform) { + case 'win32': + switch (process.arch) { + case 'x64': + nativeBinding = requireBinding(['win32-x64']); + break; + case 'ia32': + nativeBinding = requireBinding(['win32-ia32']); + break; + case 'arm64': + nativeBinding = requireBinding(['win32-arm64']); + break; + default: + throw new Error(`Unsupported architecture on Windows: ${process.arch}`); + } + break; + case 'darwin': + switch (process.arch) { + case 'x64': + nativeBinding = requireBinding([ + 'darwin-x64', + 'darwin-x64+arm64', + 'darwin-arm64+x64', + ]); + break; + case 'arm64': + nativeBinding = requireBinding([ + 'darwin-arm64', + 'darwin-arm64+x64', + 'darwin-x64+arm64', + ]); + break; + default: + throw new Error(`Unsupported architecture on macOS: ${process.arch}`); + } + break; + case 'linux': + switch (process.arch) { + case 'x64': + nativeBinding = requireBinding(['linux-x64']); + break; + case 'arm64': + nativeBinding = requireBinding(['linux-arm64']); + break; + case 'arm': + nativeBinding = requireBinding(['linux-arm']); + break; + default: + throw new Error(`Unsupported architecture on Linux: ${process.arch}`); + } + break; + default: + throw new Error( + `Unsupported OS: ${process.platform}, architecture: ${process.arch}`, + ); +} + +export default nativeBinding; export type { RocksDB };