From 218427b4f20f6ab7a7f247bc3fafd00de0720a40 Mon Sep 17 00:00:00 2001 From: Miki Date: Mon, 20 Feb 2023 11:10:00 -0800 Subject: [PATCH] Make build scripts find and use the latest version of Node.js that satisfies `engines.node` * While building distributables, Node.js runtime is downloaded to be placed in the archivea. This logicwas modified to honor a range for `engines.node` by fetching the latest release of Node.js that satisfied the range. * Some tests covering the build, read a version from `.node-version` to compare with the results of actual function runs; these were changed to either use mocked values or honor the range and use the latest Node.js version. * Some variable and functions referred to `engines.node` as a version; they were corrected to call it a range. Signed-off-by: Miki --- CHANGELOG.md | 1 + src/dev/build/lib/config.test.ts | 6 ++-- src/dev/build/lib/config.ts | 6 ++-- .../tasks/create_archives_sources_task.ts | 2 +- .../tasks/nodejs/download_node_builds_task.ts | 7 +++-- .../nodejs/extract_node_builds_task.test.ts | 23 ++++++++------- .../tasks/nodejs/extract_node_builds_task.ts | 2 +- .../build/tasks/nodejs/node_download_info.ts | 28 +++++++++++++++++-- .../verify_existing_node_builds_task.test.ts | 23 ++++++--------- .../verify_existing_node_builds_task.ts | 7 +++-- src/dev/build/tasks/notice_file_task.ts | 2 +- src/dev/build/tasks/verify_env_task.ts | 9 ++++-- 12 files changed, 70 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 958dce2a375f..fa50fdd53d18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - [Data] Update `createAggConfig` so that newly created configs can be added to beginning of `aggConfig` array ([#3160](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3160)) - Add disablePrototypePoisoningProtection configuration to prevent JS client from erroring when cluster utilizes JS reserved words ([#2992](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2992)) - [Multiple DataSource] Add support for SigV4 authentication ([#3058](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/3058)) +- Make build scripts find and use the latest version of Node.js that satisfies `engines.node` ([#3467](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/3467)) ### 🐛 Bug Fixes diff --git a/src/dev/build/lib/config.test.ts b/src/dev/build/lib/config.test.ts index a625aab9e256..145954c1fb40 100644 --- a/src/dev/build/lib/config.test.ts +++ b/src/dev/build/lib/config.test.ts @@ -79,10 +79,10 @@ describe('#getOpenSearchDashboardsPkg()', () => { }); }); -describe('#getNodeVersion()', () => { - it('returns the node version from the OpenSearch Dashboards package.json', async () => { +describe('#getNodeRange()', () => { + it('returns the node version range from the OpenSearch Dashboards package.json', async () => { const config = await setup(); - expect(config.getNodeVersion()).toEqual(pkg.engines.node); + expect(config.getNodeRange()).toEqual(pkg.engines.node); }); }); diff --git a/src/dev/build/lib/config.ts b/src/dev/build/lib/config.ts index 4f47a5ec8f50..03fdd14b05c0 100644 --- a/src/dev/build/lib/config.ts +++ b/src/dev/build/lib/config.ts @@ -86,7 +86,7 @@ export class Config { private readonly targetAllPlatforms: boolean, private readonly targetPlatforms: TargetPlatforms, private readonly pkg: Package, - private readonly nodeVersion: string, + private readonly nodeRange: string, private readonly repoRoot: string, private readonly versionInfo: VersionInfo, public readonly isRelease: boolean @@ -102,8 +102,8 @@ export class Config { /** * Get the node version required by OpenSearch Dashboards */ - getNodeVersion() { - return this.nodeVersion; + getNodeRange() { + return this.nodeRange; } /** diff --git a/src/dev/build/tasks/create_archives_sources_task.ts b/src/dev/build/tasks/create_archives_sources_task.ts index 0409c4067088..bf53c2915560 100644 --- a/src/dev/build/tasks/create_archives_sources_task.ts +++ b/src/dev/build/tasks/create_archives_sources_task.ts @@ -50,7 +50,7 @@ export const CreateArchivesSources: Task = { // copy node.js install await scanCopy({ - source: getNodeDownloadInfo(config, platform).extractDir, + source: (await getNodeDownloadInfo(config, platform)).extractDir, destination: build.resolvePathForPlatform(platform, 'node'), }); diff --git a/src/dev/build/tasks/nodejs/download_node_builds_task.ts b/src/dev/build/tasks/nodejs/download_node_builds_task.ts index 91331184ce76..9771e92e209f 100644 --- a/src/dev/build/tasks/nodejs/download_node_builds_task.ts +++ b/src/dev/build/tasks/nodejs/download_node_builds_task.ts @@ -30,16 +30,17 @@ import { download, GlobalTask } from '../../lib'; import { getNodeShasums } from './node_shasums'; -import { getNodeDownloadInfo } from './node_download_info'; +import { getLatestNodeVersion, getNodeDownloadInfo } from './node_download_info'; export const DownloadNodeBuilds: GlobalTask = { global: true, description: 'Downloading node.js builds for all platforms', async run(config, log) { - const shasums = await getNodeShasums(log, config.getNodeVersion()); + const latestNodeVersion = await getLatestNodeVersion(config); + const shasums = await getNodeShasums(log, latestNodeVersion); await Promise.all( config.getTargetPlatforms().map(async (platform) => { - const { url, downloadPath, downloadName } = getNodeDownloadInfo(config, platform); + const { url, downloadPath, downloadName } = await getNodeDownloadInfo(config, platform); await download({ log, url, diff --git a/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts index 6b3a066a1d2d..02a6a7bfa723 100644 --- a/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts @@ -28,10 +28,6 @@ * under the License. */ -import { readFileSync } from 'fs'; -import Path from 'path'; - -import { REPO_ROOT } from '@osd/utils'; import { ToolingLog, ToolingLogCollectingWriter, @@ -41,6 +37,7 @@ import { import { Config } from '../../lib'; import { ExtractNodeBuilds } from './extract_node_builds_task'; +import { getLatestNodeVersion } from './node_download_info'; jest.mock('../../lib/fs'); jest.mock('../../lib/get_build_number'); @@ -53,14 +50,6 @@ log.setWriters([testWriter]); expect.addSnapshotSerializer(createAbsolutePathSerializer()); -const nodeVersion = readFileSync(Path.resolve(REPO_ROOT, '.node-version'), 'utf8').trim(); -expect.addSnapshotSerializer( - createRecursiveSerializer( - (s) => typeof s === 'string' && s.includes(nodeVersion), - (s) => s.split(nodeVersion).join('') - ) -); - async function setup() { const config = await Config.create({ isRelease: true, @@ -73,6 +62,16 @@ async function setup() { }, }); + const realNodeVersion = await getLatestNodeVersion(config); + if (realNodeVersion) { + expect.addSnapshotSerializer( + createRecursiveSerializer( + (s) => typeof s === 'string' && s.includes(realNodeVersion), + (s) => s.split(realNodeVersion).join('') + ) + ); + } + return { config }; } diff --git a/src/dev/build/tasks/nodejs/extract_node_builds_task.ts b/src/dev/build/tasks/nodejs/extract_node_builds_task.ts index 8252ce2153f5..28b2ebe24d4b 100644 --- a/src/dev/build/tasks/nodejs/extract_node_builds_task.ts +++ b/src/dev/build/tasks/nodejs/extract_node_builds_task.ts @@ -37,7 +37,7 @@ export const ExtractNodeBuilds: GlobalTask = { async run(config) { await Promise.all( config.getTargetPlatforms().map(async (platform) => { - const { downloadPath, extractDir } = getNodeDownloadInfo(config, platform); + const { downloadPath, extractDir } = await getNodeDownloadInfo(config, platform); if (platform.isWindows()) { await unzip(downloadPath, extractDir, { strip: 1 }); } else { diff --git a/src/dev/build/tasks/nodejs/node_download_info.ts b/src/dev/build/tasks/nodejs/node_download_info.ts index 86e0c680ab0d..e2101d3453be 100644 --- a/src/dev/build/tasks/nodejs/node_download_info.ts +++ b/src/dev/build/tasks/nodejs/node_download_info.ts @@ -29,11 +29,15 @@ */ import { basename } from 'path'; +import fetch from 'node-fetch'; +import semver from 'semver'; import { Config, Platform } from '../../lib'; -export function getNodeDownloadInfo(config: Config, platform: Platform) { - const version = config.getNodeVersion(); +const NODE_RANGE_CACHE: { [key: string]: string } = {}; + +export async function getNodeDownloadInfo(config: Config, platform: Platform) { + const version = await getLatestNodeVersion(config); const arch = platform.getNodeArch(); const downloadName = platform.isWindows() @@ -52,3 +56,23 @@ export function getNodeDownloadInfo(config: Config, platform: Platform) { version, }; } + +export async function getLatestNodeVersion(config: Config) { + const range = config.getNodeRange(); + // Check cache and return if known + if (NODE_RANGE_CACHE[range]) return NODE_RANGE_CACHE[range]; + + const releaseDoc = await fetch('https://nodejs.org/dist/index.json'); + const releaseList: [{ version: string }] = await releaseDoc.json(); + const releases = releaseList.map(({ version }) => version.replace(/^v/, '')); + const maxVersion = semver.maxSatisfying(releases, range); + + if (!maxVersion) { + throw new Error(`Cannot find a version of Node.js that satisfies ${range}.`); + } + + // Cache it + NODE_RANGE_CACHE[range] = maxVersion; + + return maxVersion; +} diff --git a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts index ccb75897ccfd..4724fa73e9f0 100644 --- a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts @@ -28,15 +28,10 @@ * under the License. */ -import Path from 'path'; -import Fs from 'fs'; - -import { REPO_ROOT } from '@osd/utils'; import { ToolingLog, ToolingLogCollectingWriter, createAnyInstanceSerializer, - createRecursiveSerializer, } from '@osd/dev-utils'; import { Config, Platform } from '../../lib'; @@ -48,7 +43,7 @@ jest.mock('../../lib/fs'); jest.mock('../../lib/get_build_number'); const { getNodeShasums } = jest.requireMock('./node_shasums'); -const { getNodeDownloadInfo } = jest.requireMock('./node_download_info'); +const { getNodeDownloadInfo, getLatestNodeVersion } = jest.requireMock('./node_download_info'); const { getFileHash } = jest.requireMock('../../lib/fs'); const log = new ToolingLog(); @@ -58,14 +53,6 @@ log.setWriters([testWriter]); expect.addSnapshotSerializer(createAnyInstanceSerializer(Config)); expect.addSnapshotSerializer(createAnyInstanceSerializer(ToolingLog)); -const nodeVersion = Fs.readFileSync(Path.resolve(REPO_ROOT, '.node-version'), 'utf8').trim(); -expect.addSnapshotSerializer( - createRecursiveSerializer( - (s) => typeof s === 'string' && s.includes(nodeVersion), - (s) => s.split(nodeVersion).join('') - ) -); - async function setup(actualShaSums?: Record) { const config = await Config.create({ isRelease: true, @@ -74,6 +61,7 @@ async function setup(actualShaSums?: Record) { linux: false, linuxArm: false, darwin: false, + windows: false, }, }); @@ -89,9 +77,12 @@ async function setup(actualShaSums?: Record) { return { downloadPath: `${platform.getName()}:${platform.getNodeArch()}:downloadPath`, downloadName: `${platform.getName()}:${platform.getNodeArch()}:downloadName`, + version: '', }; }); + getLatestNodeVersion.mockReturnValue(''); + getFileHash.mockImplementation((downloadPath: string) => { if (actualShaSums?.[downloadPath]) { return actualShaSums[downloadPath]; @@ -176,6 +167,7 @@ it('checks shasums for each downloaded node build', async () => { "value": Object { "downloadName": "linux:linux-x64:downloadName", "downloadPath": "linux:linux-x64:downloadPath", + "version": "", }, }, Object { @@ -183,6 +175,7 @@ it('checks shasums for each downloaded node build', async () => { "value": Object { "downloadName": "linux:linux-arm64:downloadName", "downloadPath": "linux:linux-arm64:downloadPath", + "version": "", }, }, Object { @@ -190,6 +183,7 @@ it('checks shasums for each downloaded node build', async () => { "value": Object { "downloadName": "darwin:darwin-x64:downloadName", "downloadPath": "darwin:darwin-x64:downloadPath", + "version": "", }, }, Object { @@ -197,6 +191,7 @@ it('checks shasums for each downloaded node build', async () => { "value": Object { "downloadName": "win32:win32-x64:downloadName", "downloadPath": "win32:win32-x64:downloadPath", + "version": "", }, }, ], diff --git a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.ts b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.ts index 665774bf5bc7..4c64c4bbd25d 100644 --- a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.ts +++ b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.ts @@ -29,18 +29,19 @@ */ import { getFileHash, GlobalTask } from '../../lib'; -import { getNodeDownloadInfo } from './node_download_info'; +import { getNodeDownloadInfo, getLatestNodeVersion } from './node_download_info'; import { getNodeShasums } from './node_shasums'; export const VerifyExistingNodeBuilds: GlobalTask = { global: true, description: 'Verifying previously downloaded node.js build for all platforms', async run(config, log) { - const shasums = await getNodeShasums(log, config.getNodeVersion()); + const latestNodeVersion = await getLatestNodeVersion(config); + const shasums = await getNodeShasums(log, latestNodeVersion); await Promise.all( config.getTargetPlatforms().map(async (platform) => { - const { downloadPath, downloadName } = getNodeDownloadInfo(config, platform); + const { downloadPath, downloadName } = await getNodeDownloadInfo(config, platform); const sha256 = await getFileHash(downloadPath, 'sha256'); if (sha256 !== shasums[downloadName]) { diff --git a/src/dev/build/tasks/notice_file_task.ts b/src/dev/build/tasks/notice_file_task.ts index 5532d2d5db41..08792b653f49 100644 --- a/src/dev/build/tasks/notice_file_task.ts +++ b/src/dev/build/tasks/notice_file_task.ts @@ -57,7 +57,7 @@ export const CreateNoticeFile: Task = { log.info('Generating build notice'); - const { extractDir: nodeDir, version: nodeVersion } = getNodeDownloadInfo( + const { extractDir: nodeDir, version: nodeVersion } = await getNodeDownloadInfo( config, config.hasSpecifiedPlatform() ? config.getPlatform( diff --git a/src/dev/build/tasks/verify_env_task.ts b/src/dev/build/tasks/verify_env_task.ts index a6b3bd885576..ceef5c8d38c5 100644 --- a/src/dev/build/tasks/verify_env_task.ts +++ b/src/dev/build/tasks/verify_env_task.ts @@ -28,6 +28,7 @@ * under the License. */ +import semver from 'semver'; import { GlobalTask } from '../lib'; export const VerifyEnv: GlobalTask = { @@ -35,10 +36,12 @@ export const VerifyEnv: GlobalTask = { description: 'Verifying environment meets requirements', async run(config, log) { - const version = `v${config.getNodeVersion()}`; + const range = config.getNodeRange(); - if (version !== process.version) { - throw new Error(`Invalid nodejs version, please use ${version}`); + if (!semver.satisfies(process.version, range)) { + throw new Error( + `Invalid Node.js version (${process.version}); please use a version that satisfies ${range}.` + ); } log.success('Node.js version verified');