From 447a94c445539385b22d119dea0790a1f940112b Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Tue, 8 Jun 2021 02:16:14 +0000 Subject: [PATCH] [Tests] updated artifact script Updating artifact to enable running integration tests and functional tests to pull an artifact from the current hosted distributions. At the time of this commit, there is not manifest hosted but there are static links which can have an unknown amount of RC versions if the GA snapshot does not exist for that version. But the assumption is that it will not be too high. Deprecating the previous implementation but we can remove that in a future iteration. Wanted to leave that available incase others use that for custom manifests. Enable tests that depended on snapshots as well. Issues resolved: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/242 https://github.com/opensearch-project/OpenSearch-Dashboards/issues/19 Signed-off-by: Kawika Avilla --- .../__tests__/generate_doc_records_stream.ts | 2 +- packages/osd-opensearch/src/artifact.js | 93 +++++++++- packages/osd-opensearch/src/artifact.test.js | 165 ++++++++---------- .../create_or_upgrade.test.ts | 3 +- .../integration_tests/index.test.ts | 3 +- 5 files changed, 164 insertions(+), 102 deletions(-) diff --git a/packages/osd-opensearch-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts b/packages/osd-opensearch-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts index 86e2bd43eab2..1292ace5e56e 100644 --- a/packages/osd-opensearch-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts +++ b/packages/osd-opensearch-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts @@ -104,7 +104,7 @@ describe('opensearchArchiver: createGenerateDocRecordsStream()', () => { await delay(200); return { _scroll_id: 'index1ScrollId', - hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_foo' }] }, + hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_1' }] }, }; }, async (name, params) => { diff --git a/packages/osd-opensearch/src/artifact.js b/packages/osd-opensearch/src/artifact.js index 491eaec8f1f1..8ece5c02b8dd 100644 --- a/packages/osd-opensearch/src/artifact.js +++ b/packages/osd-opensearch/src/artifact.js @@ -29,7 +29,6 @@ * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ - const fetch = require('node-fetch'); const AbortController = require('abort-controller'); const fs = require('fs'); @@ -40,8 +39,11 @@ const { createHash } = require('crypto'); const path = require('path'); const asyncPipeline = promisify(pipeline); -const DAILY_SNAPSHOTS_BASE_URL = ''; +const DAILY_SNAPSHOTS_BASE_URL = 'https://artifacts.opensearch.org/snapshots/core/opensearch'; +// TODO: [RENAMEME] currently do not have an existing replacement const PERMANENT_SNAPSHOTS_BASE_URL = ''; +// Since we do not have a manifest URL, limiting how many RC checks to 5 should be more than enough +const MAX_RC_CHECK = 5; const { cache } = require('./utils'); const { resolveCustomSnapshotUrl } = require('./custom_snapshots'); @@ -96,6 +98,21 @@ async function fetchSnapshotManifest(url, log) { return { abc, resp, json }; } +async function verifySnapshotUrl(url, log) { + log.info('Verifying snapshot URL at %s', chalk.bold(url)); + + const abc = new AbortController(); + const resp = await retry( + log, + async () => await fetch(url, { signal: abc.signal }, { method: 'HEAD' }) + ); + + return { abc, resp }; +} + +/** + * @deprecated This method should not be used, uses logic for resources we do not have access to. + */ async function getArtifactSpecForSnapshot(urlVersion, license, log) { const desiredVersion = urlVersion.replace('-SNAPSHOT', ''); const desiredLicense = license === 'oss' ? 'oss' : 'default'; @@ -153,6 +170,76 @@ async function getArtifactSpecForSnapshot(urlVersion, license, log) { }; } +async function getArtifactSpecForSnapshotFromUrl(urlVersion, log) { + const desiredVersion = urlVersion.replace('-SNAPSHOT', ''); + + if (process.env.OPENSEARCH_SNAPSHOT_MANIFEST) { + return await getArtifactSpecForSnapshot(urlVersion, 'oss', log); + } + + // [RENAMEME] Need replacement for other platforms. + const platform = process.platform === 'win32' ? 'windows' : process.platform; + const arch = process.arch === 'arm64' ? 'arm64' : 'x64'; + if (platform !== 'linux' || arch !== 'x64') { + throw createCliError(`Snapshots are only available for Linux x64`); + } + + const latestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}`; + const latestFile = `opensearch-${desiredVersion}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`; + const completeLatestUrl = `${latestUrl}/${latestFile}`; + + let { abc, resp } = await verifySnapshotUrl(completeLatestUrl, log); + + if (resp.ok) { + return { + url: completeLatestUrl, + checksumUrl: completeLatestUrl + '.sha512', + checksumType: 'sha512', + filename: latestFile, + }; + } + + log.info( + 'Daily general-availability snapshot URL not found for current version, falling back to release-candidate snapshot URL.' + ); + + let completeUrl = null; + let snapshotFile = null; + + // This checks and uses an RC if a RC exists at a higher increment than RC1 or it tries to use RC1 + // This is in replacement of having a manifest URL, so the exact RC number is unknown but expect it not to be a large number + let rcCheck = MAX_RC_CHECK; + do { + const secondaryLatestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}-rc${rcCheck}`; + const secondaryLatestFile = `opensearch-${desiredVersion}-rc${rcCheck}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`; + const completeSecondaryLatestUrl = `${secondaryLatestUrl}/${secondaryLatestFile}`; + ({ abc, resp } = await verifySnapshotUrl(completeSecondaryLatestUrl, log)); + + if (resp.ok) { + completeUrl = completeSecondaryLatestUrl; + snapshotFile = secondaryLatestFile; + break; + } + } while (rcCheck-- >= 1); + + if (resp.status === 404 || !completeUrl || !snapshotFile) { + abc.abort(); + throw createCliError(`Snapshots for ${desiredVersion} are not available`); + } + + if (!resp.ok) { + abc.abort(); + throw new Error(`Unable to read snapshot url: ${resp.statusText}`); + } + + return { + url: completeUrl, + checksumUrl: completeUrl + '.sha512', + checksumType: 'sha512', + filename: snapshotFile, + }; +} + exports.Artifact = class Artifact { /** * Fetch an Artifact from the Artifact API for a license level and version @@ -168,7 +255,7 @@ exports.Artifact = class Artifact { return new Artifact(customSnapshotArtifactSpec, log); } - const artifactSpec = await getArtifactSpecForSnapshot(urlVersion, license, log); + const artifactSpec = await getArtifactSpecForSnapshotFromUrl(urlVersion, log); return new Artifact(artifactSpec, log); } diff --git a/packages/osd-opensearch/src/artifact.test.js b/packages/osd-opensearch/src/artifact.test.js index f1e99918febd..e5f59ba9aef0 100644 --- a/packages/osd-opensearch/src/artifact.test.js +++ b/packages/osd-opensearch/src/artifact.test.js @@ -40,26 +40,33 @@ import { Artifact } from './artifact'; const log = new ToolingLog(); let MOCKS; +const DAILY_SNAPSHOT_BASE_URL = 'https://artifacts.opensearch.org/snapshots/core/opensearch'; +const PERMANENT_SNAPSHOT_BASE_URL = ''; + +const ORIGINAL_PLATFROM = process.platform; +const ORIGINAL_ARCHITECTURE = process.arch; const PLATFORM = process.platform === 'win32' ? 'windows' : process.platform; const ARCHITECTURE = process.arch === 'arm64' ? 'arm64' : 'x64'; const MOCK_VERSION = 'test-version'; -const MOCK_URL = 'http://127.0.0.1:12345'; -const MOCK_FILENAME = 'test-filename'; +const MOCK_RC_VERSION = `test-version-rc4`; +const MOCK_FILENAME = 'opensearch-test-version-SNAPSHOT-linux-x64-latest.tar.gz'; +const MOCK_RC_FILENAME = `opensearch-test-version-rc4-SNAPSHOT-linux-x64-latest.tar.gz`; +const MOCK_URL = `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/${MOCK_FILENAME}`; +const MOCK_RC_URL = `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_RC_VERSION}/${MOCK_RC_FILENAME}`; -const DAILY_SNAPSHOT_BASE_URL = ''; -const PERMANENT_SNAPSHOT_BASE_URL = ''; +const itif = process.platfom !== 'linux' || process.arch !== 'x64' ? it : it.skip; const createArchive = (params = {}) => { - const license = params.license || 'default'; const architecture = params.architecture || ARCHITECTURE; + const useRCVersion = params.useRCVersion || false; return { - license: 'default', + license: 'oss', architecture, - version: MOCK_VERSION, - url: MOCK_URL + `/${license}`, + version: !useRCVersion ? MOCK_VERSION : MOCK_RC_VERSION, + url: !useRCVersion ? MOCK_URL : MOCK_RC_URL, platform: PLATFORM, - filename: MOCK_FILENAME + `-${architecture}.${license}`, + filename: !useRCVersion ? MOCK_FILENAME : MOCK_RC_FILENAME, ...params, }; }; @@ -92,99 +99,87 @@ beforeEach(() => { jest.resetAllMocks(); MOCKS = { - valid: { - archives: [createArchive({ license: 'oss' }), createArchive({ license: 'default' })], + GA: { + archives: [createArchive({ useRCVersion: false })], + }, + RC: { + archives: [createArchive({ useRCVersion: true })], }, multipleArch: { archives: [ - createArchive({ architecture: 'fake_arch', license: 'oss' }), - createArchive({ architecture: ARCHITECTURE, license: 'oss' }), + createArchive({ architecture: 'fake_arch', useRCVersion: false }), + createArchive({ architecture: ARCHITECTURE, useRCVersion: false }), ], }, }; }); -const artifactTest = (requestedLicense, expectedLicense, fetchTimesCalled = 1) => { +const artifactTest = (fetchTimesCalled = 1) => { return async () => { - const artifact = await Artifact.getSnapshot(requestedLicense, MOCK_VERSION, log); + const artifact = await Artifact.getSnapshot('oss', MOCK_VERSION, log); + const expectedUrl = fetchTimesCalled === 1 ? MOCK_URL : MOCK_RC_URL; + const expectedFilename = fetchTimesCalled === 1 ? MOCK_FILENAME : MOCK_RC_FILENAME; expect(fetch).toHaveBeenCalledTimes(fetchTimesCalled); - expect(fetch.mock.calls[0][0]).toEqual( - `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest-verified.json` - ); - if (fetchTimesCalled === 2) { - expect(fetch.mock.calls[1][0]).toEqual( - `${PERMANENT_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest.json` - ); + expect(fetch.mock.calls[0][0]).toEqual(MOCK_URL); + if (fetchTimesCalled !== 1) { + expect(fetch.mock.calls[fetchTimesCalled - 1][0]).toEqual(MOCK_RC_URL); } - expect(artifact.getUrl()).toEqual(MOCK_URL + `/${expectedLicense}`); - expect(artifact.getChecksumUrl()).toEqual(MOCK_URL + `/${expectedLicense}.sha512`); + expect(artifact.getUrl()).toEqual(expectedUrl); + expect(artifact.getChecksumUrl()).toEqual(expectedUrl + '.sha512'); expect(artifact.getChecksumType()).toEqual('sha512'); - expect(artifact.getFilename()).toEqual(MOCK_FILENAME + `-${ARCHITECTURE}.${expectedLicense}`); + expect(artifact.getFilename()).toEqual(expectedFilename); }; }; describe('Artifact', () => { describe('getSnapshot()', () => { - describe('with default snapshot', () => { - beforeEach(() => { - mockFetch(MOCKS.valid); - }); - - it('should return artifact metadata for a daily oss artifact', artifactTest('oss', 'oss')); + itif('should return artifact metadata for a daily GA artifact', () => { + mockFetch(MOCKS.GA); + artifactTest(); + }); - it( - 'should return artifact metadata for a daily default artifact', - artifactTest('default', 'default') - ); + itif('should return artifact metadata for a RC artifact', () => { + fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 }))); + fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 }))); + mockFetch(MOCKS.RC); + artifactTest(3); + }); - it( - 'should default to default license with anything other than "oss"', - artifactTest('INVALID_LICENSE', 'default') + itif('should throw when an artifact cannot be found for the specified parameters', async () => { + fetch.mockReturnValue(Promise.resolve(new Response('', { status: 404 }))); + await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow( + 'Snapshots for INVALID_VERSION are not available' ); - - it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => { - await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow( - "couldn't find an artifact" - ); - }); }); - describe('with missing default snapshot', () => { - beforeEach(() => { - fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 }))); - mockFetch(MOCKS.valid); + describe('with snapshots for multiple architectures', () => { + afterAll(() => { + Object.defineProperties(process, { + platform: { + value: ORIGINAL_PLATFROM, + }, + arch: { + value: ORIGINAL_ARCHITECTURE, + }, + }); }); - it( - 'should return artifact metadata for a permanent oss artifact', - artifactTest('oss', 'oss', 2) - ); - - it( - 'should return artifact metadata for a permanent default artifact', - artifactTest('default', 'default', 2) - ); - - it( - 'should default to default license with anything other than "oss"', - artifactTest('INVALID_LICENSE', 'default', 2) - ); - - it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => { - await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow( - "couldn't find an artifact" + it('should throw when on a non-Linux platform', async () => { + Object.defineProperty(process, 'platform', { + value: 'win32', + }); + await expect(Artifact.getSnapshot('default', 'INVALID_PLATFORM', log)).rejects.toThrow( + 'Snapshots are only available for Linux x64' ); }); - }); - - describe('with snapshots for multiple architectures', () => { - beforeEach(() => { - mockFetch(MOCKS.multipleArch); - }); - it('should return artifact metadata for the correct architecture', async () => { - const artifact = await Artifact.getSnapshot('oss', MOCK_VERSION, log); - expect(artifact.getFilename()).toEqual(MOCK_FILENAME + `-${ARCHITECTURE}.oss`); + it('should throw when on a non-x64 arch', async () => { + Object.defineProperty(process, 'arch', { + value: 'arm64', + }); + await expect(Artifact.getSnapshot('default', 'INVALID_ARCH', log)).rejects.toThrow( + 'Snapshots are only available for Linux x64' + ); }); }); @@ -193,7 +188,7 @@ describe('Artifact', () => { beforeEach(() => { process.env.OPENSEARCH_SNAPSHOT_MANIFEST = CUSTOM_URL; - mockFetch(MOCKS.valid); + mockFetch(MOCKS.GA); }); it('should use the custom URL when looking for a snapshot', async () => { @@ -205,23 +200,5 @@ describe('Artifact', () => { delete process.env.OPENSEARCH_SNAPSHOT_MANIFEST; }); }); - - describe('with latest unverified snapshot', () => { - beforeEach(() => { - process.env.OSD_OPENSEARCH_SNAPSHOT_USE_UNVERIFIED = 1; - mockFetch(MOCKS.valid); - }); - - it('should use the daily unverified URL when looking for a snapshot', async () => { - await Artifact.getSnapshot('oss', MOCK_VERSION, log); - expect(fetch.mock.calls[0][0]).toEqual( - `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest.json` - ); - }); - - afterEach(() => { - delete process.env.OSD_OPENSEARCH_SNAPSHOT_USE_UNVERIFIED; - }); - }); }); }); diff --git a/src/core/server/ui_settings/create_or_upgrade_saved_config/integration_tests/create_or_upgrade.test.ts b/src/core/server/ui_settings/create_or_upgrade_saved_config/integration_tests/create_or_upgrade.test.ts index 5c65d30a2b0d..fa701bed75e8 100644 --- a/src/core/server/ui_settings/create_or_upgrade_saved_config/integration_tests/create_or_upgrade.test.ts +++ b/src/core/server/ui_settings/create_or_upgrade_saved_config/integration_tests/create_or_upgrade.test.ts @@ -95,8 +95,7 @@ describe('createOrUpgradeSavedConfig()', () => { await osd.stop(); }, 30000); - // TODO: [RENAMEME] Test can be enabled once there is a valid snapshot URL - xit('upgrades the previous version on each increment', async function () { + it('upgrades the previous version on each increment', async function () { // ------------------------------------ // upgrade to 5.4.0 await createOrUpgradeSavedConfig({ diff --git a/src/core/server/ui_settings/integration_tests/index.test.ts b/src/core/server/ui_settings/integration_tests/index.test.ts index cdd542fc219d..67c4c7950b61 100644 --- a/src/core/server/ui_settings/integration_tests/index.test.ts +++ b/src/core/server/ui_settings/integration_tests/index.test.ts @@ -36,8 +36,7 @@ import { docExistsSuite } from './doc_exists'; import { docMissingSuite } from './doc_missing'; import { docMissingAndIndexReadOnlySuite } from './doc_missing_and_index_read_only'; -// TODO: [RENAMEME] Test can be enabled once there is a valid snapshot URL -xdescribe('uiSettings/routes', function () { +describe('uiSettings/routes', function () { jest.setTimeout(10000); beforeAll(startServers);