diff --git a/packages/osd-opensearch-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts b/packages/osd-opensearch-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts index 86e2bd43eab2..1292ace5e56e 100644 --- a/packages/osd-opensearch-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts +++ b/packages/osd-opensearch-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts @@ -104,7 +104,7 @@ describe('opensearchArchiver: createGenerateDocRecordsStream()', () => { await delay(200); return { _scroll_id: 'index1ScrollId', - hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_foo' }] }, + hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_1' }] }, }; }, async (name, params) => { diff --git a/packages/osd-opensearch/src/artifact.js b/packages/osd-opensearch/src/artifact.js index 491eaec8f1f1..8ece5c02b8dd 100644 --- a/packages/osd-opensearch/src/artifact.js +++ b/packages/osd-opensearch/src/artifact.js @@ -29,7 +29,6 @@ * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ - const fetch = require('node-fetch'); const AbortController = require('abort-controller'); const fs = require('fs'); @@ -40,8 +39,11 @@ const { createHash } = require('crypto'); const path = require('path'); const asyncPipeline = promisify(pipeline); -const DAILY_SNAPSHOTS_BASE_URL = ''; +const DAILY_SNAPSHOTS_BASE_URL = 'https://artifacts.opensearch.org/snapshots/core/opensearch'; +// TODO: [RENAMEME] currently do not have an existing replacement const PERMANENT_SNAPSHOTS_BASE_URL = ''; +// Since we do not have a manifest URL, limiting how many RC checks to 5 should be more than enough +const MAX_RC_CHECK = 5; const { cache } = require('./utils'); const { resolveCustomSnapshotUrl } = require('./custom_snapshots'); @@ -96,6 +98,21 @@ async function fetchSnapshotManifest(url, log) { return { abc, resp, json }; } +async function verifySnapshotUrl(url, log) { + log.info('Verifying snapshot URL at %s', chalk.bold(url)); + + const abc = new AbortController(); + const resp = await retry( + log, + async () => await fetch(url, { signal: abc.signal }, { method: 'HEAD' }) + ); + + return { abc, resp }; +} + +/** + * @deprecated This method should not be used, uses logic for resources we do not have access to. + */ async function getArtifactSpecForSnapshot(urlVersion, license, log) { const desiredVersion = urlVersion.replace('-SNAPSHOT', ''); const desiredLicense = license === 'oss' ? 'oss' : 'default'; @@ -153,6 +170,76 @@ async function getArtifactSpecForSnapshot(urlVersion, license, log) { }; } +async function getArtifactSpecForSnapshotFromUrl(urlVersion, log) { + const desiredVersion = urlVersion.replace('-SNAPSHOT', ''); + + if (process.env.OPENSEARCH_SNAPSHOT_MANIFEST) { + return await getArtifactSpecForSnapshot(urlVersion, 'oss', log); + } + + // [RENAMEME] Need replacement for other platforms. + const platform = process.platform === 'win32' ? 'windows' : process.platform; + const arch = process.arch === 'arm64' ? 'arm64' : 'x64'; + if (platform !== 'linux' || arch !== 'x64') { + throw createCliError(`Snapshots are only available for Linux x64`); + } + + const latestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}`; + const latestFile = `opensearch-${desiredVersion}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`; + const completeLatestUrl = `${latestUrl}/${latestFile}`; + + let { abc, resp } = await verifySnapshotUrl(completeLatestUrl, log); + + if (resp.ok) { + return { + url: completeLatestUrl, + checksumUrl: completeLatestUrl + '.sha512', + checksumType: 'sha512', + filename: latestFile, + }; + } + + log.info( + 'Daily general-availability snapshot URL not found for current version, falling back to release-candidate snapshot URL.' + ); + + let completeUrl = null; + let snapshotFile = null; + + // This checks and uses an RC if a RC exists at a higher increment than RC1 or it tries to use RC1 + // This is in replacement of having a manifest URL, so the exact RC number is unknown but expect it not to be a large number + let rcCheck = MAX_RC_CHECK; + do { + const secondaryLatestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}-rc${rcCheck}`; + const secondaryLatestFile = `opensearch-${desiredVersion}-rc${rcCheck}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`; + const completeSecondaryLatestUrl = `${secondaryLatestUrl}/${secondaryLatestFile}`; + ({ abc, resp } = await verifySnapshotUrl(completeSecondaryLatestUrl, log)); + + if (resp.ok) { + completeUrl = completeSecondaryLatestUrl; + snapshotFile = secondaryLatestFile; + break; + } + } while (rcCheck-- >= 1); + + if (resp.status === 404 || !completeUrl || !snapshotFile) { + abc.abort(); + throw createCliError(`Snapshots for ${desiredVersion} are not available`); + } + + if (!resp.ok) { + abc.abort(); + throw new Error(`Unable to read snapshot url: ${resp.statusText}`); + } + + return { + url: completeUrl, + checksumUrl: completeUrl + '.sha512', + checksumType: 'sha512', + filename: snapshotFile, + }; +} + exports.Artifact = class Artifact { /** * Fetch an Artifact from the Artifact API for a license level and version @@ -168,7 +255,7 @@ exports.Artifact = class Artifact { return new Artifact(customSnapshotArtifactSpec, log); } - const artifactSpec = await getArtifactSpecForSnapshot(urlVersion, license, log); + const artifactSpec = await getArtifactSpecForSnapshotFromUrl(urlVersion, log); return new Artifact(artifactSpec, log); } diff --git a/packages/osd-opensearch/src/artifact.test.js b/packages/osd-opensearch/src/artifact.test.js index f1e99918febd..e5f59ba9aef0 100644 --- a/packages/osd-opensearch/src/artifact.test.js +++ b/packages/osd-opensearch/src/artifact.test.js @@ -40,26 +40,33 @@ import { Artifact } from './artifact'; const log = new ToolingLog(); let MOCKS; +const DAILY_SNAPSHOT_BASE_URL = 'https://artifacts.opensearch.org/snapshots/core/opensearch'; +const PERMANENT_SNAPSHOT_BASE_URL = ''; + +const ORIGINAL_PLATFROM = process.platform; +const ORIGINAL_ARCHITECTURE = process.arch; const PLATFORM = process.platform === 'win32' ? 'windows' : process.platform; const ARCHITECTURE = process.arch === 'arm64' ? 'arm64' : 'x64'; const MOCK_VERSION = 'test-version'; -const MOCK_URL = 'http://127.0.0.1:12345'; -const MOCK_FILENAME = 'test-filename'; +const MOCK_RC_VERSION = `test-version-rc4`; +const MOCK_FILENAME = 'opensearch-test-version-SNAPSHOT-linux-x64-latest.tar.gz'; +const MOCK_RC_FILENAME = `opensearch-test-version-rc4-SNAPSHOT-linux-x64-latest.tar.gz`; +const MOCK_URL = `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/${MOCK_FILENAME}`; +const MOCK_RC_URL = `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_RC_VERSION}/${MOCK_RC_FILENAME}`; -const DAILY_SNAPSHOT_BASE_URL = ''; -const PERMANENT_SNAPSHOT_BASE_URL = ''; +const itif = process.platfom !== 'linux' || process.arch !== 'x64' ? it : it.skip; const createArchive = (params = {}) => { - const license = params.license || 'default'; const architecture = params.architecture || ARCHITECTURE; + const useRCVersion = params.useRCVersion || false; return { - license: 'default', + license: 'oss', architecture, - version: MOCK_VERSION, - url: MOCK_URL + `/${license}`, + version: !useRCVersion ? MOCK_VERSION : MOCK_RC_VERSION, + url: !useRCVersion ? MOCK_URL : MOCK_RC_URL, platform: PLATFORM, - filename: MOCK_FILENAME + `-${architecture}.${license}`, + filename: !useRCVersion ? MOCK_FILENAME : MOCK_RC_FILENAME, ...params, }; }; @@ -92,99 +99,87 @@ beforeEach(() => { jest.resetAllMocks(); MOCKS = { - valid: { - archives: [createArchive({ license: 'oss' }), createArchive({ license: 'default' })], + GA: { + archives: [createArchive({ useRCVersion: false })], + }, + RC: { + archives: [createArchive({ useRCVersion: true })], }, multipleArch: { archives: [ - createArchive({ architecture: 'fake_arch', license: 'oss' }), - createArchive({ architecture: ARCHITECTURE, license: 'oss' }), + createArchive({ architecture: 'fake_arch', useRCVersion: false }), + createArchive({ architecture: ARCHITECTURE, useRCVersion: false }), ], }, }; }); -const artifactTest = (requestedLicense, expectedLicense, fetchTimesCalled = 1) => { +const artifactTest = (fetchTimesCalled = 1) => { return async () => { - const artifact = await Artifact.getSnapshot(requestedLicense, MOCK_VERSION, log); + const artifact = await Artifact.getSnapshot('oss', MOCK_VERSION, log); + const expectedUrl = fetchTimesCalled === 1 ? MOCK_URL : MOCK_RC_URL; + const expectedFilename = fetchTimesCalled === 1 ? MOCK_FILENAME : MOCK_RC_FILENAME; expect(fetch).toHaveBeenCalledTimes(fetchTimesCalled); - expect(fetch.mock.calls[0][0]).toEqual( - `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest-verified.json` - ); - if (fetchTimesCalled === 2) { - expect(fetch.mock.calls[1][0]).toEqual( - `${PERMANENT_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest.json` - ); + expect(fetch.mock.calls[0][0]).toEqual(MOCK_URL); + if (fetchTimesCalled !== 1) { + expect(fetch.mock.calls[fetchTimesCalled - 1][0]).toEqual(MOCK_RC_URL); } - expect(artifact.getUrl()).toEqual(MOCK_URL + `/${expectedLicense}`); - expect(artifact.getChecksumUrl()).toEqual(MOCK_URL + `/${expectedLicense}.sha512`); + expect(artifact.getUrl()).toEqual(expectedUrl); + expect(artifact.getChecksumUrl()).toEqual(expectedUrl + '.sha512'); expect(artifact.getChecksumType()).toEqual('sha512'); - expect(artifact.getFilename()).toEqual(MOCK_FILENAME + `-${ARCHITECTURE}.${expectedLicense}`); + expect(artifact.getFilename()).toEqual(expectedFilename); }; }; describe('Artifact', () => { describe('getSnapshot()', () => { - describe('with default snapshot', () => { - beforeEach(() => { - mockFetch(MOCKS.valid); - }); - - it('should return artifact metadata for a daily oss artifact', artifactTest('oss', 'oss')); + itif('should return artifact metadata for a daily GA artifact', () => { + mockFetch(MOCKS.GA); + artifactTest(); + }); - it( - 'should return artifact metadata for a daily default artifact', - artifactTest('default', 'default') - ); + itif('should return artifact metadata for a RC artifact', () => { + fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 }))); + fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 }))); + mockFetch(MOCKS.RC); + artifactTest(3); + }); - it( - 'should default to default license with anything other than "oss"', - artifactTest('INVALID_LICENSE', 'default') + itif('should throw when an artifact cannot be found for the specified parameters', async () => { + fetch.mockReturnValue(Promise.resolve(new Response('', { status: 404 }))); + await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow( + 'Snapshots for INVALID_VERSION are not available' ); - - it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => { - await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow( - "couldn't find an artifact" - ); - }); }); - describe('with missing default snapshot', () => { - beforeEach(() => { - fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 }))); - mockFetch(MOCKS.valid); + describe('with snapshots for multiple architectures', () => { + afterAll(() => { + Object.defineProperties(process, { + platform: { + value: ORIGINAL_PLATFROM, + }, + arch: { + value: ORIGINAL_ARCHITECTURE, + }, + }); }); - it( - 'should return artifact metadata for a permanent oss artifact', - artifactTest('oss', 'oss', 2) - ); - - it( - 'should return artifact metadata for a permanent default artifact', - artifactTest('default', 'default', 2) - ); - - it( - 'should default to default license with anything other than "oss"', - artifactTest('INVALID_LICENSE', 'default', 2) - ); - - it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => { - await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow( - "couldn't find an artifact" + it('should throw when on a non-Linux platform', async () => { + Object.defineProperty(process, 'platform', { + value: 'win32', + }); + await expect(Artifact.getSnapshot('default', 'INVALID_PLATFORM', log)).rejects.toThrow( + 'Snapshots are only available for Linux x64' ); }); - }); - - describe('with snapshots for multiple architectures', () => { - beforeEach(() => { - mockFetch(MOCKS.multipleArch); - }); - it('should return artifact metadata for the correct architecture', async () => { - const artifact = await Artifact.getSnapshot('oss', MOCK_VERSION, log); - expect(artifact.getFilename()).toEqual(MOCK_FILENAME + `-${ARCHITECTURE}.oss`); + it('should throw when on a non-x64 arch', async () => { + Object.defineProperty(process, 'arch', { + value: 'arm64', + }); + await expect(Artifact.getSnapshot('default', 'INVALID_ARCH', log)).rejects.toThrow( + 'Snapshots are only available for Linux x64' + ); }); }); @@ -193,7 +188,7 @@ describe('Artifact', () => { beforeEach(() => { process.env.OPENSEARCH_SNAPSHOT_MANIFEST = CUSTOM_URL; - mockFetch(MOCKS.valid); + mockFetch(MOCKS.GA); }); it('should use the custom URL when looking for a snapshot', async () => { @@ -205,23 +200,5 @@ describe('Artifact', () => { delete process.env.OPENSEARCH_SNAPSHOT_MANIFEST; }); }); - - describe('with latest unverified snapshot', () => { - beforeEach(() => { - process.env.OSD_OPENSEARCH_SNAPSHOT_USE_UNVERIFIED = 1; - mockFetch(MOCKS.valid); - }); - - it('should use the daily unverified URL when looking for a snapshot', async () => { - await Artifact.getSnapshot('oss', MOCK_VERSION, log); - expect(fetch.mock.calls[0][0]).toEqual( - `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest.json` - ); - }); - - afterEach(() => { - delete process.env.OSD_OPENSEARCH_SNAPSHOT_USE_UNVERIFIED; - }); - }); }); }); diff --git a/src/core/server/ui_settings/create_or_upgrade_saved_config/integration_tests/create_or_upgrade.test.ts b/src/core/server/ui_settings/create_or_upgrade_saved_config/integration_tests/create_or_upgrade.test.ts index 5c65d30a2b0d..fa701bed75e8 100644 --- a/src/core/server/ui_settings/create_or_upgrade_saved_config/integration_tests/create_or_upgrade.test.ts +++ b/src/core/server/ui_settings/create_or_upgrade_saved_config/integration_tests/create_or_upgrade.test.ts @@ -95,8 +95,7 @@ describe('createOrUpgradeSavedConfig()', () => { await osd.stop(); }, 30000); - // TODO: [RENAMEME] Test can be enabled once there is a valid snapshot URL - xit('upgrades the previous version on each increment', async function () { + it('upgrades the previous version on each increment', async function () { // ------------------------------------ // upgrade to 5.4.0 await createOrUpgradeSavedConfig({ diff --git a/src/core/server/ui_settings/integration_tests/index.test.ts b/src/core/server/ui_settings/integration_tests/index.test.ts index cdd542fc219d..67c4c7950b61 100644 --- a/src/core/server/ui_settings/integration_tests/index.test.ts +++ b/src/core/server/ui_settings/integration_tests/index.test.ts @@ -36,8 +36,7 @@ import { docExistsSuite } from './doc_exists'; import { docMissingSuite } from './doc_missing'; import { docMissingAndIndexReadOnlySuite } from './doc_missing_and_index_read_only'; -// TODO: [RENAMEME] Test can be enabled once there is a valid snapshot URL -xdescribe('uiSettings/routes', function () { +describe('uiSettings/routes', function () { jest.setTimeout(10000); beforeAll(startServers);