Skip to content

Commit

Permalink
[Tests] updated artifact script
Browse files Browse the repository at this point in the history 
Updating artifact to enable running integration tests and functional
tests to pull an artifact from the current hosted distributions.

At the time of this commit, there is not manifest hosted but there are
static links which can have an unknown amount of RC versions if the
GA snapshot does not exist for that version. But the assumption
is that it will not be too high.

Deprecating the previous implementation but we can remove that in a
future iteration. Wanted to leave that available incase others use
that for custom manifests.

Enable tests that depended on snapshots as well.

Issues resolved:
opensearch-project#242
opensearch-project#19

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
  • Loading branch information
@kavilla
kavilla committed Jun 9, 2021
1 parent b102ed9 commit 447a94c
Show file tree
Hide file tree
Showing 5 changed files with 164 additions and 102 deletions.
2 changes: 1 addition & 1 deletion packages/osd-opensearch-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ describe('opensearchArchiver: createGenerateDocRecordsStream()', () => {
await delay(200);
return {
_scroll_id: 'index1ScrollId',
hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_foo' }] },
hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_1' }] },
};
},
async (name, params) => {
Expand Down
93 changes: 90 additions & 3 deletions packages/osd-opensearch/src/artifact.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

const fetch = require('node-fetch');
const AbortController = require('abort-controller');
const fs = require('fs');
Expand All @@ -40,8 +39,11 @@ const { createHash } = require('crypto');
const path = require('path');

const asyncPipeline = promisify(pipeline);
const DAILY_SNAPSHOTS_BASE_URL = '';
const DAILY_SNAPSHOTS_BASE_URL = 'https://artifacts.opensearch.org/snapshots/core/opensearch';
// TODO: [RENAMEME] currently do not have an existing replacement
const PERMANENT_SNAPSHOTS_BASE_URL = '';
// Since we do not have a manifest URL, limiting how many RC checks to 5 should be more than enough
const MAX_RC_CHECK = 5;

const { cache } = require('./utils');
const { resolveCustomSnapshotUrl } = require('./custom_snapshots');
Expand Down Expand Up @@ -96,6 +98,21 @@ async function fetchSnapshotManifest(url, log) {
return { abc, resp, json };
}

async function verifySnapshotUrl(url, log) {
log.info('Verifying snapshot URL at %s', chalk.bold(url));

const abc = new AbortController();
const resp = await retry(
log,
async () => await fetch(url, { signal: abc.signal }, { method: 'HEAD' })
);

return { abc, resp };
}

/**
* @deprecated This method should not be used, uses logic for resources we do not have access to.
*/
async function getArtifactSpecForSnapshot(urlVersion, license, log) {
const desiredVersion = urlVersion.replace('-SNAPSHOT', '');
const desiredLicense = license === 'oss' ? 'oss' : 'default';
Expand Down Expand Up @@ -153,6 +170,76 @@ async function getArtifactSpecForSnapshot(urlVersion, license, log) {
};
}

async function getArtifactSpecForSnapshotFromUrl(urlVersion, log) {
const desiredVersion = urlVersion.replace('-SNAPSHOT', '');

if (process.env.OPENSEARCH_SNAPSHOT_MANIFEST) {
return await getArtifactSpecForSnapshot(urlVersion, 'oss', log);
}

// [RENAMEME] Need replacement for other platforms.
const platform = process.platform === 'win32' ? 'windows' : process.platform;
const arch = process.arch === 'arm64' ? 'arm64' : 'x64';
if (platform !== 'linux' || arch !== 'x64') {
throw createCliError(`Snapshots are only available for Linux x64`);
}

const latestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}`;
const latestFile = `opensearch-${desiredVersion}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`;
const completeLatestUrl = `${latestUrl}/${latestFile}`;

let { abc, resp } = await verifySnapshotUrl(completeLatestUrl, log);

if (resp.ok) {
return {
url: completeLatestUrl,
checksumUrl: completeLatestUrl + '.sha512',
checksumType: 'sha512',
filename: latestFile,
};
}

log.info(
'Daily general-availability snapshot URL not found for current version, falling back to release-candidate snapshot URL.'
);

let completeUrl = null;
let snapshotFile = null;

// This checks and uses an RC if a RC exists at a higher increment than RC1 or it tries to use RC1
// This is in replacement of having a manifest URL, so the exact RC number is unknown but expect it not to be a large number
let rcCheck = MAX_RC_CHECK;
do {
const secondaryLatestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}-rc${rcCheck}`;
const secondaryLatestFile = `opensearch-${desiredVersion}-rc${rcCheck}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`;
const completeSecondaryLatestUrl = `${secondaryLatestUrl}/${secondaryLatestFile}`;
({ abc, resp } = await verifySnapshotUrl(completeSecondaryLatestUrl, log));

if (resp.ok) {
completeUrl = completeSecondaryLatestUrl;
snapshotFile = secondaryLatestFile;
break;
}
} while (rcCheck-- >= 1);

if (resp.status === 404 || !completeUrl || !snapshotFile) {
abc.abort();
throw createCliError(`Snapshots for ${desiredVersion} are not available`);
}

if (!resp.ok) {
abc.abort();
throw new Error(`Unable to read snapshot url: ${resp.statusText}`);
}

return {
url: completeUrl,
checksumUrl: completeUrl + '.sha512',
checksumType: 'sha512',
filename: snapshotFile,
};
}

exports.Artifact = class Artifact {
/**
* Fetch an Artifact from the Artifact API for a license level and version
Expand All @@ -168,7 +255,7 @@ exports.Artifact = class Artifact {
return new Artifact(customSnapshotArtifactSpec, log);
}

const artifactSpec = await getArtifactSpecForSnapshot(urlVersion, license, log);
const artifactSpec = await getArtifactSpecForSnapshotFromUrl(urlVersion, log);
return new Artifact(artifactSpec, log);
}

Expand Down
165 changes: 71 additions & 94 deletions packages/osd-opensearch/src/artifact.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,26 +40,33 @@ import { Artifact } from './artifact';
const log = new ToolingLog();
let MOCKS;

const DAILY_SNAPSHOT_BASE_URL = 'https://artifacts.opensearch.org/snapshots/core/opensearch';
const PERMANENT_SNAPSHOT_BASE_URL = '';

const ORIGINAL_PLATFROM = process.platform;
const ORIGINAL_ARCHITECTURE = process.arch;
const PLATFORM = process.platform === 'win32' ? 'windows' : process.platform;
const ARCHITECTURE = process.arch === 'arm64' ? 'arm64' : 'x64';
const MOCK_VERSION = 'test-version';
const MOCK_URL = 'http://127.0.0.1:12345';
const MOCK_FILENAME = 'test-filename';
const MOCK_RC_VERSION = `test-version-rc4`;
const MOCK_FILENAME = 'opensearch-test-version-SNAPSHOT-linux-x64-latest.tar.gz';
const MOCK_RC_FILENAME = `opensearch-test-version-rc4-SNAPSHOT-linux-x64-latest.tar.gz`;
const MOCK_URL = `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/${MOCK_FILENAME}`;
const MOCK_RC_URL = `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_RC_VERSION}/${MOCK_RC_FILENAME}`;

const DAILY_SNAPSHOT_BASE_URL = '';
const PERMANENT_SNAPSHOT_BASE_URL = '';
const itif = process.platfom !== 'linux' || process.arch !== 'x64' ? it : it.skip;

const createArchive = (params = {}) => {
const license = params.license || 'default';
const architecture = params.architecture || ARCHITECTURE;
const useRCVersion = params.useRCVersion || false;

return {
license: 'default',
license: 'oss',
architecture,
version: MOCK_VERSION,
url: MOCK_URL + `/${license}`,
version: !useRCVersion ? MOCK_VERSION : MOCK_RC_VERSION,
url: !useRCVersion ? MOCK_URL : MOCK_RC_URL,
platform: PLATFORM,
filename: MOCK_FILENAME + `-${architecture}.${license}`,
filename: !useRCVersion ? MOCK_FILENAME : MOCK_RC_FILENAME,
...params,
};
};
Expand Down Expand Up @@ -92,99 +99,87 @@ beforeEach(() => {
jest.resetAllMocks();

MOCKS = {
valid: {
archives: [createArchive({ license: 'oss' }), createArchive({ license: 'default' })],
GA: {
archives: [createArchive({ useRCVersion: false })],
},
RC: {
archives: [createArchive({ useRCVersion: true })],
},
multipleArch: {
archives: [
createArchive({ architecture: 'fake_arch', license: 'oss' }),
createArchive({ architecture: ARCHITECTURE, license: 'oss' }),
createArchive({ architecture: 'fake_arch', useRCVersion: false }),
createArchive({ architecture: ARCHITECTURE, useRCVersion: false }),
],
},
};
});

const artifactTest = (requestedLicense, expectedLicense, fetchTimesCalled = 1) => {
const artifactTest = (fetchTimesCalled = 1) => {
return async () => {
const artifact = await Artifact.getSnapshot(requestedLicense, MOCK_VERSION, log);
const artifact = await Artifact.getSnapshot('oss', MOCK_VERSION, log);
const expectedUrl = fetchTimesCalled === 1 ? MOCK_URL : MOCK_RC_URL;
const expectedFilename = fetchTimesCalled === 1 ? MOCK_FILENAME : MOCK_RC_FILENAME;
expect(fetch).toHaveBeenCalledTimes(fetchTimesCalled);
expect(fetch.mock.calls[0][0]).toEqual(
`${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest-verified.json`
);
if (fetchTimesCalled === 2) {
expect(fetch.mock.calls[1][0]).toEqual(
`${PERMANENT_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest.json`
);
expect(fetch.mock.calls[0][0]).toEqual(MOCK_URL);
if (fetchTimesCalled !== 1) {
expect(fetch.mock.calls[fetchTimesCalled - 1][0]).toEqual(MOCK_RC_URL);
}
expect(artifact.getUrl()).toEqual(MOCK_URL + `/${expectedLicense}`);
expect(artifact.getChecksumUrl()).toEqual(MOCK_URL + `/${expectedLicense}.sha512`);
expect(artifact.getUrl()).toEqual(expectedUrl);
expect(artifact.getChecksumUrl()).toEqual(expectedUrl + '.sha512');
expect(artifact.getChecksumType()).toEqual('sha512');
expect(artifact.getFilename()).toEqual(MOCK_FILENAME + `-${ARCHITECTURE}.${expectedLicense}`);
expect(artifact.getFilename()).toEqual(expectedFilename);
};
};

describe('Artifact', () => {
describe('getSnapshot()', () => {
describe('with default snapshot', () => {
beforeEach(() => {
mockFetch(MOCKS.valid);
});

it('should return artifact metadata for a daily oss artifact', artifactTest('oss', 'oss'));
itif('should return artifact metadata for a daily GA artifact', () => {
mockFetch(MOCKS.GA);
artifactTest();
});

it(
'should return artifact metadata for a daily default artifact',
artifactTest('default', 'default')
);
itif('should return artifact metadata for a RC artifact', () => {
fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 })));
fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 })));
mockFetch(MOCKS.RC);
artifactTest(3);
});

it(
'should default to default license with anything other than "oss"',
artifactTest('INVALID_LICENSE', 'default')
itif('should throw when an artifact cannot be found for the specified parameters', async () => {
fetch.mockReturnValue(Promise.resolve(new Response('', { status: 404 })));
await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow(
'Snapshots for INVALID_VERSION are not available'
);

it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => {
await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow(
"couldn't find an artifact"
);
});
});

describe('with missing default snapshot', () => {
beforeEach(() => {
fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 })));
mockFetch(MOCKS.valid);
describe('with snapshots for multiple architectures', () => {
afterAll(() => {
Object.defineProperties(process, {
platform: {
value: ORIGINAL_PLATFROM,
},
arch: {
value: ORIGINAL_ARCHITECTURE,
},
});
});

it(
'should return artifact metadata for a permanent oss artifact',
artifactTest('oss', 'oss', 2)
);

it(
'should return artifact metadata for a permanent default artifact',
artifactTest('default', 'default', 2)
);

it(
'should default to default license with anything other than "oss"',
artifactTest('INVALID_LICENSE', 'default', 2)
);

it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => {
await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow(
"couldn't find an artifact"
it('should throw when on a non-Linux platform', async () => {
Object.defineProperty(process, 'platform', {
value: 'win32',
});
await expect(Artifact.getSnapshot('default', 'INVALID_PLATFORM', log)).rejects.toThrow(
'Snapshots are only available for Linux x64'
);
});
});

describe('with snapshots for multiple architectures', () => {
beforeEach(() => {
mockFetch(MOCKS.multipleArch);
});

it('should return artifact metadata for the correct architecture', async () => {
const artifact = await Artifact.getSnapshot('oss', MOCK_VERSION, log);
expect(artifact.getFilename()).toEqual(MOCK_FILENAME + `-${ARCHITECTURE}.oss`);
it('should throw when on a non-x64 arch', async () => {
Object.defineProperty(process, 'arch', {
value: 'arm64',
});
await expect(Artifact.getSnapshot('default', 'INVALID_ARCH', log)).rejects.toThrow(
'Snapshots are only available for Linux x64'
);
});
});

Expand All @@ -193,7 +188,7 @@ describe('Artifact', () => {

beforeEach(() => {
process.env.OPENSEARCH_SNAPSHOT_MANIFEST = CUSTOM_URL;
mockFetch(MOCKS.valid);
mockFetch(MOCKS.GA);
});

it('should use the custom URL when looking for a snapshot', async () => {
Expand All @@ -205,23 +200,5 @@ describe('Artifact', () => {
delete process.env.OPENSEARCH_SNAPSHOT_MANIFEST;
});
});

describe('with latest unverified snapshot', () => {
beforeEach(() => {
process.env.OSD_OPENSEARCH_SNAPSHOT_USE_UNVERIFIED = 1;
mockFetch(MOCKS.valid);
});

it('should use the daily unverified URL when looking for a snapshot', async () => {
await Artifact.getSnapshot('oss', MOCK_VERSION, log);
expect(fetch.mock.calls[0][0]).toEqual(
`${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest.json`
);
});

afterEach(() => {
delete process.env.OSD_OPENSEARCH_SNAPSHOT_USE_UNVERIFIED;
});
});
});
});
3 changes: 1 addition & 2 deletions ...er/ui_settings/create_or_upgrade_saved_config/integration_tests/create_or_upgrade.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,7 @@ describe('createOrUpgradeSavedConfig()', () => {
await osd.stop();
}, 30000);

// TODO: [RENAMEME] Test can be enabled once there is a valid snapshot URL
xit('upgrades the previous version on each increment', async function () {
it('upgrades the previous version on each increment', async function () {
// ------------------------------------
// upgrade to 5.4.0
await createOrUpgradeSavedConfig({
Expand Down
3 changes: 1 addition & 2 deletions src/core/server/ui_settings/integration_tests/index.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@ import { docExistsSuite } from './doc_exists';
import { docMissingSuite } from './doc_missing';
import { docMissingAndIndexReadOnlySuite } from './doc_missing_and_index_read_only';

// TODO: [RENAMEME] Test can be enabled once there is a valid snapshot URL
xdescribe('uiSettings/routes', function () {
describe('uiSettings/routes', function () {
jest.setTimeout(10000);

beforeAll(startServers);
Expand Down

0 comments on commit 447a94c

Please sign in to comment.