From 9a22ef29d4f0385a7cc6a6c9640b3b25fa47e29a Mon Sep 17 00:00:00 2001 From: Stratoula Kalafateli Date: Wed, 19 Aug 2020 18:04:03 +0300 Subject: [PATCH 1/2] [Functional]Table Vis increase sleep time in order filter to be applied (#75138) * Increase sleep time in order filter to be applied * revert Co-authored-by: Elastic Machine --- test/functional/page_objects/visualize_chart_page.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/page_objects/visualize_chart_page.ts b/test/functional/page_objects/visualize_chart_page.ts index ade78cbb810d5..cb114866322db 100644 --- a/test/functional/page_objects/visualize_chart_page.ts +++ b/test/functional/page_objects/visualize_chart_page.ts @@ -244,7 +244,7 @@ export function VisualizeChartPageProvider({ getService, getPageObjects }: FtrPr const firstCount = await this.getVisualizationRenderingCount(); log.debug(`-- firstCount=${firstCount}`); - await common.sleep(1000); + await common.sleep(2000); const secondCount = await this.getVisualizationRenderingCount(); log.debug(`-- secondCount=${secondCount}`); From 9698a0720d424531aa528a4fe71112d53492b837 Mon Sep 17 00:00:00 2001 From: Gidi Meir Morris Date: Wed, 19 Aug 2020 16:10:33 +0100 Subject: [PATCH 2/2] [Saved objects] Add support for version on create & bulkCreate when overwriting a document (#75172) Adds support for `version` one the SavedObjectsClient's create api. This sallows us to retain Optimistic concurrency control when using create to overwrite an existing document. --- ...ore-server.savedobjectsbulkcreateobject.md | 1 + ...er.savedobjectsbulkcreateobject.version.md | 11 +++++ ...n-core-server.savedobjectscreateoptions.md | 1 + ...erver.savedobjectscreateoptions.version.md | 13 ++++++ .../import/import_saved_objects.ts | 7 ++- .../import/resolve_import_errors.ts | 12 ++++-- .../service/lib/repository.test.js | 43 ++++++++++++++++++- .../saved_objects/service/lib/repository.ts | 12 +++++- .../service/saved_objects_client.ts | 6 +++ src/core/server/server.api.md | 3 ++ .../services/sample_data/routes/install.ts | 2 +- .../services/epm/kibana/assets/install.ts | 4 +- 12 files changed, 105 insertions(+), 10 deletions(-) create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsbulkcreateobject.version.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectscreateoptions.version.md diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsbulkcreateobject.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsbulkcreateobject.md index 5a9ca36ba56f4..5ccad134248f6 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsbulkcreateobject.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsbulkcreateobject.md @@ -20,4 +20,5 @@ export interface SavedObjectsBulkCreateObject | [migrationVersion](./kibana-plugin-core-server.savedobjectsbulkcreateobject.migrationversion.md) | SavedObjectsMigrationVersion | Information about the migrations that have been applied to this SavedObject. When Kibana starts up, KibanaMigrator detects outdated documents and migrates them based on this value. For each migration that has been applied, the plugin's name is used as a key and the latest migration version as the value. | | [references](./kibana-plugin-core-server.savedobjectsbulkcreateobject.references.md) | SavedObjectReference[] | | | [type](./kibana-plugin-core-server.savedobjectsbulkcreateobject.type.md) | string | | +| [version](./kibana-plugin-core-server.savedobjectsbulkcreateobject.version.md) | string | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsbulkcreateobject.version.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsbulkcreateobject.version.md new file mode 100644 index 0000000000000..ca2a38693d036 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsbulkcreateobject.version.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsBulkCreateObject](./kibana-plugin-core-server.savedobjectsbulkcreateobject.md) > [version](./kibana-plugin-core-server.savedobjectsbulkcreateobject.version.md) + +## SavedObjectsBulkCreateObject.version property + +Signature: + +```typescript +version?: string; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectscreateoptions.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectscreateoptions.md index 5e9433c5c9196..c5201efd0608d 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectscreateoptions.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectscreateoptions.md @@ -20,4 +20,5 @@ export interface SavedObjectsCreateOptions extends SavedObjectsBaseOptions | [overwrite](./kibana-plugin-core-server.savedobjectscreateoptions.overwrite.md) | boolean | Overwrite existing documents (defaults to false) | | [references](./kibana-plugin-core-server.savedobjectscreateoptions.references.md) | SavedObjectReference[] | | | [refresh](./kibana-plugin-core-server.savedobjectscreateoptions.refresh.md) | MutatingOperationRefreshSetting | The Elasticsearch Refresh setting for this operation | +| [version](./kibana-plugin-core-server.savedobjectscreateoptions.version.md) | string | An opaque version number which changes on each successful write operation. Can be used in conjunction with overwrite for implementing optimistic concurrency control. | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectscreateoptions.version.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectscreateoptions.version.md new file mode 100644 index 0000000000000..51da57064abb9 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectscreateoptions.version.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsCreateOptions](./kibana-plugin-core-server.savedobjectscreateoptions.md) > [version](./kibana-plugin-core-server.savedobjectscreateoptions.version.md) + +## SavedObjectsCreateOptions.version property + +An opaque version number which changes on each successful write operation. Can be used in conjunction with `overwrite` for implementing optimistic concurrency control. + +Signature: + +```typescript +version?: string; +``` diff --git a/src/core/server/saved_objects/import/import_saved_objects.ts b/src/core/server/saved_objects/import/import_saved_objects.ts index 6065e03fb1628..4956491a79aa9 100644 --- a/src/core/server/saved_objects/import/import_saved_objects.ts +++ b/src/core/server/saved_objects/import/import_saved_objects.ts @@ -25,6 +25,7 @@ import { SavedObjectsImportOptions, } from './types'; import { validateReferences } from './validate_references'; +import { SavedObject } from '../types'; /** * Import saved objects from given stream. See the {@link SavedObjectsImportOptions | options} for more @@ -67,7 +68,7 @@ export async function importSavedObjectsFromStream({ } // Create objects in bulk - const bulkCreateResult = await savedObjectsClient.bulkCreate(filteredObjects, { + const bulkCreateResult = await savedObjectsClient.bulkCreate(omitVersion(filteredObjects), { overwrite, namespace, }); @@ -82,3 +83,7 @@ export async function importSavedObjectsFromStream({ ...(errorAccumulator.length ? { errors: errorAccumulator } : {}), }; } + +export function omitVersion(objects: SavedObject[]): SavedObject[] { + return objects.map(({ version, ...object }) => object); +} diff --git a/src/core/server/saved_objects/import/resolve_import_errors.ts b/src/core/server/saved_objects/import/resolve_import_errors.ts index a5175aa080598..dce044a31a577 100644 --- a/src/core/server/saved_objects/import/resolve_import_errors.ts +++ b/src/core/server/saved_objects/import/resolve_import_errors.ts @@ -26,6 +26,7 @@ import { SavedObjectsResolveImportErrorsOptions, } from './types'; import { validateReferences } from './validate_references'; +import { omitVersion } from './import_saved_objects'; /** * Resolve and return saved object import errors. @@ -91,7 +92,7 @@ export async function resolveSavedObjectsImportErrors({ // Bulk create in two batches, overwrites and non-overwrites const { objectsToOverwrite, objectsToNotOverwrite } = splitOverwrites(filteredObjects, retries); if (objectsToOverwrite.length) { - const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToOverwrite, { + const bulkCreateResult = await savedObjectsClient.bulkCreate(omitVersion(objectsToOverwrite), { overwrite: true, namespace, }); @@ -102,9 +103,12 @@ export async function resolveSavedObjectsImportErrors({ successCount += bulkCreateResult.saved_objects.filter((obj) => !obj.error).length; } if (objectsToNotOverwrite.length) { - const bulkCreateResult = await savedObjectsClient.bulkCreate(objectsToNotOverwrite, { - namespace, - }); + const bulkCreateResult = await savedObjectsClient.bulkCreate( + omitVersion(objectsToNotOverwrite), + { + namespace, + } + ); errorAccumulator = [ ...errorAccumulator, ...extractErrors(bulkCreateResult.saved_objects, objectsToNotOverwrite), diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index 4a9fceb9bf357..941c4091a66a7 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -464,8 +464,16 @@ describe('SavedObjectsRepository', () => { { method, _index = expect.any(String), getId = () => expect.any(String) } ) => { const body = []; - for (const { type, id } of objects) { - body.push({ [method]: { _index, _id: getId(type, id) } }); + for (const { type, id, if_primary_term: ifPrimaryTerm, if_seq_no: ifSeqNo } of objects) { + body.push({ + [method]: { + _index, + _id: getId(type, id), + ...(ifPrimaryTerm && ifSeqNo + ? { if_primary_term: expect.any(Number), if_seq_no: expect.any(Number) } + : {}), + }, + }); body.push(expect.any(Object)); } expect(client.bulk).toHaveBeenCalledWith( @@ -525,6 +533,27 @@ describe('SavedObjectsRepository', () => { expectClientCallArgsAction([obj1, obj2], { method: 'index' }); }); + it(`should use the ES index method with version if ID and version are defined and overwrite=true`, async () => { + await bulkCreateSuccess( + [ + { + ...obj1, + version: mockVersion, + }, + obj2, + ], + { overwrite: true } + ); + + const obj1WithSeq = { + ...obj1, + if_seq_no: mockVersionProps._seq_no, + if_primary_term: mockVersionProps._primary_term, + }; + + expectClientCallArgsAction([obj1WithSeq, obj2], { method: 'index' }); + }); + it(`should use the ES create method if ID is defined and overwrite=false`, async () => { await bulkCreateSuccess([obj1, obj2]); expectClientCallArgsAction([obj1, obj2], { method: 'create' }); @@ -1516,6 +1545,16 @@ describe('SavedObjectsRepository', () => { expect(client.index).toHaveBeenCalled(); }); + it(`should use the ES index with version if ID and version are defined and overwrite=true`, async () => { + await createSuccess(type, attributes, { id, overwrite: true, version: mockVersion }); + expect(client.index).toHaveBeenCalled(); + + expect(client.index.mock.calls[0][0]).toMatchObject({ + if_seq_no: mockVersionProps._seq_no, + if_primary_term: mockVersionProps._primary_term, + }); + }); + it(`should use the ES create action if ID is defined and overwrite=false`, async () => { await createSuccess(type, attributes, { id }); expect(client.create).toHaveBeenCalled(); diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index d7e1ecba0370b..9f6db446ea195 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -220,6 +220,7 @@ export class SavedObjectsRepository { overwrite = false, references = [], refresh = DEFAULT_REFRESH_SETTING, + version, } = options; if (!this._allowedTypes.includes(type)) { @@ -259,6 +260,7 @@ export class SavedObjectsRepository { index: this.getIndexForType(type), refresh, body: raw._source, + ...(overwrite && version ? decodeRequestVersion(version) : {}), }; const { body } = @@ -345,7 +347,12 @@ export class SavedObjectsRepository { let savedObjectNamespace; let savedObjectNamespaces; - const { esRequestIndex, object, method } = expectedBulkGetResult.value; + let versionProperties; + const { + esRequestIndex, + object: { version, ...object }, + method, + } = expectedBulkGetResult.value; if (esRequestIndex !== undefined) { const indexFound = bulkGetResponse?.statusCode !== 404; const actualResult = indexFound ? bulkGetResponse?.body.docs[esRequestIndex] : undefined; @@ -362,12 +369,14 @@ export class SavedObjectsRepository { }; } savedObjectNamespaces = getSavedObjectNamespaces(namespace, docFound && actualResult); + versionProperties = getExpectedVersionProperties(version, actualResult); } else { if (this._registry.isSingleNamespace(object.type)) { savedObjectNamespace = namespace; } else if (this._registry.isMultiNamespace(object.type)) { savedObjectNamespaces = getSavedObjectNamespaces(namespace); } + versionProperties = getExpectedVersionProperties(version); } const expectedResult = { @@ -392,6 +401,7 @@ export class SavedObjectsRepository { [method]: { _id: expectedResult.rawMigratedDoc._id, _index: this.getIndexForType(object.type), + ...(overwrite && versionProperties), }, }, expectedResult.rawMigratedDoc._source diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index e15a92c92772f..6a9f4f5143e84 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -37,6 +37,11 @@ export interface SavedObjectsCreateOptions extends SavedObjectsBaseOptions { id?: string; /** Overwrite existing documents (defaults to false) */ overwrite?: boolean; + /** + * An opaque version number which changes on each successful write operation. + * Can be used in conjunction with `overwrite` for implementing optimistic concurrency control. + **/ + version?: string; /** {@inheritDoc SavedObjectsMigrationVersion} */ migrationVersion?: SavedObjectsMigrationVersion; references?: SavedObjectReference[]; @@ -52,6 +57,7 @@ export interface SavedObjectsBulkCreateObject { id?: string; type: string; attributes: T; + version?: string; references?: SavedObjectReference[]; /** {@inheritDoc SavedObjectsMigrationVersion} */ migrationVersion?: SavedObjectsMigrationVersion; diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 03545284e14fb..01558c1460f1b 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -2044,6 +2044,8 @@ export interface SavedObjectsBulkCreateObject { references?: SavedObjectReference[]; // (undocumented) type: string; + // (undocumented) + version?: string; } // @public (undocumented) @@ -2178,6 +2180,7 @@ export interface SavedObjectsCreateOptions extends SavedObjectsBaseOptions { // (undocumented) references?: SavedObjectReference[]; refresh?: MutatingOperationRefreshSetting; + version?: string; } // @public (undocumented) diff --git a/src/plugins/home/server/services/sample_data/routes/install.ts b/src/plugins/home/server/services/sample_data/routes/install.ts index 2d1a53fbb09dc..b94456682afcc 100644 --- a/src/plugins/home/server/services/sample_data/routes/install.ts +++ b/src/plugins/home/server/services/sample_data/routes/install.ts @@ -154,7 +154,7 @@ export function createInstallRoute( let createResults; try { createResults = await context.core.savedObjects.client.bulkCreate( - sampleDataset.savedObjects, + sampleDataset.savedObjects.map(({ version, ...savedObject }) => savedObject), { overwrite: true } ); } catch (err) { diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts index 5741764164b83..84892d2027847 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts @@ -14,7 +14,9 @@ import * as Registry from '../../registry'; import { AssetType, KibanaAssetType, AssetReference } from '../../../../types'; import { savedObjectTypes } from '../../packages'; -type SavedObjectToBe = Required & { type: AssetType }; +type SavedObjectToBe = Required> & { + type: AssetType; +}; export type ArchiveAsset = Pick< SavedObject, 'id' | 'attributes' | 'migrationVersion' | 'references'