From a34b033cb6c8815e0eb22bddaa58ede0ec6d7f29 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 21 Feb 2023 11:14:38 +0100 Subject: [PATCH 1/9] Set allowNoIndex to default true in getFieldsAndIndicesForDataView and getFieldsAndIndicesForWildcard --- src/plugins/data_views/common/data_views/data_views.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index 65d23292486f3..a119d2ffe0caa 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -530,7 +530,7 @@ export class DataViewsService { return this.apiClient.getFieldsForWildcard({ type: dataView.type, rollupIndex: dataView?.typeMeta?.params?.rollup_index, - allowNoIndex: dataView.allowNoIndex, + allowNoIndex: dataView.allowNoIndex !== false, pattern: dataView.getIndexPattern(), metaFields, }); @@ -543,7 +543,7 @@ export class DataViewsService { metaFields, type: options.type, rollupIndex: options.rollupIndex, - allowNoIndex: options.allowNoIndex, + allowNoIndex: options.allowNoIndex !== false, indexFilter: options.indexFilter, }); }; From bf8cd8507709246c77420692e32742935cd4e6fc Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 21 Feb 2023 11:44:55 +0100 Subject: [PATCH 2/9] Add tests --- .../data_views/common/data_views/data_view.ts | 4 +- .../common/data_views/data_views.test.ts | 51 ++++++++++++++++++- .../common/data_views/data_views.ts | 4 +- 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/src/plugins/data_views/common/data_views/data_view.ts b/src/plugins/data_views/common/data_views/data_view.ts index 428a5c4321c6d..f92f286a183d8 100644 --- a/src/plugins/data_views/common/data_views/data_view.ts +++ b/src/plugins/data_views/common/data_views/data_view.ts @@ -141,7 +141,7 @@ export class DataView implements DataViewBase { /** * Prevents errors when index pattern exists before indices */ - public readonly allowNoIndex: boolean = false; + public readonly allowNoIndex: boolean = true; /** * Name of the data view. Human readable name used to differentiate data view. */ @@ -183,7 +183,7 @@ export class DataView implements DataViewBase { this.type = spec.type; this.typeMeta = spec.typeMeta; this.fieldAttrs = cloneDeep(spec.fieldAttrs) || {}; - this.allowNoIndex = spec.allowNoIndex || false; + this.allowNoIndex = spec.allowNoIndex ?? true; this.runtimeFieldMap = cloneDeep(spec.runtimeFieldMap) || {}; this.namespaces = spec.namespaces || []; this.name = spec.name || ''; diff --git a/src/plugins/data_views/common/data_views/data_views.test.ts b/src/plugins/data_views/common/data_views/data_views.test.ts index 01e2b9b1d95f3..9a775a834a957 100644 --- a/src/plugins/data_views/common/data_views/data_views.test.ts +++ b/src/plugins/data_views/common/data_views/data_views.test.ts @@ -162,6 +162,41 @@ describe('IndexPatterns', () => { expect(apiClient.getFieldsForWildcard).toBeCalledTimes(2); }); + test('getFieldsForWildcard called with allowNoIndex set to true as default ', async () => { + const id = '1'; + await indexPatterns.get(id); + expect(apiClient.getFieldsForWildcard).toBeCalledWith({ + allowNoIndex: true, + indexFilter: undefined, + metaFields: false, + pattern: 'something', + rollupIndex: undefined, + type: undefined, + }); + }); + + test('getFieldsForWildcard called with allowNoIndex set to false if configured', async () => { + const id = '2'; + setDocsourcePayload(id, { + id: 'foo', + version: 'foo', + attributes: { + title: 'something', + allowNoIndex: false, + }, + }); + + await indexPatterns.get(id); + expect(apiClient.getFieldsForWildcard).toBeCalledWith({ + allowNoIndex: false, + indexFilter: undefined, + metaFields: false, + pattern: 'something', + rollupIndex: undefined, + type: undefined, + }); + }); + test('does cache ad-hoc data views', async () => { const id = '1'; @@ -577,9 +612,8 @@ describe('IndexPatterns', () => { expect(indexPattern.fields.length).toBe(1); }); - test('refreshFields properly includes allowNoIndex', async () => { + test('refreshFields defaults allowNoIndex to true', async () => { const indexPatternSpec: DataViewSpec = { - allowNoIndex: true, title: 'test', }; @@ -589,5 +623,18 @@ describe('IndexPatterns', () => { // @ts-expect-error expect(apiClient.getFieldsForWildcard.mock.calls[0][0].allowNoIndex).toBe(true); }); + + test('refreshFields properly includes allowNoIndex=false', async () => { + const indexPatternSpec: DataViewSpec = { + allowNoIndex: false, + title: 'test', + }; + + const indexPattern = await indexPatterns.create(indexPatternSpec); + + indexPatterns.refreshFields(indexPattern); + // @ts-expect-error + expect(apiClient.getFieldsForWildcard.mock.calls[0][0].allowNoIndex).toBe(false); + }); }); }); diff --git a/src/plugins/data_views/common/data_views/data_views.ts b/src/plugins/data_views/common/data_views/data_views.ts index a119d2ffe0caa..40e9631822140 100644 --- a/src/plugins/data_views/common/data_views/data_views.ts +++ b/src/plugins/data_views/common/data_views/data_views.ts @@ -530,7 +530,7 @@ export class DataViewsService { return this.apiClient.getFieldsForWildcard({ type: dataView.type, rollupIndex: dataView?.typeMeta?.params?.rollup_index, - allowNoIndex: dataView.allowNoIndex !== false, + allowNoIndex: dataView.allowNoIndex ?? true, pattern: dataView.getIndexPattern(), metaFields, }); @@ -543,7 +543,7 @@ export class DataViewsService { metaFields, type: options.type, rollupIndex: options.rollupIndex, - allowNoIndex: options.allowNoIndex !== false, + allowNoIndex: options.allowNoIndex ?? true, indexFilter: options.indexFilter, }); }; From aae2031e66e91333a84196342bac6e4a49ff42c3 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 21 Feb 2023 19:08:37 +0100 Subject: [PATCH 3/9] Fix some tests --- .../common/data_views/__snapshots__/data_view.test.ts.snap | 2 +- .../infra/server/services/log_views/log_views_client.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap b/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap index 22e08e1b3f023..9304453b91a82 100644 --- a/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap +++ b/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap @@ -28,7 +28,7 @@ Object { exports[`IndexPattern toSpec can optionally exclude fields 1`] = ` Object { - "allowNoIndex": false, + "allowNoIndex": true, "fieldAttrs": Object {}, "fieldFormats": Object {}, "id": "test-pattern", diff --git a/x-pack/plugins/infra/server/services/log_views/log_views_client.test.ts b/x-pack/plugins/infra/server/services/log_views/log_views_client.test.ts index e517ae8aef7f0..104f671dba19c 100644 --- a/x-pack/plugins/infra/server/services/log_views/log_views_client.test.ts +++ b/x-pack/plugins/infra/server/services/log_views/log_views_client.test.ts @@ -281,7 +281,7 @@ describe('LogViewsClient class', () => { }, ], "dataViewReference": DataView { - "allowNoIndex": false, + "allowNoIndex": true, "deleteFieldFormat": [Function], "fieldAttrs": Object {}, "fieldFormatMap": Object {}, From 2d9305876108fe270ea3ab9aa3bca5a90687e2f4 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 21 Feb 2023 19:19:51 +0100 Subject: [PATCH 4/9] Fix functional --- test/functional/apps/discover/group1/_sidebar.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/apps/discover/group1/_sidebar.ts b/test/functional/apps/discover/group1/_sidebar.ts index d57b3edfe83e0..d095716687fb0 100644 --- a/test/functional/apps/discover/group1/_sidebar.ts +++ b/test/functional/apps/discover/group1/_sidebar.ts @@ -423,7 +423,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.discover.waitUntilSidebarHasLoaded(); expect(await PageObjects.discover.getSidebarAriaDescription()).to.be( - '0 available fields. 0 meta fields.' + '0 available fields. 0 meta fields. 3 meta fields.' ); await testSubjects.existOrFail( `${PageObjects.discover.getSidebarSectionSelector('available')}-fetchWarning` From 9c704a1c65fdc58e83ce8c5dce6d67357b7e9f5b Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 21 Feb 2023 20:47:38 +0100 Subject: [PATCH 5/9] Adapt filter to fix test --- test/functional/apps/discover/group1/_filter_editor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/apps/discover/group1/_filter_editor.ts b/test/functional/apps/discover/group1/_filter_editor.ts index 1f212a5dd8d63..60c08c68fdd00 100644 --- a/test/functional/apps/discover/group1/_filter_editor.ts +++ b/test/functional/apps/discover/group1/_filter_editor.ts @@ -103,7 +103,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { body: { version: '2.0.0', }, - refresh: 'wait_for', + refresh: true, }); await indexPatterns.create({ title: indexTitle }, { override: true }); From 52b0a2de0c2d846c08773c0e04d1060e1831e5ab Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 21 Feb 2023 20:48:42 +0100 Subject: [PATCH 6/9] Fix snapshot --- .../common/data_views/__snapshots__/data_view.test.ts.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap b/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap index 9304453b91a82..35b8f3f42230c 100644 --- a/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap +++ b/src/plugins/data_views/common/data_views/__snapshots__/data_view.test.ts.snap @@ -51,7 +51,7 @@ Object { exports[`IndexPattern toSpec should match snapshot 1`] = ` Object { - "allowNoIndex": false, + "allowNoIndex": true, "fieldAttrs": Object {}, "fieldFormats": Object {}, "fields": Object { From ff2d7caa6e9a60879177c86077796f50eca6cdb1 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 21 Feb 2023 20:50:50 +0100 Subject: [PATCH 7/9] Fix snapshot --- test/functional/apps/discover/group1/_sidebar.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/apps/discover/group1/_sidebar.ts b/test/functional/apps/discover/group1/_sidebar.ts index d095716687fb0..17fa3eabd5fe1 100644 --- a/test/functional/apps/discover/group1/_sidebar.ts +++ b/test/functional/apps/discover/group1/_sidebar.ts @@ -423,7 +423,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.discover.waitUntilSidebarHasLoaded(); expect(await PageObjects.discover.getSidebarAriaDescription()).to.be( - '0 available fields. 0 meta fields. 3 meta fields.' + '0 available fields. 0 empty fields. 3 meta fields.' ); await testSubjects.existOrFail( `${PageObjects.discover.getSidebarSectionSelector('available')}-fetchWarning` From b590ba64cd898f603c05e47490f5f3227e05f18b Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 22 Feb 2023 14:22:42 +0100 Subject: [PATCH 8/9] Prevent adding meta fields when no fields are returned --- .../server/fetcher/lib/field_capabilities/field_capabilities.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts index 4ec808c756873..7f7eaec576428 100644 --- a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts @@ -55,7 +55,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) { const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName) // not all meta fields are provided, so remove and manually add .filter((name) => !fieldsFromFieldCapsByName[name].metadata_field) - .concat(metaFields) + .concat(fieldsFromFieldCapsByName.length ? metaFields : []) .reduce<{ names: string[]; map: Map }>( (agg, value) => { // This is intentionally using a Map to be highly optimized with very large indexes AND be safe for user provided data From 69527504ff1348bfc6a60daa0d7bfc953185046f Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 22 Feb 2023 16:48:44 +0100 Subject: [PATCH 9/9] Fix tests --- .../fetcher/lib/field_capabilities/field_capabilities.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts index 7f7eaec576428..73e550ebd68ce 100644 --- a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts @@ -50,12 +50,13 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) { indexFilter, fields, }); - const fieldsFromFieldCapsByName = keyBy(readFieldCapsResponse(esFieldCaps.body), 'name'); + const fieldCapsArr = readFieldCapsResponse(esFieldCaps.body); + const fieldsFromFieldCapsByName = keyBy(fieldCapsArr, 'name'); const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName) // not all meta fields are provided, so remove and manually add .filter((name) => !fieldsFromFieldCapsByName[name].metadata_field) - .concat(fieldsFromFieldCapsByName.length ? metaFields : []) + .concat(fieldCapsArr.length ? metaFields : []) .reduce<{ names: string[]; map: Map }>( (agg, value) => { // This is intentionally using a Map to be highly optimized with very large indexes AND be safe for user provided data