From a6ef33ffbd899e54682be52cf46798eed87fe847 Mon Sep 17 00:00:00 2001 From: neodescis Date: Tue, 23 Apr 2024 15:41:42 -0600 Subject: [PATCH] Parse query string filter to determine fields (#6126) Signed-off-by: Nick Steinbaugh (cherry picked from commit 24177b248c74ecf078b592b6ac20cf1698768ec6) --- package.json | 3 ++ .../filter_matches_index.test.ts | 32 ++++++++++++- .../opensearch_query/filter_matches_index.ts | 46 ++++++++++++++++++- .../opensearch_query/index.ts | 1 + .../data/public/ui/filter_bar/filter_item.tsx | 7 +-- yarn.lock | 10 ++++ 6 files changed, 92 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index a90469fcd6bd..af4e18cce3d3 100644 --- a/package.json +++ b/package.json @@ -202,6 +202,8 @@ "json-stringify-safe": "5.0.1", "lodash": "^4.17.21", "lru-cache": "^4.1.5", + "lucene": "^2.1.1", + "mathjs": "^11.8.2", "minimatch": "^3.0.4", "moment": "^2.24.0", "moment-timezone": "^0.5.27", @@ -310,6 +312,7 @@ "@types/listr": "^0.14.0", "@types/lodash": "^4.14.170", "@types/lru-cache": "^5.1.0", + "@types/lucene": "^2.1.7", "@types/markdown-it": "^0.0.7", "@types/minimatch": "^2.0.29", "@types/mocha-next": "npm:@types/mocha@^10.0.1", diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.test.ts b/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.test.ts index ad68e14b2c54..07fd4aaf6962 100644 --- a/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.test.ts +++ b/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.test.ts @@ -28,7 +28,7 @@ * under the License. */ -import { Filter } from '../filters'; +import { Filter, QueryStringFilter } from '../filters'; import { filterMatchesIndex } from './filter_matches_index'; import { IIndexPattern } from '../../index_patterns'; @@ -80,4 +80,34 @@ describe('filterMatchesIndex', () => { expect(filterMatchesIndex(filter, indexPattern)).toBe(true); }); + + it('should return false if a query string filter cannot be parsed', () => { + const filter = { + meta: { key: 'query', type: 'query_string' }, + query: { query_string: { query: 'foo"bar' } }, + } as QueryStringFilter; + const indexPattern = { id: 'bar', fields: [{ name: 'foo' }] } as IIndexPattern; + + expect(filterMatchesIndex(filter, indexPattern)).toBe(false); + }); + + it('should return true if a query string filter references fields in an index', () => { + const filter = { + meta: { key: 'query', type: 'query_string' }, + query: { query_string: { query: 'foo: bar AND baz: firzle' } }, + } as QueryStringFilter; + const indexPattern = { id: 'bar', fields: [{ name: 'foo' }, { name: 'baz' }] } as IIndexPattern; + + expect(filterMatchesIndex(filter, indexPattern)).toBe(true); + }); + + it('should return false if a query string filter references fields not in an index', () => { + const filter = { + meta: { key: 'query', type: 'query_string' }, + query: { query_string: { query: 'foo: bar AND baz: firzle' } }, + } as QueryStringFilter; + const indexPattern = { id: 'bar', fields: [{ name: 'foo' }] } as IIndexPattern; + + expect(filterMatchesIndex(filter, indexPattern)).toBe(false); + }); }); diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.ts b/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.ts index 529e68609aeb..b2b155599813 100644 --- a/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.ts +++ b/src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.ts @@ -28,14 +28,58 @@ * under the License. */ +import { uniq } from 'lodash'; +import { parse, AST } from 'lucene'; import { IIndexPattern, IFieldType } from '../../index_patterns'; -import { Filter } from '../filters'; +import { Filter, QueryStringFilter } from '../filters'; + +const implicitLuceneField = ''; + +function getLuceneFields(ast: AST): string[] { + const fields: string[] = []; + + // Parse left side of AST (if it exists) + if ('left' in ast && ast.left) { + if ('field' in ast.left) { + if (ast.left.field && ast.left.field !== implicitLuceneField) { + fields.push(ast.left.field); + } + } else { + fields.push(...getLuceneFields(ast.left)); + } + } + + // Parse right side of AST (if it exists) + if ('right' in ast && ast.right) { + if ('field' in ast.right) { + if (ast.right.field && ast.right.field !== implicitLuceneField) { + fields.push(ast.right.field); + } + } else { + fields.push(...getLuceneFields(ast.right)); + } + } + return fields; +} export function filterMatchesIndex(filter: Filter, indexPattern?: IIndexPattern | null) { if (!filter.meta?.key || !indexPattern) { return true; } + if (filter.meta?.type === 'query_string') { + const qsFilter = filter as QueryStringFilter; + try { + const ast = parse(qsFilter.query.query_string.query); + const filterFields = uniq(getLuceneFields(ast)); + return filterFields.every((filterField) => + indexPattern.fields.some((field: IFieldType) => field.name === filterField) + ); + } catch { + return false; + } + } + if (filter.meta?.type === 'custom') { return filter.meta.index === indexPattern.id; } diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/index.ts b/src/plugins/data/common/opensearch_query/opensearch_query/index.ts index ba3fe8817006..3a9bf1892a8a 100644 --- a/src/plugins/data/common/opensearch_query/opensearch_query/index.ts +++ b/src/plugins/data/common/opensearch_query/opensearch_query/index.ts @@ -33,3 +33,4 @@ export { buildQueryFromFilters } from './from_filters'; export { luceneStringToDsl } from './lucene_string_to_dsl'; export { decorateQuery } from './decorate_query'; export { getOpenSearchQueryConfig } from './get_opensearch_query_config'; +export { filterMatchesIndex } from './filter_matches_index'; diff --git a/src/plugins/data/public/ui/filter_bar/filter_item.tsx b/src/plugins/data/public/ui/filter_bar/filter_item.tsx index a0631a52b53e..15e6523f3532 100644 --- a/src/plugins/data/public/ui/filter_bar/filter_item.tsx +++ b/src/plugins/data/public/ui/filter_bar/filter_item.tsx @@ -44,6 +44,7 @@ import { toggleFilterPinned, toggleFilterDisabled, getIndexPatternFromFilter, + filterMatchesIndex, } from '../../../common'; import { getIndexPatterns } from '../../services'; @@ -268,11 +269,7 @@ export function FilterItem(props: Props) { const ip = getIndexPatternFromFilter(filter, indexPatterns); if (ip) return true; - const allFields = indexPatterns.map((indexPattern) => { - return indexPattern.fields.map((field) => field.name); - }); - const flatFields = allFields.reduce((acc: string[], it: string[]) => [...acc, ...it], []); - return flatFields.includes(filter.meta?.key || ''); + return indexPatterns.some((indexPattern) => filterMatchesIndex(filter, indexPattern)); } function getValueLabel(): LabelOptions { diff --git a/yarn.lock b/yarn.lock index b07d109ae494..832c2497adb5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3403,6 +3403,11 @@ resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.1.tgz#c48c2e27b65d2a153b19bfc1a317e30872e01eef" integrity sha512-ssE3Vlrys7sdIzs5LOxCzTVMsU7i9oa/IaW92wF32JFb3CVczqOkru2xspuKczHEbG3nvmPY7IFqVmGGHdNbYw== +"@types/lucene@^2.1.7": + version "2.1.7" + resolved "https://registry.yarnpkg.com/@types/lucene/-/lucene-2.1.7.tgz#fbdea914c5b7d91fd164664ccc6019ed210e729b" + integrity sha512-i3J0OV0RoJSskOJUa76Hgz09deabWwfJajsUxc1M05HryjPpPEKqtRklKe0+O0XVhdrFIiFO1/SInXpDCacfNA== + "@types/markdown-it@^0.0.7": version "0.0.7" resolved "https://registry.yarnpkg.com/@types/markdown-it/-/markdown-it-0.0.7.tgz#75070485a3d8ad11e7deb8287f4430be15bf4d39" @@ -12248,6 +12253,11 @@ lru-queue@^0.1.0: dependencies: es5-ext "~0.10.2" +lucene@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/lucene/-/lucene-2.1.1.tgz#e710cc123b214eaf72a4c5f1da06943c0af44d86" + integrity sha512-l0qCX+pgXEZh/7sYQNG+vzhOIFRPjlJJkQ/irk9n7Ak3d+1MrU6F7IV31KILwFkUn153oLK8a2AIt48DzLdVPg== + lz-string@^1.4.4: version "1.4.4" resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26"