From 5d2589478f32f59d0d0cc6e690aadfb3a949b2f3 Mon Sep 17 00:00:00 2001 From: Stratoula Kalafateli Date: Thu, 21 Nov 2024 08:14:34 +0100 Subject: [PATCH 1/3] [ES|QL] Update the functions definitions --- .../generated/aggregation_functions.ts | 87 +++++- .../definitions/generated/scalar_functions.ts | 282 +++++++++++++++++- .../sections/generated/scalar_functions.tsx | 20 +- 3 files changed, 374 insertions(+), 15 deletions(-) diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/aggregation_functions.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/aggregation_functions.ts index 6a429f2288f94..37dfe8de5822f 100644 --- a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/aggregation_functions.ts +++ b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/aggregation_functions.ts @@ -216,7 +216,7 @@ const countDefinition: FunctionDefinition = { validate: undefined, examples: [ 'FROM employees\n| STATS COUNT(height)', - 'FROM employees \n| STATS count = COUNT(*) BY languages \n| SORT languages DESC', + 'FROM employees\n| STATS count = COUNT(*) BY languages\n| SORT languages DESC', 'ROW words="foo;bar;baz;qux;quux;foo"\n| STATS word_count = COUNT(SPLIT(words, ";"))', 'ROW n=1\n| WHERE n < 0\n| STATS COUNT(n)', 'ROW n=1\n| STATS COUNT(n > 0 OR NULL), COUNT(n < 0 OR NULL)', @@ -343,6 +343,61 @@ const countDistinctDefinition: FunctionDefinition = { ], returnType: 'long', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'long', + }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + { + name: 'precision', + type: 'integer', + optional: true, + }, + ], + returnType: 'long', + }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + { + name: 'precision', + type: 'long', + optional: true, + }, + ], + returnType: 'long', + }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + { + name: 'precision', + type: 'unsigned_long', + optional: true, + }, + ], + returnType: 'long', + }, { params: [ { @@ -769,6 +824,16 @@ const maxDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date_nanos', + }, { params: [ { @@ -984,6 +1049,16 @@ const minDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date_nanos', + }, { params: [ { @@ -1544,6 +1619,16 @@ const valuesDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date_nanos', + }, { params: [ { diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts index fb98d7cb4b212..0fdf98c00d44b 100644 --- a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts +++ b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts @@ -892,6 +892,22 @@ const coalesceDefinition: FunctionDefinition = { returnType: 'date', minParams: 1, }, + { + params: [ + { + name: 'first', + type: 'date_nanos', + optional: false, + }, + { + name: 'rest', + type: 'date_nanos', + optional: true, + }, + ], + returnType: 'date_nanos', + minParams: 1, + }, { params: [ { @@ -1624,6 +1640,21 @@ const dateTruncDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'interval', + type: 'date_period', + optional: false, + }, + { + name: 'date', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date_nanos', + }, { params: [ { @@ -1639,6 +1670,21 @@ const dateTruncDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'interval', + type: 'time_duration', + optional: false, + }, + { + name: 'date', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date_nanos', + }, ], supportedCommands: ['stats', 'inlinestats', 'metrics', 'eval', 'where', 'row', 'sort'], supportedOptions: ['by'], @@ -1954,6 +2000,22 @@ const greatestDefinition: FunctionDefinition = { returnType: 'date', minParams: 1, }, + { + params: [ + { + name: 'first', + type: 'date_nanos', + optional: false, + }, + { + name: 'rest', + type: 'date_nanos', + optional: true, + }, + ], + returnType: 'date_nanos', + minParams: 1, + }, { params: [ { @@ -2468,6 +2530,22 @@ const leastDefinition: FunctionDefinition = { returnType: 'date', minParams: 1, }, + { + params: [ + { + name: 'first', + type: 'date_nanos', + optional: false, + }, + { + name: 'rest', + type: 'date_nanos', + optional: true, + }, + ], + returnType: 'date_nanos', + minParams: 1, + }, { params: [ { @@ -3402,7 +3480,7 @@ const matchDefinition: FunctionDefinition = { supportedOptions: [], validate: undefined, examples: [ - 'from books \n| where match(author, "Faulkner")\n| keep book_no, author \n| sort book_no \n| limit 5;', + 'FROM books \n| WHERE MATCH(author, "Faulkner")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;', ], }; @@ -3808,6 +3886,16 @@ const mvCountDefinition: FunctionDefinition = { ], returnType: 'integer', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'integer', + }, { params: [ { @@ -3965,6 +4053,16 @@ const mvDedupeDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date_nanos', + }, { params: [ { @@ -4113,6 +4211,16 @@ const mvFirstDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date_nanos', + }, { params: [ { @@ -4271,6 +4379,16 @@ const mvLastDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date_nanos', + }, { params: [ { @@ -4409,6 +4527,16 @@ const mvMaxDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date_nanos', + }, { params: [ { @@ -4654,6 +4782,16 @@ const mvMinDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date_nanos', + }, { params: [ { @@ -5028,6 +5166,26 @@ const mvSliceDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + { + name: 'start', + type: 'integer', + optional: false, + }, + { + name: 'end', + type: 'integer', + optional: true, + }, + ], + returnType: 'date_nanos', + }, { params: [ { @@ -5260,6 +5418,22 @@ const mvSortDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + { + name: 'order', + type: 'keyword', + optional: true, + acceptedValues: ['asc', 'desc'], + }, + ], + returnType: 'date_nanos', + }, { params: [ { @@ -5367,7 +5541,6 @@ const mvSortDefinition: FunctionDefinition = { name: 'order', type: 'keyword', optional: true, - acceptedValues: ['asc', 'desc'], }, ], returnType: 'version', @@ -6017,7 +6190,7 @@ const qstrDefinition: FunctionDefinition = { supportedOptions: [], validate: undefined, examples: [ - 'from books \n| where qstr("author: Faulkner")\n| keep book_no, author \n| sort book_no \n| limit 5;', + 'FROM books \n| WHERE QSTR("author: Faulkner")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;', ], }; @@ -8030,7 +8203,78 @@ const toDateNanosDefinition: FunctionDefinition = { }), preview: true, alias: undefined, - signatures: [], + signatures: [ + { + params: [ + { + name: 'field', + type: 'date', + optional: false, + }, + ], + returnType: 'date_nanos', + }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date_nanos', + }, + { + params: [ + { + name: 'field', + type: 'double', + optional: false, + }, + ], + returnType: 'date_nanos', + }, + { + params: [ + { + name: 'field', + type: 'keyword', + optional: false, + }, + ], + returnType: 'date_nanos', + }, + { + params: [ + { + name: 'field', + type: 'long', + optional: false, + }, + ], + returnType: 'date_nanos', + }, + { + params: [ + { + name: 'field', + type: 'text', + optional: false, + }, + ], + returnType: 'date_nanos', + }, + { + params: [ + { + name: 'field', + type: 'unsigned_long', + optional: false, + }, + ], + returnType: 'date_nanos', + }, + ], supportedCommands: ['stats', 'inlinestats', 'metrics', 'eval', 'where', 'row', 'sort'], supportedOptions: ['by'], validate: undefined, @@ -8107,6 +8351,16 @@ const toDatetimeDefinition: FunctionDefinition = { ], returnType: 'date', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'date', + }, { params: [ { @@ -8680,6 +8934,16 @@ const toLongDefinition: FunctionDefinition = { ], returnType: 'long', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'long', + }, { params: [ { @@ -8893,6 +9157,16 @@ const toStringDefinition: FunctionDefinition = { ], returnType: 'keyword', }, + { + params: [ + { + name: 'field', + type: 'date_nanos', + optional: false, + }, + ], + returnType: 'keyword', + }, { params: [ { diff --git a/packages/kbn-language-documentation/src/sections/generated/scalar_functions.tsx b/packages/kbn-language-documentation/src/sections/generated/scalar_functions.tsx index 8444ae16b644a..55eadf85a09f6 100644 --- a/packages/kbn-language-documentation/src/sections/generated/scalar_functions.tsx +++ b/packages/kbn-language-documentation/src/sections/generated/scalar_functions.tsx @@ -1280,11 +1280,11 @@ export const functions = { Performs a match query on the specified field. Returns true if the provided query matches the row. \`\`\` - from books - | where match(author, "Faulkner") - | keep book_no, author - | sort book_no - | limit 5; + FROM books + | WHERE MATCH(author, "Faulkner") + | KEEP book_no, author + | SORT book_no + | LIMIT 5; \`\`\` `, description: @@ -1996,11 +1996,11 @@ export const functions = { Performs a query string query. Returns true if the provided query string matches the row. \`\`\` - from books - | where qstr("author: Faulkner") - | keep book_no, author - | sort book_no - | limit 5; + FROM books + | WHERE QSTR("author: Faulkner") + | KEEP book_no, author + | SORT book_no + | LIMIT 5; \`\`\` `, description: From 63b629c45d3f1116be4cfc088cfc1a6f4bb86c18 Mon Sep 17 00:00:00 2001 From: Stratoula Kalafateli Date: Thu, 21 Nov 2024 12:22:11 +0100 Subject: [PATCH 2/3] Fix unit tests --- .../__tests__/autocomplete.command.stats.test.ts | 12 +++++++++++- .../__tests__/autocomplete.suggest.eval.test.ts | 14 +++++++++----- .../src/validation/esql_validation_meta_tests.json | 8 ++------ .../src/validation/validation.test.ts | 12 ++---------- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.stats.test.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.stats.test.ts index 829c12f7dabba..5234e93c159e2 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.stats.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.stats.test.ts @@ -150,6 +150,7 @@ describe('autocomplete.suggest', () => { ...getFieldNamesByType([ ...ESQL_COMMON_NUMERIC_TYPES, 'date', + 'date_nanos', 'boolean', 'ip', 'version', @@ -158,7 +159,16 @@ describe('autocomplete.suggest', () => { ]), ...getFunctionSignaturesByReturnType( 'stats', - [...ESQL_COMMON_NUMERIC_TYPES, 'date', 'boolean', 'ip', 'version', 'text', 'keyword'], + [ + ...ESQL_COMMON_NUMERIC_TYPES, + 'date', + 'boolean', + 'ip', + 'version', + 'text', + 'keyword', + 'date_nanos', + ], { scalar: true, } diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.eval.test.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.eval.test.ts index aae715ee66749..e81e74427b721 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.eval.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.suggest.eval.test.ts @@ -370,8 +370,12 @@ describe('autocomplete.suggest', () => { // // Test suggestions for each possible param, within each signature variation, for each function for (const fn of scalarFunctionDefinitions) { // skip this fn for the moment as it's quite hard to test - // Add match in the text when the autocomplete is ready https://github.com/elastic/kibana/issues/196995 - if (!['bucket', 'date_extract', 'date_diff', 'case', 'match', 'qstr'].includes(fn.name)) { + // Add match in the test when the autocomplete is ready https://github.com/elastic/kibana/issues/196995 + if ( + !['bucket', 'date_extract', 'date_diff', 'case', 'match', 'qstr', 'date_trunc'].includes( + fn.name + ) + ) { test(`${fn.name}`, async () => { const testedCases = new Set(); @@ -539,9 +543,9 @@ describe('autocomplete.suggest', () => { 'from a | eval var0=date_trunc(/)', [ ...getLiteralsByType('time_literal').map((t) => `${t}, `), - ...getFunctionSignaturesByReturnType('eval', 'time_duration', { scalar: true }).map( - (t) => `${t.text},` - ), + ...getFunctionSignaturesByReturnType('eval', ['time_duration', 'date_period'], { + scalar: true, + }).map((t) => `${t.text},`), ], { triggerCharacter: '(' } ); diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json b/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json index fee9f90f38c93..a216ae7e965ad 100644 --- a/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json +++ b/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json @@ -923,9 +923,7 @@ { "query": "row var = mv_sort([\"a\", \"b\"], \"bogus\")", "error": [], - "warning": [ - "Invalid option [\"bogus\"] for mv_sort. Supported options: [\"asc\", \"desc\"]." - ] + "warning": [] }, { "query": "row var = mv_sort([\"a\", \"b\"], \"ASC\")", @@ -6998,9 +6996,7 @@ { "query": "from a_index | eval mv_sort([\"a\", \"b\"], \"bogus\")", "error": [], - "warning": [ - "Invalid option [\"bogus\"] for mv_sort. Supported options: [\"asc\", \"desc\"]." - ] + "warning": [] }, { "query": "from a_index | eval mv_sort([\"a\", \"b\"], \"ASC\")", diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts b/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts index 68d8ebb233f5e..7c4a0b3695ae6 100644 --- a/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts @@ -424,11 +424,7 @@ describe('validation logic', () => { ]); } - testErrorsAndWarnings( - `row var = mv_sort(["a", "b"], "bogus")`, - [], - ['Invalid option ["bogus"] for mv_sort. Supported options: ["asc", "desc"].'] - ); + testErrorsAndWarnings(`row var = mv_sort(["a", "b"], "bogus")`, [], []); testErrorsAndWarnings(`row var = mv_sort(["a", "b"], "ASC")`, []); testErrorsAndWarnings(`row var = mv_sort(["a", "b"], "DESC")`, []); @@ -1222,11 +1218,7 @@ describe('validation logic', () => { "SyntaxError: mismatched input '' expecting {',', ')'}", ]); - testErrorsAndWarnings( - 'from a_index | eval mv_sort(["a", "b"], "bogus")', - [], - ['Invalid option ["bogus"] for mv_sort. Supported options: ["asc", "desc"].'] - ); + testErrorsAndWarnings('from a_index | eval mv_sort(["a", "b"], "bogus")', [], []); testErrorsAndWarnings(`from a_index | eval mv_sort(["a", "b"], "ASC")`, []); testErrorsAndWarnings(`from a_index | eval mv_sort(["a", "b"], "DESC")`, []); From 9871adf989340dc5cc76c9a86271c21c0fc344d2 Mon Sep 17 00:00:00 2001 From: Stratoula Kalafateli Date: Thu, 21 Nov 2024 16:25:57 +0100 Subject: [PATCH 3/3] Fix mvsort order problem --- .../scripts/generate_function_definitions.ts | 2 +- .../src/definitions/generated/scalar_functions.ts | 1 + .../src/validation/esql_validation_meta_tests.json | 8 ++++++-- .../src/validation/validation.test.ts | 13 ++++++++++--- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts b/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts index 8462f9e2a050b..3a810cac3ad75 100644 --- a/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts +++ b/packages/kbn-esql-validation-autocomplete/scripts/generate_function_definitions.ts @@ -218,7 +218,7 @@ const functionEnrichments: Record> ], }, mv_sort: { - signatures: new Array(9).fill({ + signatures: new Array(10).fill({ params: [{}, { acceptedValues: ['asc', 'desc'] }], }), }, diff --git a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts index 0fdf98c00d44b..d45271b189915 100644 --- a/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts +++ b/packages/kbn-esql-validation-autocomplete/src/definitions/generated/scalar_functions.ts @@ -5541,6 +5541,7 @@ const mvSortDefinition: FunctionDefinition = { name: 'order', type: 'keyword', optional: true, + acceptedValues: ['asc', 'desc'], }, ], returnType: 'version', diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json b/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json index a216ae7e965ad..fee9f90f38c93 100644 --- a/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json +++ b/packages/kbn-esql-validation-autocomplete/src/validation/esql_validation_meta_tests.json @@ -923,7 +923,9 @@ { "query": "row var = mv_sort([\"a\", \"b\"], \"bogus\")", "error": [], - "warning": [] + "warning": [ + "Invalid option [\"bogus\"] for mv_sort. Supported options: [\"asc\", \"desc\"]." + ] }, { "query": "row var = mv_sort([\"a\", \"b\"], \"ASC\")", @@ -6996,7 +6998,9 @@ { "query": "from a_index | eval mv_sort([\"a\", \"b\"], \"bogus\")", "error": [], - "warning": [] + "warning": [ + "Invalid option [\"bogus\"] for mv_sort. Supported options: [\"asc\", \"desc\"]." + ] }, { "query": "from a_index | eval mv_sort([\"a\", \"b\"], \"ASC\")", diff --git a/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts b/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts index 7c4a0b3695ae6..03102474f6314 100644 --- a/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/validation/validation.test.ts @@ -424,8 +424,11 @@ describe('validation logic', () => { ]); } - testErrorsAndWarnings(`row var = mv_sort(["a", "b"], "bogus")`, [], []); - + testErrorsAndWarnings( + `row var = mv_sort(["a", "b"], "bogus")`, + [], + ['Invalid option ["bogus"] for mv_sort. Supported options: ["asc", "desc"].'] + ); testErrorsAndWarnings(`row var = mv_sort(["a", "b"], "ASC")`, []); testErrorsAndWarnings(`row var = mv_sort(["a", "b"], "DESC")`, []); @@ -1218,7 +1221,11 @@ describe('validation logic', () => { "SyntaxError: mismatched input '' expecting {',', ')'}", ]); - testErrorsAndWarnings('from a_index | eval mv_sort(["a", "b"], "bogus")', [], []); + testErrorsAndWarnings( + 'from a_index | eval mv_sort(["a", "b"], "bogus")', + [], + ['Invalid option ["bogus"] for mv_sort. Supported options: ["asc", "desc"].'] + ); testErrorsAndWarnings(`from a_index | eval mv_sort(["a", "b"], "ASC")`, []); testErrorsAndWarnings(`from a_index | eval mv_sort(["a", "b"], "DESC")`, []);