fix(filters): provide flag to disable special chars input filter pars…

…ing (#873)

- Slickgrid-Universal is by default parsing some special characters (<, >, =, *) when found in filter input value but in some rare occasion the user might have data that includes these special chars and might want to disable the parsing, this PR provide a new flag to do that
- ref Stack Overflow [question](https://stackoverflow.com/questions/75155658/in-angular-slickgrid-the-records-with-special-characters-are-not-gett/75160978#75160978)

.vscode/launch.json

@@ -33,8 +33,8 @@
       ],
       "console": "internalConsole",
       "internalConsoleOptions": "neverOpen",
-      "disableOptimisticBPs": true,
       "windows": {
+        "name": "Jest",
         "program": "${workspaceFolder}/node_modules/jest/bin/jest"
       }
     }

package.json

@@ -42,7 +42,8 @@
     "test:watch": "cross-env TZ='America/New_York' jest --watch --config ./test/jest.config.js"
   },
   "comments": {
-    "new-version": "To create a new version with Lerna-Lite, simply run the following script (1) 'roll-new-release'."
+    "new-version": "To create a new version with Lerna-Lite, simply run the following script (1) 'roll-new-release'.",
+    "devDependencies": "The dev deps 'jQuery', 'slickgrid', 'sortablejs' and 'whatwg-fetch' are simply installed for Jest unit tests."
   },
   "devDependencies": {
     "@4tw/cypress-drag-drop": "^2.2.3",
@@ -63,6 +64,7 @@
     "jest-cli": "^29.3.1",
     "jest-environment-jsdom": "^29.3.1",
     "jest-extended": "^3.2.3",
+    "jquery": "^3.6.3",
     "jsdom": "^21.0.0",
     "jsdom-global": "^3.0.2",
     "moment-mini": "^2.29.4",
@@ -71,8 +73,11 @@
     "rimraf": "^3.0.2",
     "rxjs": "^7.5.7",
     "serve": "^14.1.2",
+    "slickgrid": "^3.0.2",
+    "sortablejs": "^1.15.0",
     "ts-jest": "^29.0.5",
-    "typescript": "^4.9.4"
+    "typescript": "^4.9.4",
+    "whatwg-fetch": "^3.6.2"
   },
   "packageManager": "pnpm@7.25.1",
   "engines": {

packages/common/src/global-grid-options.ts

@@ -13,6 +13,7 @@ export const GlobalGridOptions: GridOption = {
   autoFixResizeTimeout: 5 * 60 * 5, // interval is 200ms, so 4x is 1sec, so (5 * 60 * 5 = 5min)
   autoFixResizeRequiredGoodCount: 2,
   autoFixResizeWhenBrokenStyleDetected: false,
+  autoParseInputFilterOperator: true,
   autoResize: {
     applyResizeToContainer: true,
     calculateAvailableSizeBy: 'window',

packages/common/src/interfaces/column.interface.ts

@@ -37,6 +37,14 @@ export interface Column<T = any> {
   /** async background post-rendering formatter */
   asyncPostRender?: (domCellNode: any, row: number, dataContext: T, columnDef: Column) => void;
 
+  /**
+   * Defaults to true, when enabled it will parse the filter input string and extract filter operator (<, <=, >=, >, =, *) when found.
+   * When an operators is found in the input string, it will automatically be converted to a Filter Operators and will no longer be part of the search value itself.
+   * For example when the input value is "> 100", it will transform the search as to a Filter Operator of ">" and a search value of "100".
+   * The only time that the user would want to disable this flag is when the user's data has any of these special characters and the user really wants to filter them as part of the string (ie: >, <, ...)
+   */
+  autoParseInputFilterOperator?: boolean;
+
   /** optional Behavior of a column with action, for example it's used by the Row Move Manager Plugin */
   behavior?: string;
 

packages/common/src/interfaces/gridOption.interface.ts

@@ -78,6 +78,14 @@ export interface GridOption {
    */
   autoFitColumnsOnFirstLoad?: boolean;
 
+  /**
+   * Defaults to true, when enabled it will parse the filter input string and extract filter operator (<, <=, >=, >, =, *) when found.
+   * When an operators is found in the input string, it will automatically be converted to a Filter Operators and will no longer be part of the search value itself.
+   * For example when the input value is "> 100", it will transform the search as to a Filter Operator of ">" and a search value of "100".
+   * The only time that the user would want to disable this flag is when the user's data has any of these special characters and the user really wants to filter them as part of the string (ie: >, <, ...)
+   */
+  autoParseInputFilterOperator?: boolean;
+
   /**
    * Defaults to false, which leads to automatically adjust the width of each column by their cell value content and only on first page/component load.
    * If you wish this resize to also re-evaluate when resizing the browser, then you should also use `enableAutoResizeColumnsByCellContent`

packages/common/src/services/__tests__/filter.service.spec.ts

@@ -849,6 +849,21 @@ describe('FilterService', () => {
       expect(output).toBe(true);
     });
 
+    it('should return False when input value has special char "*" substring but "autoParseInputFilterOperator" is set to false so the text "Jo*" will not be found', () => {
+      const searchTerms = ['Jo*'];
+      const mockColumn1 = { id: 'firstName', field: 'firstName', filterable: true, autoParseInputFilterOperator: false } as Column;
+      jest.spyOn(gridStub, 'getColumns').mockReturnValue([mockColumn1]);
+
+      service.init(gridStub);
+      const columnFilter = { columnDef: mockColumn1, columnId: 'firstName', type: FieldType.string };
+      const filterCondition = service.parseFormInputFilterConditions(searchTerms, columnFilter);
+      const parsedSearchTerms = getParsedSearchTermsByFieldType(filterCondition.searchTerms, 'text');
+      const columnFilters = { firstName: { ...columnFilter, operator: filterCondition.operator, searchTerms: filterCondition.searchTerms, parsedSearchTerms } } as ColumnFilters;
+      const output = service.customLocalFilter(mockItem1, { dataView: dataViewStub, grid: gridStub, columnFilters });
+
+      expect(output).toBe(false);
+    });
+
     it('should return True when input value from datacontext is equal to endsWith substring', () => {
       const searchTerms = ['*hn'];
       const mockColumn1 = { id: 'firstName', field: 'firstName', filterable: true } as Column;
@@ -892,6 +907,36 @@ describe('FilterService', () => {
       expect(output).toBe(true);
     });
 
+    it('should return True when input value from datacontext contains an operator ">=" and its value is greater than 10', () => {
+      const searchTerms = ['>=10'];
+      const mockColumn1 = { id: 'age', field: 'age', filterable: true, autoParseInputFilterOperator: false } as Column;
+      jest.spyOn(gridStub, 'getColumns').mockReturnValue([mockColumn1]);
+
+      service.init(gridStub);
+      const columnFilter = { columnDef: mockColumn1, columnId: 'age', type: FieldType.number };
+      const filterCondition = service.parseFormInputFilterConditions(searchTerms, columnFilter);
+      const parsedSearchTerms = getParsedSearchTermsByFieldType(filterCondition.searchTerms, 'number');
+      const columnFilters = { age: { ...columnFilter, operator: filterCondition.operator, searchTerms: filterCondition.searchTerms, parsedSearchTerms } } as ColumnFilters;
+      const output = service.customLocalFilter(mockItem1, { dataView: dataViewStub, grid: gridStub, columnFilters });
+
+      expect(output).toBe(true);
+    });
+
+    it('should return False when input value from datacontext contains an operator >= and its value is greater than 10 substring but "autoParseInputFilterOperator" is set to false', () => {
+      const searchTerms = ['>=10'];
+      const mockColumn1 = { id: 'age', field: 'age', filterable: true, autoParseInputFilterOperator: false } as Column;
+      jest.spyOn(gridStub, 'getColumns').mockReturnValue([mockColumn1]);
+
+      service.init(gridStub);
+      const columnFilter = { columnDef: mockColumn1, columnId: 'age', type: FieldType.string };
+      const filterCondition = service.parseFormInputFilterConditions(searchTerms, columnFilter);
+      const parsedSearchTerms = getParsedSearchTermsByFieldType(filterCondition.searchTerms, 'string');
+      const columnFilters = { age: { ...columnFilter, operator: filterCondition.operator, searchTerms: filterCondition.searchTerms, parsedSearchTerms } } as ColumnFilters;
+      const output = service.customLocalFilter(mockItem1, { dataView: dataViewStub, grid: gridStub, columnFilters });
+
+      expect(output).toBe(false);
+    });
+
     it('should return True when input value is a complex object searchTerms value is found following the dot notation', () => {
       const searchTerms = [123456];
       const mockColumn1 = { id: 'zip', field: 'zip', filterable: true, queryFieldFilter: 'address.zip', type: FieldType.number } as Column;

packages/common/src/services/filter.service.ts

@@ -396,7 +396,12 @@ export class FilterService {
     let matches = null;
     if (fieldType !== FieldType.object) {
       fieldSearchValue = (fieldSearchValue === undefined || fieldSearchValue === null) ? '' : `${fieldSearchValue}`; // make sure it's a string
-      matches = fieldSearchValue.match(/^([<>!=\*]{0,2})(.*[^<>!=\*])?([\*]?)$/); // group 1: Operator, 2: searchValue, 3: last char is '*' (meaning starts with, ex.: abc*)
+
+      // run regex to find possible filter operators unless the user disabled the feature
+      const autoParseInputFilterOperator = columnDef.autoParseInputFilterOperator ?? this._gridOptions.autoParseInputFilterOperator;
+      matches = autoParseInputFilterOperator !== false
+        ? fieldSearchValue.match(/^([<>!=\*]{0,2})(.*[^<>!=\*])?([\*]?)$/) // group 1: Operator, 2: searchValue, 3: last char is '*' (meaning starts with, ex.: abc*)
+        : [fieldSearchValue, '', fieldSearchValue, '']; // when parsing is disabled, we'll only keep the search value in the index 2 to make it easy for code reuse
     }
 
     let operator = matches?.[1] || columnFilter.operator;

packages/graphql/src/services/__tests__/graphql.service.spec.ts

@@ -744,7 +744,7 @@ describe('GraphqlService', () => {
       expect(currentFilters).toEqual([{ columnId: 'gender', operator: 'EQ', searchTerms: ['female'] }]);
     });
 
-    it('should return a query with search having the operator StartsWith when search value has the * symbol as the last character', () => {
+    it('should return a query with search having the operator StartsWith when search value has the "*" symbol as the last character', () => {
       const expectation = `query{users(first:10, offset:0, filterBy:[{field:gender, operator:StartsWith, value:"fem"}]) { totalCount,nodes{ id,company,gender,name } }}`;
       const mockColumn = { id: 'gender', field: 'gender' } as Column;
       const mockColumnFilters = {
@@ -758,7 +758,7 @@ describe('GraphqlService', () => {
       expect(removeSpaces(query)).toBe(removeSpaces(expectation));
     });
 
-    it('should return a query with search having the operator EndsWith when search value has the * symbol as the first character', () => {
+    it('should return a query with search having the operator EndsWith when search value has the "*" symbol as the first character', () => {
       const expectation = `query{users(first:10, offset:0, filterBy:[{field:gender, operator:EndsWith, value:"le"}]) { totalCount,nodes{ id,company,gender,name } }}`;
       const mockColumn = { id: 'gender', field: 'gender' } as Column;
       const mockColumnFilters = {
@@ -772,7 +772,7 @@ describe('GraphqlService', () => {
       expect(removeSpaces(query)).toBe(removeSpaces(expectation));
     });
 
-    it('should return a query with search having the operator EndsWith when the operator was provided as *z', () => {
+    it('should return a query with search having the operator EndsWith when the operator was provided as "*z"', () => {
       const expectation = `query{users(first:10, offset:0, filterBy:[{field:gender, operator:EndsWith, value:"le"}]) { totalCount,nodes{ id,company,gender,name } }}`;
       const mockColumn = { id: 'gender', field: 'gender' } as Column;
       const mockColumnFilters = {
@@ -786,7 +786,7 @@ describe('GraphqlService', () => {
       expect(removeSpaces(query)).toBe(removeSpaces(expectation));
     });
 
-    it('should return a query with search having the operator StartsWith even when search value last char is * symbol but the operator provided is *z', () => {
+    it('should return a query with search having the operator StartsWith even when search value last char is "*" symbol but the operator provided is "*z"', () => {
       const expectation = `query{users(first:10, offset:0, filterBy:[{field:gender, operator:StartsWith, value:"le"}]) { totalCount,nodes{ id,company,gender,name } }}`;
       const mockColumn = { id: 'gender', field: 'gender' } as Column;
       const mockColumnFilters = {
@@ -800,7 +800,7 @@ describe('GraphqlService', () => {
       expect(removeSpaces(query)).toBe(removeSpaces(expectation));
     });
 
-    it('should return a query with search having the operator EndsWith when the Column Filter was provided as *z', () => {
+    it('should return a query with search having the operator EndsWith when the Column Filter was provided as "*z"', () => {
       const expectation = `query{users(first:10, offset:0, filterBy:[{field:gender, operator:EndsWith, value:"le"}]) { totalCount,nodes{ id,company,gender,name } }}`;
       const mockColumn = { id: 'gender', field: 'gender', filter: { operator: '*z' } } as Column;
       const mockColumnFilters = {
@@ -828,7 +828,7 @@ describe('GraphqlService', () => {
       expect(removeSpaces(query)).toBe(removeSpaces(expectation));
     });
 
-    it('should return a query with search having the operator StartsWith when the operator was provided as a*', () => {
+    it('should return a query with search having the operator StartsWith when the operator was provided as "a*"', () => {
       const expectation = `query{users(first:10, offset:0, filterBy:[{field:gender, operator:StartsWith, value:"le"}]) { totalCount,nodes{ id,company,gender,name } }}`;
       const mockColumn = { id: 'gender', field: 'gender' } as Column;
       const mockColumnFilters = {
@@ -856,6 +856,34 @@ describe('GraphqlService', () => {
       expect(removeSpaces(query)).toBe(removeSpaces(expectation));
     });
 
+    it('should return a query with search having the operator Greater of Equal when the search value was provided as ">=10"', () => {
+      const expectation = `query{users(first:10, offset:0, filterBy:[{field:age, operator:GE, value:"10"}]) { totalCount,nodes{ id,company,gender,name } }}`;
+      const mockColumn = { id: 'age', field: 'age' } as Column;
+      const mockColumnFilters = {
+        age: { columnId: 'age', columnDef: mockColumn, searchTerms: ['>=10'], type: FieldType.string },
+      } as ColumnFilters;
+
+      service.init(serviceOptions, paginationOptions, gridStub);
+      service.updateFilters(mockColumnFilters, false);
+      const query = service.buildQuery();
+
+      expect(removeSpaces(query)).toBe(removeSpaces(expectation));
+    });
+
+    it('should return a query with search NOT having the operator Greater of Equal when the search value was provided as ">=10" but "autoParseInputFilterOperator" is set to false', () => {
+      const expectation = `query{users(first:10, offset:0, filterBy:[{field:age, operator:Contains, value:">=10"}]) { totalCount,nodes{ id,company,gender,name } }}`;
+      const mockColumn = { id: 'age', field: 'age', autoParseInputFilterOperator: false } as Column;
+      const mockColumnFilters = {
+        age: { columnId: 'age', columnDef: mockColumn, searchTerms: ['>=10'], type: FieldType.string },
+      } as ColumnFilters;
+
+      service.init(serviceOptions, paginationOptions, gridStub);
+      service.updateFilters(mockColumnFilters, false);
+      const query = service.buildQuery();
+
+      expect(removeSpaces(query)).toBe(removeSpaces(expectation));
+    });
+
     it('should return a query with search having a range of exclusive numbers when the search value contains 2 dots (..) to represent a range of numbers', () => {
       const expectation = `query{users(first:10, offset:0, filterBy:[{field:duration, operator:GE, value:"2"}, {field:duration, operator:LE, value:"33"}]) { totalCount,nodes{ id,company,gender,name } }}`;
       const mockColumn = { id: 'duration', field: 'duration' } as Column;

packages/graphql/src/services/graphql.service.ts

@@ -406,10 +406,16 @@ export class GraphqlService implements BackendService {
         }
 
         fieldSearchValue = (fieldSearchValue === undefined || fieldSearchValue === null) ? '' : `${fieldSearchValue}`; // make sure it's a string
-        const matches = fieldSearchValue.match(/^([<>!=\*]{0,2})(.*[^<>!=\*])([\*]?)$/); // group 1: Operator, 2: searchValue, 3: last char is '*' (meaning starts with, ex.: abc*)
-        let operator: OperatorString = columnFilter.operator || ((matches) ? matches[1] : '');
-        searchValue = (!!matches) ? matches[2] : '';
-        const lastValueChar = (!!matches) ? matches[3] : (operator === '*z' ? '*' : '');
+
+        // run regex to find possible filter operators unless the user disabled the feature
+        const autoParseInputFilterOperator = columnDef.autoParseInputFilterOperator ?? this._gridOptions.autoParseInputFilterOperator;
+        const matches = autoParseInputFilterOperator !== false
+          ? fieldSearchValue.match(/^([<>!=\*]{0,2})(.*[^<>!=\*])([\*]?)$/) // group 1: Operator, 2: searchValue, 3: last char is '*' (meaning starts with, ex.: abc*)
+          : [fieldSearchValue, '', fieldSearchValue, '']; // when parsing is disabled, we'll only keep the search value in the index 2 to make it easy for code reuse
+
+        let operator: OperatorString = columnFilter.operator ||  matches?.[1] || '';
+        searchValue = matches?.[2] || '';
+        const lastValueChar = matches?.[3] || (operator === '*z' ? '*' : '');
 
         // no need to query if search value is empty
         if (fieldName && searchValue === '' && searchTerms.length === 0) {