Merge pull request #3797 from terascope/ip-duplicate-code

Ip duplicate code
terascope · Oct 17, 2024 · c19e131 · c19e131
2 parents 1dbda7f + 0f5f4e9
commit c19e131
Show file tree

Hide file tree

Showing 15 changed files with 77 additions and 141 deletions.
diff --git a/e2e/package.json b/e2e/package.json
@@ -45,7 +45,7 @@
     "devDependencies": {
         "@terascope/types": "^1.2.0",
         "bunyan": "^1.8.15",
-        "elasticsearch-store": "^1.3.0",
+        "elasticsearch-store": "^1.3.1",
         "fs-extra": "^11.2.0",
         "ms": "^2.1.3",
         "nanoid": "^5.0.7",

diff --git a/packages/data-mate/package.json b/packages/data-mate/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-mate",
     "displayName": "Data-Mate",
-    "version": "1.3.0",
+    "version": "1.3.1",
     "description": "Library of data validations/transformations",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme",
     "repository": {

diff --git a/packages/data-mate/src/data-frame/search/buildMatcherForNode.ts b/packages/data-mate/src/data-frame/search/buildMatcherForNode.ts
@@ -8,8 +8,9 @@ import { inspect } from 'node:util';
 import * as p from 'xlucene-parser';
 import type { DataFrame } from '../DataFrame.js';
 import { compareTermDates, dateRange } from './date-utils.js';
-import { MatchRowFn, MatchValueFn } from './interfaces.js';
-import { ipRange, ipTerm } from './ip-utils.js';
+import { MatchRowFn } from './interfaces.js';
+import { MatchValueFn } from '../../interfaces.js';
+import { ipRangeOrThrow, ipTermOrThrow } from '../../ip-utils.js';
 import { findWildcardFields, regexp, wildcard } from './wildcards-and-regex-utils.js';
 
 /**
@@ -180,7 +181,7 @@ function typeFunctions(
     if (type === xLuceneFieldType.IP) {
         if (p.isRange(node)) {
             const rangeQuery = p.parseRange(node, variables);
-            return ipRange(rangeQuery);
+            return ipRangeOrThrow(rangeQuery);
         }
 
         const value = p.getFieldValue(node.value, variables);
@@ -189,7 +190,7 @@ function typeFunctions(
             return () => false;
         }
 
-        return ipTerm(value);
+        return ipTermOrThrow(value);
     }
 
     if (type === xLuceneFieldType.Boolean) {

diff --git a/packages/data-mate/src/data-frame/search/date-utils.ts b/packages/data-mate/src/data-frame/search/date-utils.ts
@@ -4,7 +4,7 @@ import {
 } from 'date-fns';
 import { getTypeOf, getValidDate } from '@terascope/utils';
 import { isInfiniteMax, isInfiniteMin, ParsedRange } from 'xlucene-parser';
-import { MatchValueFn } from './interfaces.js';
+import { MatchValueFn } from '../../interfaces.js';
 
 // TODO: handle datemath
 

diff --git a/packages/data-mate/src/data-frame/search/interfaces.ts b/packages/data-mate/src/data-frame/search/interfaces.ts
@@ -1,7 +1,3 @@
 export interface MatchRowFn {
     (rowIndex: number): boolean;
 }
-
-export interface MatchValueFn {
-    (value: unknown): boolean;
-}
diff --git a/packages/data-mate/src/data-frame/search/wildcards-and-regex-utils.ts b/packages/data-mate/src/data-frame/search/wildcards-and-regex-utils.ts
@@ -2,7 +2,7 @@ import {
     wildCardToRegex, matchWildcard, match,
     isString, getTypeOf
 } from '@terascope/utils';
-import { MatchValueFn } from './interfaces.js';
+import { MatchValueFn } from '../../interfaces.js';
 
 export function regexp(regexStr: unknown): MatchValueFn {
     if (!isString(regexStr)) {

diff --git a/packages/data-mate/src/document-matcher/logic-builder/index.ts b/packages/data-mate/src/document-matcher/logic-builder/index.ts
@@ -8,7 +8,7 @@ import {
 import { compareTermDates, dateRange } from './dates.js';
 import { regexp, wildcard, findWildcardField } from './string.js';
 import { BooleanCB } from '../interfaces.js';
-import { ipTerm, ipRange } from './ip.js';
+import { ipTerm, ipRange } from '../../ip-utils.js';
 
 export default function buildLogicFn(
     parser: p.Parser,

diff --git a/packages/data-mate/src/document-matcher/logic-builder/ip.ts b/packages/data-mate/src/document-matcher/logic-builder/ip.ts
diff --git a/packages/data-mate/src/interfaces.ts b/packages/data-mate/src/interfaces.ts
@@ -35,3 +35,7 @@ export interface ExtractFieldConfig {
     start?: any;
     end?: any;
 }
+
+export interface MatchValueFn {
+    (value: unknown): boolean;
+}
diff --git a/...ta-mate/src/data-frame/search/ip-utils.ts → packages/data-mate/src/ip-utils.ts b/...ta-mate/src/data-frame/search/ip-utils.ts → packages/data-mate/src/ip-utils.ts
@@ -27,22 +27,43 @@ function getRangeValues(rangeQuery: ParsedRange): {
     };
 }
 
+export function ipTermOrThrow(value: unknown) {
+    const argCidr = isString(value) ? isCidr(value) : false;
+
+    if (argCidr) {
+        const range = ip6addr.createCIDR(`${value}`);
+        return pRangeTerm(range, true);
+    }
+
+    return isIPTerm(value, true);
+}
+
 export function ipTerm(value: unknown): MatchValueFn {
     const argCidr = isString(value) ? isCidr(value) : false;
+
     if (argCidr) {
         const range = ip6addr.createCIDR(`${value}`);
-        return pRangeTerm(range);
+        return pRangeTerm(range, false);
     }
 
-    return function isIPTerm(ip) {
-        if (ip == null) return false;
-        if (!isString(ip)) {
+    return isIPTerm(value, false);
+}
+
+function isIPTerm(value: unknown, shouldThrow: boolean) {
+    return function _isIPTerm(ip: unknown) {
+        if (ip == null) {
+            return false;
+        }
+
+        if (shouldThrow && !isString(ip)) {
             throw new TypeError(`Expected string for IP term match, got ${ip} (${getTypeOf(ip)})`);
         }
-        if (isNonZeroCidr(ip)) {
-            const argRange = ip6addr.createCIDR(ip);
+
+        if (isNonZeroCidr(ip as any)) {
+            const argRange = ip6addr.createCIDR(ip as any);
             return argRange.contains(`${value}`);
         }
+
         return ip === value;
     };
 }
@@ -73,11 +94,13 @@ function validateIPRange(rangeQuery: ParsedRange) {
         if (!parsed) throw new Error(`Invalid min IP value ${minValue}`);
         minValue = parsed.toString();
     }
+
     if (!incMax) {
         const parsed = ip6addr.parse(maxValue).offset(-1);
         if (!parsed) throw new Error(`Invalid max IP value ${maxValue}`);
         maxValue = parsed.toString();
     }
+
     return { minValue, maxValue };
 }
 
@@ -91,22 +114,39 @@ function checkCidr(ip: string, range: ip6addr.AddrRange) {
     );
 }
 
-function pRangeTerm(range: ip6addr.AddrRange): MatchValueFn {
+function pRangeTerm(range: ip6addr.AddrRange, shouldThrow: boolean): MatchValueFn {
     return function checkIP(ip) {
-        if (ip == null) return false;
-        if (!isString(ip)) {
+        if (ip == null) {
+            return false;
+        }
+
+        if (shouldThrow && !isString(ip)) {
             throw new TypeError(`Expected string for IP Range match, got ${ip} (${getTypeOf(ip)})`);
         }
-        if (isNonZeroCidr(ip)) {
-            return checkCidr(ip, range);
+
+        if (isNonZeroCidr(ip as any)) {
+            return checkCidr(ip as any, range);
+        }
+
+        if (isIP(ip as any)) {
+            return range.contains(ip as any);
         }
-        if (isIP(ip)) return range.contains(ip);
+
         return false;
     };
 }
 
-export function ipRange(rangeQuery: ParsedRange): MatchValueFn {
+function createRange(rangeQuery: ParsedRange) {
     const { minValue, maxValue } = validateIPRange(rangeQuery);
-    const range = ip6addr.createAddrRange(minValue, maxValue);
-    return pRangeTerm(range);
+    return ip6addr.createAddrRange(minValue, maxValue);
+}
+
+export function ipRange(rangeQuery: ParsedRange): MatchValueFn {
+    const range = createRange(rangeQuery);
+    return pRangeTerm(range, false);
+}
+
+export function ipRangeOrThrow(rangeQuery: ParsedRange) {
+    const range = createRange(rangeQuery);
+    return pRangeTerm(range, true);
 }
diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json
@@ -33,7 +33,7 @@
         "@opensearch-project/opensearch": "^1.2.0",
         "@types/elasticsearch": "^5.0.43",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^1.3.0",
+        "elasticsearch-store": "^1.3.1",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"

diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json
@@ -1,7 +1,7 @@
 {
     "name": "elasticsearch-store",
     "displayName": "Elasticsearch Store",
-    "version": "1.3.0",
+    "version": "1.3.1",
     "description": "An API for managing an elasticsearch index, with versioning and migration support.",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme",
     "bugs": {
@@ -30,7 +30,7 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-mate": "^1.3.0",
+        "@terascope/data-mate": "^1.3.1",
         "@terascope/data-types": "^1.3.0",
         "@terascope/types": "^1.2.0",
         "@terascope/utils": "^1.3.0",

diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json
@@ -1,7 +1,7 @@
 {
     "name": "terafoundation",
     "displayName": "Terafoundation",
-    "version": "1.5.0",
+    "version": "1.5.1",
     "description": "A Clustering and Foundation tool for Terascope Tools",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme",
     "bugs": {
@@ -37,7 +37,7 @@
         "convict-format-with-moment": "^6.2.0",
         "convict-format-with-validator": "^6.2.0",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^1.3.0",
+        "elasticsearch-store": "^1.3.1",
         "express": "^4.21.0",
         "js-yaml": "^4.1.0",
         "nanoid": "^5.0.7",

diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json
@@ -63,7 +63,7 @@
         "semver": "^7.6.3",
         "socket.io": "^1.7.4",
         "socket.io-client": "^1.7.4",
-        "terafoundation": "^1.5.0",
+        "terafoundation": "^1.5.1",
         "uuid": "^10.0.0"
     },
     "devDependencies": {

diff --git a/packages/ts-transforms/package.json b/packages/ts-transforms/package.json
@@ -1,7 +1,7 @@
 {
     "name": "ts-transforms",
     "displayName": "TS Transforms",
-    "version": "1.3.0",
+    "version": "1.3.1",
     "description": "An ETL framework built upon xlucene-evaluator",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/ts-transforms#readme",
     "bugs": {
@@ -36,7 +36,7 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-mate": "^1.3.0",
+        "@terascope/data-mate": "^1.3.1",
         "@terascope/types": "^1.2.0",
         "@terascope/utils": "^1.3.0",
         "awesome-phonenumber": "^7.2.0",