diff --git a/e2e/package.json b/e2e/package.json index ff1be15b37c..3b22722442c 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -45,7 +45,7 @@ "devDependencies": { "@terascope/types": "^1.2.0", "bunyan": "^1.8.15", - "elasticsearch-store": "^1.3.0", + "elasticsearch-store": "^1.3.1", "fs-extra": "^11.2.0", "ms": "^2.1.3", "nanoid": "^5.0.7", diff --git a/packages/data-mate/package.json b/packages/data-mate/package.json index e0bdf6e381b..b3a6786f75f 100644 --- a/packages/data-mate/package.json +++ b/packages/data-mate/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-mate", "displayName": "Data-Mate", - "version": "1.3.0", + "version": "1.3.1", "description": "Library of data validations/transformations", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme", "repository": { diff --git a/packages/data-mate/src/data-frame/search/buildMatcherForNode.ts b/packages/data-mate/src/data-frame/search/buildMatcherForNode.ts index 19d1511152b..5c7e2ad47d3 100644 --- a/packages/data-mate/src/data-frame/search/buildMatcherForNode.ts +++ b/packages/data-mate/src/data-frame/search/buildMatcherForNode.ts @@ -8,8 +8,9 @@ import { inspect } from 'node:util'; import * as p from 'xlucene-parser'; import type { DataFrame } from '../DataFrame.js'; import { compareTermDates, dateRange } from './date-utils.js'; -import { MatchRowFn, MatchValueFn } from './interfaces.js'; -import { ipRange, ipTerm } from './ip-utils.js'; +import { MatchRowFn } from './interfaces.js'; +import { MatchValueFn } from '../../interfaces.js'; +import { ipRangeOrThrow, ipTermOrThrow } from '../../ip-utils.js'; import { findWildcardFields, regexp, wildcard } from './wildcards-and-regex-utils.js'; /** @@ -180,7 +181,7 @@ function typeFunctions( if (type === xLuceneFieldType.IP) { if (p.isRange(node)) { const rangeQuery = p.parseRange(node, variables); - return ipRange(rangeQuery); + return ipRangeOrThrow(rangeQuery); } const value = p.getFieldValue(node.value, variables); @@ -189,7 +190,7 @@ function typeFunctions( return () => false; } - return ipTerm(value); + return ipTermOrThrow(value); } if (type === xLuceneFieldType.Boolean) { diff --git a/packages/data-mate/src/data-frame/search/date-utils.ts b/packages/data-mate/src/data-frame/search/date-utils.ts index 1173626a6b4..95cc047323e 100644 --- a/packages/data-mate/src/data-frame/search/date-utils.ts +++ b/packages/data-mate/src/data-frame/search/date-utils.ts @@ -4,7 +4,7 @@ import { } from 'date-fns'; import { getTypeOf, getValidDate } from '@terascope/utils'; import { isInfiniteMax, isInfiniteMin, ParsedRange } from 'xlucene-parser'; -import { MatchValueFn } from './interfaces.js'; +import { MatchValueFn } from '../../interfaces.js'; // TODO: handle datemath diff --git a/packages/data-mate/src/data-frame/search/interfaces.ts b/packages/data-mate/src/data-frame/search/interfaces.ts index 0c4b1e68fe7..8ef0a75762e 100644 --- a/packages/data-mate/src/data-frame/search/interfaces.ts +++ b/packages/data-mate/src/data-frame/search/interfaces.ts @@ -1,7 +1,3 @@ export interface MatchRowFn { (rowIndex: number): boolean; } - -export interface MatchValueFn { - (value: unknown): boolean; -} diff --git a/packages/data-mate/src/data-frame/search/wildcards-and-regex-utils.ts b/packages/data-mate/src/data-frame/search/wildcards-and-regex-utils.ts index 68f45dc9e5d..0e97750e8bc 100644 --- a/packages/data-mate/src/data-frame/search/wildcards-and-regex-utils.ts +++ b/packages/data-mate/src/data-frame/search/wildcards-and-regex-utils.ts @@ -2,7 +2,7 @@ import { wildCardToRegex, matchWildcard, match, isString, getTypeOf } from '@terascope/utils'; -import { MatchValueFn } from './interfaces.js'; +import { MatchValueFn } from '../../interfaces.js'; export function regexp(regexStr: unknown): MatchValueFn { if (!isString(regexStr)) { diff --git a/packages/data-mate/src/document-matcher/logic-builder/index.ts b/packages/data-mate/src/document-matcher/logic-builder/index.ts index 94faf2a2252..9eb9000ee81 100644 --- a/packages/data-mate/src/document-matcher/logic-builder/index.ts +++ b/packages/data-mate/src/document-matcher/logic-builder/index.ts @@ -8,7 +8,7 @@ import { import { compareTermDates, dateRange } from './dates.js'; import { regexp, wildcard, findWildcardField } from './string.js'; import { BooleanCB } from '../interfaces.js'; -import { ipTerm, ipRange } from './ip.js'; +import { ipTerm, ipRange } from '../../ip-utils.js'; export default function buildLogicFn( parser: p.Parser, diff --git a/packages/data-mate/src/document-matcher/logic-builder/ip.ts b/packages/data-mate/src/document-matcher/logic-builder/ip.ts deleted file mode 100644 index d2d91e237d4..00000000000 --- a/packages/data-mate/src/document-matcher/logic-builder/ip.ts +++ /dev/null @@ -1,105 +0,0 @@ -import isCidr from 'is-cidr'; -import ip6addr from 'ip6addr'; -import { isIP, isIPv6 } from 'is-ip'; -import { isInfiniteMin, isInfiniteMax, ParsedRange } from 'xlucene-parser'; -import { isString, isNonZeroCidr } from '@terascope/utils'; -import { BooleanCB } from '../interfaces.js'; - -const MIN_IPV4_IP = '0.0.0.0'; -const MAX_IPV4_IP = '255.255.255.255'; - -const MIN_IPV6_IP = '::'; -const MAX_IPV6_IP = 'ffff.ffff.ffff.ffff.ffff.ffff.ffff.ffff'; - -function getRangeValues(rangeQuery: ParsedRange): { - incMin: boolean; - incMax: boolean; - minValue: string; - maxValue: string; -} { - const incMin = rangeQuery.gte != null; - const incMax = rangeQuery.lte != null; - const minValue = `${rangeQuery.gte || rangeQuery.gt || '*'}`; - const maxValue = `${rangeQuery.lte || rangeQuery.lt || '*'}`; - - return { - incMin, incMax, minValue, maxValue - }; -} - -export function ipTerm(value: unknown): BooleanCB { - const argCidr = isString(value) ? isCidr(value) : false; - if (argCidr) { - const range = ip6addr.createCIDR(`${value}`); - return pRangeTerm(range); - } - - return function isIPTerm(ip: string) { - if (isNonZeroCidr(ip)) { - const argRange = ip6addr.createCIDR(ip); - return argRange.contains(`${value}`); - } - return ip === value; - }; -} - -function validateIPRange(rangeQuery: ParsedRange) { - const values = getRangeValues(rangeQuery); - const { incMin, incMax } = values; - let { minValue, maxValue } = values; - - if (isInfiniteMin(minValue)) { - if (isIPv6(maxValue)) { - minValue = MIN_IPV6_IP; - } else { - minValue = MIN_IPV4_IP; - } - } - - if (isInfiniteMax(maxValue)) { - if (isIPv6(minValue)) { - maxValue = MAX_IPV6_IP; - } else { - maxValue = MAX_IPV4_IP; - } - } - - if (!incMin) { - const parsed = ip6addr.parse(minValue).offset(1); - if (!parsed) throw new Error(`Invalid min IP value ${minValue}`); - minValue = parsed.toString(); - } - if (!incMax) { - const parsed = ip6addr.parse(maxValue).offset(-1); - if (!parsed) throw new Error(`Invalid max IP value ${maxValue}`); - maxValue = parsed.toString(); - } - return { minValue, maxValue }; -} - -function checkCidr(ip: string, range: any) { - const argRange = ip6addr.createCIDR(ip); - return ( - range.contains(argRange.first().toString()) - || range.contains(argRange.last().toString()) - || argRange.contains(range.first().toString()) - || argRange.contains(range.last().toString()) - ); -} - -function pRangeTerm(range: any) { - return function checkIP(ip: unknown) { - if (ip === null || ip === undefined) return false; - if (isNonZeroCidr(ip as any)) { - return checkCidr(ip as any, range); - } - if (isIP(ip as any)) return range.contains(ip); - return false; - }; -} - -export function ipRange(rangeQuery: ParsedRange): BooleanCB { - const { minValue, maxValue } = validateIPRange(rangeQuery); - const range = ip6addr.createAddrRange(minValue, maxValue); - return pRangeTerm(range); -} diff --git a/packages/data-mate/src/interfaces.ts b/packages/data-mate/src/interfaces.ts index db71a0981d2..aed5fb28ecc 100644 --- a/packages/data-mate/src/interfaces.ts +++ b/packages/data-mate/src/interfaces.ts @@ -35,3 +35,7 @@ export interface ExtractFieldConfig { start?: any; end?: any; } + +export interface MatchValueFn { + (value: unknown): boolean; +} diff --git a/packages/data-mate/src/data-frame/search/ip-utils.ts b/packages/data-mate/src/ip-utils.ts similarity index 68% rename from packages/data-mate/src/data-frame/search/ip-utils.ts rename to packages/data-mate/src/ip-utils.ts index e7bb2098e7b..7ae5f07d0ca 100644 --- a/packages/data-mate/src/data-frame/search/ip-utils.ts +++ b/packages/data-mate/src/ip-utils.ts @@ -27,22 +27,43 @@ function getRangeValues(rangeQuery: ParsedRange): { }; } +export function ipTermOrThrow(value: unknown) { + const argCidr = isString(value) ? isCidr(value) : false; + + if (argCidr) { + const range = ip6addr.createCIDR(`${value}`); + return pRangeTerm(range, true); + } + + return isIPTerm(value, true); +} + export function ipTerm(value: unknown): MatchValueFn { const argCidr = isString(value) ? isCidr(value) : false; + if (argCidr) { const range = ip6addr.createCIDR(`${value}`); - return pRangeTerm(range); + return pRangeTerm(range, false); } - return function isIPTerm(ip) { - if (ip == null) return false; - if (!isString(ip)) { + return isIPTerm(value, false); +} + +function isIPTerm(value: unknown, shouldThrow: boolean) { + return function _isIPTerm(ip: unknown) { + if (ip == null) { + return false; + } + + if (shouldThrow && !isString(ip)) { throw new TypeError(`Expected string for IP term match, got ${ip} (${getTypeOf(ip)})`); } - if (isNonZeroCidr(ip)) { - const argRange = ip6addr.createCIDR(ip); + + if (isNonZeroCidr(ip as any)) { + const argRange = ip6addr.createCIDR(ip as any); return argRange.contains(`${value}`); } + return ip === value; }; } @@ -73,11 +94,13 @@ function validateIPRange(rangeQuery: ParsedRange) { if (!parsed) throw new Error(`Invalid min IP value ${minValue}`); minValue = parsed.toString(); } + if (!incMax) { const parsed = ip6addr.parse(maxValue).offset(-1); if (!parsed) throw new Error(`Invalid max IP value ${maxValue}`); maxValue = parsed.toString(); } + return { minValue, maxValue }; } @@ -91,22 +114,39 @@ function checkCidr(ip: string, range: ip6addr.AddrRange) { ); } -function pRangeTerm(range: ip6addr.AddrRange): MatchValueFn { +function pRangeTerm(range: ip6addr.AddrRange, shouldThrow: boolean): MatchValueFn { return function checkIP(ip) { - if (ip == null) return false; - if (!isString(ip)) { + if (ip == null) { + return false; + } + + if (shouldThrow && !isString(ip)) { throw new TypeError(`Expected string for IP Range match, got ${ip} (${getTypeOf(ip)})`); } - if (isNonZeroCidr(ip)) { - return checkCidr(ip, range); + + if (isNonZeroCidr(ip as any)) { + return checkCidr(ip as any, range); + } + + if (isIP(ip as any)) { + return range.contains(ip as any); } - if (isIP(ip)) return range.contains(ip); + return false; }; } -export function ipRange(rangeQuery: ParsedRange): MatchValueFn { +function createRange(rangeQuery: ParsedRange) { const { minValue, maxValue } = validateIPRange(rangeQuery); - const range = ip6addr.createAddrRange(minValue, maxValue); - return pRangeTerm(range); + return ip6addr.createAddrRange(minValue, maxValue); +} + +export function ipRange(rangeQuery: ParsedRange): MatchValueFn { + const range = createRange(rangeQuery); + return pRangeTerm(range, false); +} + +export function ipRangeOrThrow(rangeQuery: ParsedRange) { + const range = createRange(rangeQuery); + return pRangeTerm(range, true); } diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json index 5e70784af51..70fc6801348 100644 --- a/packages/elasticsearch-api/package.json +++ b/packages/elasticsearch-api/package.json @@ -33,7 +33,7 @@ "@opensearch-project/opensearch": "^1.2.0", "@types/elasticsearch": "^5.0.43", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^1.3.0", + "elasticsearch-store": "^1.3.1", "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0", "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0", "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0" diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json index e0562d7fafe..377f73a540a 100644 --- a/packages/elasticsearch-store/package.json +++ b/packages/elasticsearch-store/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-store", "displayName": "Elasticsearch Store", - "version": "1.3.0", + "version": "1.3.1", "description": "An API for managing an elasticsearch index, with versioning and migration support.", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme", "bugs": { @@ -30,7 +30,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^1.3.0", + "@terascope/data-mate": "^1.3.1", "@terascope/data-types": "^1.3.0", "@terascope/types": "^1.2.0", "@terascope/utils": "^1.3.0", diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json index c1cd6fc45f4..5e1239828df 100644 --- a/packages/terafoundation/package.json +++ b/packages/terafoundation/package.json @@ -1,7 +1,7 @@ { "name": "terafoundation", "displayName": "Terafoundation", - "version": "1.5.0", + "version": "1.5.1", "description": "A Clustering and Foundation tool for Terascope Tools", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme", "bugs": { @@ -37,7 +37,7 @@ "convict-format-with-moment": "^6.2.0", "convict-format-with-validator": "^6.2.0", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^1.3.0", + "elasticsearch-store": "^1.3.1", "express": "^4.21.0", "js-yaml": "^4.1.0", "nanoid": "^5.0.7", diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 54b0bcf092e..4dc2ab6aee6 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -63,7 +63,7 @@ "semver": "^7.6.3", "socket.io": "^1.7.4", "socket.io-client": "^1.7.4", - "terafoundation": "^1.5.0", + "terafoundation": "^1.5.1", "uuid": "^10.0.0" }, "devDependencies": { diff --git a/packages/ts-transforms/package.json b/packages/ts-transforms/package.json index 6a00d4c5cf1..ec74376b57e 100644 --- a/packages/ts-transforms/package.json +++ b/packages/ts-transforms/package.json @@ -1,7 +1,7 @@ { "name": "ts-transforms", "displayName": "TS Transforms", - "version": "1.3.0", + "version": "1.3.1", "description": "An ETL framework built upon xlucene-evaluator", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/ts-transforms#readme", "bugs": { @@ -36,7 +36,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^1.3.0", + "@terascope/data-mate": "^1.3.1", "@terascope/types": "^1.2.0", "@terascope/utils": "^1.3.0", "awesome-phonenumber": "^7.2.0",