Skip to content

Commit

Permalink
Search dv only IP masks (opensearch-project#16628)
Browse files Browse the repository at this point in the history 
* search dv only ip masks

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

* search dv only ip masks: changes

Signed-off-by: Mikhail Khludnev <mkhl@apache.org>

* drop fancy closures

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

* drop fancy closures. one more

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

* fix unit tests; add some more dvOnly

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

* drop skipping dvOnly in 2.x

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

* drop redundant brackets

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

* extract conditions

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

* asserts

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

* spotless apply

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

* bring back skip before

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

* combine asserts

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

* inline, copy-paste

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>

---------

Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>
Signed-off-by: Mikhail Khludnev <mkhl@apache.org>
Co-authored-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com>
  • Loading branch information
@mkhludnev
mkhludnev and mikhail-khludnev authored Nov 15, 2024
1 parent 8e097ee commit d4d70d8
Show file tree
Hide file tree
Showing 4 changed files with 240 additions and 46 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Support retrieving doc values of unsigned long field ([#16543](https://github.com/opensearch-project/OpenSearch/pull/16543))
- Fix rollover alias supports restored searchable snapshot index([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483))
- Fix permissions error on scripted query against remote snapshot ([#16544](https://github.com/opensearch-project/OpenSearch/pull/16544))
- Fix `doc_values` only (`index:false`) IP field searching for masks ([#16628](https://github.com/opensearch-project/OpenSearch/pull/16628))

### Security

Expand Down
92 changes: 90 additions & 2 deletions rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,28 @@

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
index: test-iodvq
body:
query:
term:
ip_field: "192.168.0.1/24"

- match: { hits.total: 3 }

- do:
search:
rest_total_hits_as_int: true
index: test-iodvq
body:
query:
term:
ip_field: "192.168.0.1/31"

- match: { hits.total: 1 }

- do:
search:
rest_total_hits_as_int: true
Expand Down Expand Up @@ -987,6 +1009,28 @@

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
index: test-index
body:
query:
term:
ip_field: "192.168.0.1/24"

- match: { hits.total: 3 }

- do:
search:
rest_total_hits_as_int: true
index: test-index
body:
query:
term:
ip_field: "192.168.0.1/31"

- match: { hits.total: 1 }

- do:
search:
rest_total_hits_as_int: true
Expand Down Expand Up @@ -1077,8 +1121,8 @@
"search on fields with only doc_values enabled":
- skip:
features: [ "headers" ]
version: " - 2.99.99"
reason: "searching with only doc_values was added in 3.0.0"
version: " - 2.18.99"
reason: "searching with only doc_values was finally added in 2.19.0"
- do:
indices.create:
index: test-doc-values
Expand Down Expand Up @@ -1372,6 +1416,28 @@

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
index: test-doc-values
body:
query:
terms:
ip_field: ["192.168.0.1", "192.168.0.2"]

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
index: test-doc-values
body:
query:
terms:
ip_field: ["192.168.0.1/31", "192.168.0.3"]

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
Expand Down Expand Up @@ -1516,6 +1582,28 @@

- match: { hits.total: 2 }

- do:
search:
rest_total_hits_as_int: true
index: test-doc-values
body:
query:
term:
ip_field: "192.168.0.1/31"

- match: { hits.total: 1 }

- do:
search:
rest_total_hits_as_int: true
index: test-doc-values
body:
query:
term:
ip_field: "192.168.0.1/24"

- match: { hits.total: 3 }

- do:
search:
rest_total_hits_as_int: true
Expand Down
86 changes: 46 additions & 40 deletions server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
import java.util.Map;
import java.util.function.BiFunction;
import java.util.function.Supplier;
import java.util.stream.Collectors;

/**
* A {@link FieldMapper} for ip addresses.
Expand Down Expand Up @@ -225,42 +226,37 @@ protected Object parseSourceValue(Object value) {
@Override
public Query termQuery(Object value, @Nullable QueryShardContext context) {
failIfNotIndexedAndNoDocValues();
Query query;
final PointRangeQuery pointQuery;
if (value instanceof InetAddress) {
query = InetAddressPoint.newExactQuery(name(), (InetAddress) value);
pointQuery = (PointRangeQuery) InetAddressPoint.newExactQuery(name(), (InetAddress) value);
} else {
if (value instanceof BytesRef) {
value = ((BytesRef) value).utf8ToString();
}
String term = value.toString();
if (term.contains("/")) {
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
query = InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
pointQuery = (PointRangeQuery) InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
} else {
InetAddress address = InetAddresses.forString(term);
query = InetAddressPoint.newExactQuery(name(), address);
pointQuery = (PointRangeQuery) InetAddressPoint.newExactQuery(name(), address);
}
}
if (isSearchable() && hasDocValues()) {
String term = value.toString();
if (term.contains("/")) {
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
}
return new IndexOrDocValuesQuery(
query,
SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint()))
Query dvQuery = null;
if (hasDocValues()) {
dvQuery = SortedSetDocValuesField.newSlowRangeQuery(
name(),
new BytesRef(pointQuery.getLowerPoint()),
new BytesRef(pointQuery.getUpperPoint()),
true,
true
);
}
if (hasDocValues()) {
String term = value.toString();
if (term.contains("/")) {
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
}
return SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint()));
if (isSearchable() && hasDocValues()) {
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
} else {
return isSearchable() ? pointQuery : dvQuery;
}
return query;
}

@Override
Expand All @@ -285,36 +281,46 @@ public Query termsQuery(List<?> values, QueryShardContext context) {
}
addresses[i++] = address;
}
return InetAddressPoint.newSetQuery(name(), addresses);
Query dvQuery = null;
if (hasDocValues()) {
List<BytesRef> bytesRefs = Arrays.stream(addresses)
.distinct()
.map(InetAddressPoint::encode)
.map(BytesRef::new)
.collect(Collectors.<BytesRef>toList());
dvQuery = SortedSetDocValuesField.newSlowSetQuery(name(), bytesRefs);
}
Query pointQuery = null;
if (isSearchable()) {
pointQuery = InetAddressPoint.newSetQuery(name(), addresses);
}
if (isSearchable() && hasDocValues()) {
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
} else {
return isSearchable() ? pointQuery : dvQuery;
}
}

@Override
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
failIfNotIndexedAndNoDocValues();
return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (lower, upper) -> {
Query query = InetAddressPoint.newRangeQuery(name(), lower, upper);
if (isSearchable() && hasDocValues()) {
return new IndexOrDocValuesQuery(
query,
SortedSetDocValuesField.newSlowRangeQuery(
((PointRangeQuery) query).getField(),
new BytesRef(((PointRangeQuery) query).getLowerPoint()),
new BytesRef(((PointRangeQuery) query).getUpperPoint()),
true,
true
)
);
}
PointRangeQuery pointQuery = (PointRangeQuery) InetAddressPoint.newRangeQuery(name(), lower, upper);
Query dvQuery = null;
if (hasDocValues()) {
return SortedSetDocValuesField.newSlowRangeQuery(
((PointRangeQuery) query).getField(),
new BytesRef(((PointRangeQuery) query).getLowerPoint()),
new BytesRef(((PointRangeQuery) query).getUpperPoint()),
dvQuery = SortedSetDocValuesField.newSlowRangeQuery(
pointQuery.getField(),
new BytesRef(pointQuery.getLowerPoint()),
new BytesRef(pointQuery.getUpperPoint()),
true,
true
);
}
return query;
if (isSearchable() && hasDocValues()) {
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
} else {
return isSearchable() ? pointQuery : dvQuery;
}
});
}

Expand Down
Loading

0 comments on commit d4d70d8

Please sign in to comment.