Skip to content

Commit

Permalink
clean up
Browse files Browse the repository at this point in the history 
Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>
  • Loading branch information
@bowenlan-amzn
bowenlan-amzn committed Feb 9, 2024
1 parent 70b9aad commit 096d7fa
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 64 deletions.
25 changes: 5 additions & 20 deletions ...c/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterRewriteIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
package org.opensearch.search.aggregations.bucket;

import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;

import org.opensearch.action.index.IndexRequestBuilder;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.common.settings.Settings;
Expand Down Expand Up @@ -59,17 +60,13 @@ public static Collection<Object[]> parameters() {
protected void setupSuiteScopeCluster() throws Exception {
assertAcked(client().admin().indices().prepareCreate("idx").get());

// one segment has some docs
// the other segment has some other docs
// check the results are correct
final int segmentCount = randomIntBetween(2, 10);
final Set<Long> longTerms = new HashSet();

final Map<String, Integer> dateTerms = new HashMap<>();
for (int i = 0; i < segmentCount; i++) {
final List<IndexRequestBuilder> indexRequests = new ArrayList<>();


long longTerm;
do {
longTerm = randomInt(segmentCount * 2);
Expand All @@ -81,39 +78,27 @@ protected void setupSuiteScopeCluster() throws Exception {
for (int j = 0; j < frequency; j++) {
indexRequests.add(
client().prepareIndex("idx")
.setSource(
jsonBuilder().startObject()
.field("date", dateTerm)
.field("match", true)
.endObject()
)
.setSource(jsonBuilder().startObject().field("date", dateTerm).field("match", true).endObject())
);
}
expected.put(dateTerm+"T00:00:00.000Z", (long) frequency);
expected.put(dateTerm + "T00:00:00.000Z", (long) frequency);

indexRandom(true, false, indexRequests);
}

logger.info("expected results dateTerms={}", dateTerms);

ensureSearchable();
}

public void testMinDocCountOnDateHistogram() throws Exception {
final SearchResponse allResponse = client().prepareSearch("idx")
.setSize(0)
.setQuery(QUERY)
.addAggregation(
dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).minDocCount(0)
)
.addAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).minDocCount(0))
.get();

final Histogram allHisto = allResponse.getAggregations().get("histo");
logger.info("allHisto={}", allHisto);
Map<String, Long> results = new HashMap<>();
allHisto.getBuckets().forEach(bucket ->
results.put(bucket.getKeyAsString(), bucket.getDocCount())
);
allHisto.getBuckets().forEach(bucket -> results.put(bucket.getKeyAsString(), bucket.getDocCount()));

for (Map.Entry<String, Long> entry : expected.entrySet()) {
assertEquals(entry.getValue(), results.get(entry.getKey()));
Expand Down
27 changes: 1 addition & 26 deletions ...src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/MinDocCountIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,19 +34,11 @@

import com.carrotsearch.randomizedtesting.generators.RandomStrings;

import org.opensearch.action.admin.indices.segments.IndexShardSegments;
import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse;
import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest;
import org.opensearch.action.admin.indices.segments.ShardSegments;
import org.opensearch.action.index.IndexRequestBuilder;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.time.DateFormatter;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.engine.Segment;
import org.opensearch.index.fielddata.ScriptDocValues;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
Expand Down Expand Up @@ -135,7 +127,6 @@ public void setupSuiteScopeCluster() throws Exception {
final List<IndexRequestBuilder> indexRequests = new ArrayList<>();
final Set<String> stringTerms = new HashSet<>();
final Set<Long> longTerms = new HashSet();
final HashMap<String, Integer> dateTerms = new HashMap<>();
for (int i = 0; i < cardinality; ++i) {
String stringTerm;
do {
Expand All @@ -151,7 +142,6 @@ public void setupSuiteScopeCluster() throws Exception {
String dateTerm = DateFormatter.forPattern("yyyy-MM-dd").format(time);
final int frequency = randomBoolean() ? 1 : randomIntBetween(2, 20);
for (int j = 0; j < frequency; ++j) {
final boolean match = randomBoolean();
indexRequests.add(
client().prepareIndex("idx")
.setSource(
Expand All @@ -160,18 +150,14 @@ public void setupSuiteScopeCluster() throws Exception {
.field("l", longTerm)
.field("d", doubleTerm)
.field("date", dateTerm)
.field("match", match)
.field("match", randomBoolean())
.endObject()
)
);
if (match) {
dateTerms.put(dateTerm, dateTerms.getOrDefault(dateTerm, 0) + 1);
}
}
}
cardinality = stringTerms.size();

logger.info("date terms {}", dateTerms);
indexRandom(true, indexRequests);
ensureSearchable();
}
Expand Down Expand Up @@ -441,14 +427,6 @@ private void testMinDocCountOnHistogram(BucketOrder order) throws Exception {
}

private void testMinDocCountOnDateHistogram(BucketOrder order) throws Exception {

IndicesSegmentsRequest segmentReq = new IndicesSegmentsRequest("idx");
IndicesSegmentResponse segmentRes = client().admin().indices().segments(segmentReq).get();
XContentBuilder builder = XContentFactory.jsonBuilder();
segmentRes.toXContent(builder, ToXContent.EMPTY_PARAMS);
String jsonString = builder.toString();
logger.info("segmentRes={}", jsonString);

final SearchResponse allResponse = client().prepareSearch("idx")
.setSize(0)
.setQuery(QUERY)
Expand All @@ -458,7 +436,6 @@ private void testMinDocCountOnDateHistogram(BucketOrder order) throws Exception
.get();

final Histogram allHisto = allResponse.getAggregations().get("histo");
logger.info("allHisto={}", allHisto);

for (long minDocCount = 0; minDocCount < 50; ++minDocCount) {
final SearchResponse response = client().prepareSearch("idx")
Expand All @@ -471,8 +448,6 @@ private void testMinDocCountOnDateHistogram(BucketOrder order) throws Exception
.minDocCount(minDocCount)
)
.get();
final Histogram minDocHisto = response.getAggregations().get("histo");
logger.info("minDocCount={},minDocHisto={}", minDocCount, minDocHisto);
assertSubset(allHisto, response.getAggregations().get("histo"), minDocCount);
}
}
Expand Down
19 changes: 1 addition & 18 deletions server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,7 +340,6 @@ public Weight[] buildFastFilter(LeafReaderContext leaf, SearchContext context) t

private Weight[] buildFastFilter(SearchContext context, long[] bounds) throws IOException {
bounds = processHardBounds(bounds);
logger.debug("Bounds are {} for shard {} with hard bound", bounds, context.indexShard().shardId());
if (bounds == null) {
return null;
}
Expand Down Expand Up @@ -424,11 +423,6 @@ public static boolean tryFastFilterAggregation(
if (!fastFilterContext.rewriteable) {
return false;
}
logger.debug(
"try fast filter on Shard {} segment {}",
fastFilterContext.context.indexShard().shardId(),
ctx.ord
);

NumericDocValues docCountValues = DocValues.getNumeric(ctx.reader(), DocCountFieldMapper.NAME);
if (docCountValues.nextDoc() != NO_MORE_DOCS) {
Expand All @@ -447,11 +441,6 @@ public static boolean tryFastFilterAggregation(
}
Weight[] filters = fastFilterContext.filters;
boolean filtersBuiltAtSegmentLevel = false;
logger.debug(
"Shard {} segment {} functionally match all documents",
fastFilterContext.context.indexShard().shardId(),
ctx.ord
);
if (filters == null) {
logger.debug(
"Shard {} segment {} functionally match all documents. Build the fast filter",
Expand Down Expand Up @@ -498,7 +487,7 @@ public static boolean tryFastFilterAggregation(
}
}

// each segment computes its own filters, so reset
// each segment computes its own filters, so reset the filters built at segment level
if (filtersBuiltAtSegmentLevel) {
fastFilterContext.filters = null;
}
Expand All @@ -509,12 +498,6 @@ public static boolean tryFastFilterAggregation(

private static boolean segmentMatchAll(SearchContext ctx, LeafReaderContext leafCtx) throws IOException {
Weight weight = ctx.searcher().createWeight(ctx.query(), ScoreMode.COMPLETE_NO_SCORES, 1f);
if (weight == null) {
return false;
}
int count = weight.count(leafCtx);
int numDocs = leafCtx.reader().numDocs();
logger.debug("Shard {} segment {} has {} count and {} num docs", ctx.indexShard().shardId(), leafCtx.ord, count, numDocs);
return weight != null && weight.count(leafCtx) == leafCtx.reader().numDocs();
}
}

0 comments on commit 096d7fa

Please sign in to comment.