From 50d9ce2689758bbef5cf889d1893a57511314c7e Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 24 Jan 2022 09:20:44 +0100 Subject: [PATCH 01/19] New GeoHexGrid aggregation This commit introduces a new geogrid aggregation called GeoHexGridAggregation that is based in Uber h3 grid. It only supports geo_point fields. --- .../bucket/geohexgrid-aggregation.asciidoc | 245 ++++++++++++++++++ .../bucket/geogrid/InternalGeoGridBucket.java | 2 +- .../bucket/geogrid/ParsedGeoGrid.java | 2 +- .../spatial/action/SpatialStatsAction.java | 3 +- x-pack/plugin/spatial/build.gradle | 1 + .../xpack/spatial/SpatialPlugin.java | 60 ++++- .../bucket/geogrid/GeoHexCellIdSource.java | 133 ++++++++++ .../geogrid/GeoHexGridAggregationBuilder.java | 123 +++++++++ .../bucket/geogrid/GeoHexGridAggregator.java | 62 +++++ .../geogrid/GeoHexGridAggregatorFactory.java | 81 ++++++ .../bucket/geogrid/InternalGeoHexGrid.java | 67 +++++ .../geogrid/InternalGeoHexGridBucket.java | 42 +++ .../bucket/geogrid/ParsedGeoHexGrid.java | 34 +++ .../geogrid/ParsedGeoHexGridBucket.java | 34 +++ .../xpack/spatial/SpatialPluginTests.java | 41 +++ .../GeoHexAggregationBuilderTests.java | 65 +++++ .../bucket/geogrid/GeoHexAggregatorTests.java | 87 +++++++ .../bucket/geogrid/GeoHexGridTests.java | 67 +++++ .../test/spatial/80_geohex_grid.yml | 143 ++++++++++ 19 files changed, 1288 insertions(+), 4 deletions(-) create mode 100644 docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexCellIdSource.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregator.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregatorFactory.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/InternalGeoHexGrid.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/InternalGeoHexGridBucket.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ParsedGeoHexGrid.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ParsedGeoHexGridBucket.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregationBuilderTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregatorTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridTests.java create mode 100644 x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/80_geohex_grid.yml diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc new file mode 100644 index 0000000000000..d78b45a7f38d9 --- /dev/null +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -0,0 +1,245 @@ +[role="xpack"] +[[search-aggregations-bucket-geohexgrid-aggregation]] +=== Geohex grid aggregation +++++ +Geohex grid +++++ + +A multi-bucket aggregation that groups <> +values into buckets that represent a grid. +The resulting grid can be sparse and only +contains cells that have matching data. Each cell corresponds to a +https://h3geo.org/docs/core-library/h3Indexing#h3-cell-indexp[H3 cell index] a +and labeled using the https://h3geo.org/docs/core-library/h3Indexing#h3index-representation +[H3Index representation]. + +* High precision keys have a larger range for x and y, and represent tiles that +cover only a small area. +* Low precision keys have a smaller range for x and y, and represent tiles that +each cover a large area. + +See https://h3geo.org/docs/core-library/restable[the table od cell areas for H3] +resolutions on how precision (zoom) correlates to size on the ground. +Precision for this aggregation can be between 0 and 15, inclusive. + +WARNING: The highest-precision geohex for precision 15 produces cells that cover +less than a 10cm by 10cm of land and so high-precision requests can be very +costly in terms of RAM and result sizes. Please see the example below on how +to first filter the aggregation to a smaller geographic area before requesting +high-levels of detail. + +You can only use `geohex_grid` to aggregate an explicitly mapped `geo_point`. +If the `geo_point` field contains an array, `geotile_grid` aggregates all the array values. + + +==== Simple low-precision request + +[source,console,id=geotilegrid-aggregation-example] +-------------------------------------------------- +PUT /museums +{ + "mappings": { + "properties": { + "location": { + "type": "geo_point" + } + } + } +} + +POST /museums/_bulk?refresh +{"index":{"_id":1}} +{"location": "52.374081,4.912350", "name": "NEMO Science Museum"} +{"index":{"_id":2}} +{"location": "52.369219,4.901618", "name": "Museum Het Rembrandthuis"} +{"index":{"_id":3}} +{"location": "52.371667,4.914722", "name": "Nederlands Scheepvaartmuseum"} +{"index":{"_id":4}} +{"location": "51.222900,4.405200", "name": "Letterenhuis"} +{"index":{"_id":5}} +{"location": "48.861111,2.336389", "name": "Musée du Louvre"} +{"index":{"_id":6}} +{"location": "48.860000,2.327000", "name": "Musée d'Orsay"} + +POST /museums/_search?size=0 +{ + "aggregations": { + "large-grid": { + "geohex_grid": { + "field": "location", + "precision": 8 + } + } + } +} +-------------------------------------------------- + +Response: + +[source,console-result] +-------------------------------------------------- +{ + ... + "aggregations": { + "large-grid": { + "buckets": [ + { + "key": "8/131/84", + "doc_count": 3 + }, + { + "key": "8/129/88", + "doc_count": 2 + }, + { + "key": "8/131/85", + "doc_count": 1 + } + ] + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/] + +==== High-precision requests + +When requesting detailed buckets (typically for displaying a "zoomed in" map) +a filter like <> should be +applied to narrow the subject area otherwise potentially millions of buckets +will be created and returned. + +[source,console] +-------------------------------------------------- +POST /museums/_search?size=0 +{ + "aggregations": { + "zoomed-in": { + "filter": { + "geo_bounding_box": { + "location": { + "top_left": "52.4, 4.9", + "bottom_right": "52.3, 5.0" + } + } + }, + "aggregations": { + "zoom1": { + "geohex_grid": { + "field": "location", + "precision": 12 + } + } + } + } + } +} +-------------------------------------------------- +// TEST[continued] + +[source,console-result] +-------------------------------------------------- +{ + ... + "aggregations": { + "zoomed-in": { + "doc_count": 3, + "zoom1": { + "buckets": [ + { + "key": "22/2154412/1378379", + "doc_count": 1 + }, + { + "key": "22/2154385/1378332", + "doc_count": 1 + }, + { + "key": "22/2154259/1378425", + "doc_count": 1 + } + ] + } + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/] + +==== Requests with additional bounding box filtering + +The `geohex_grid` aggregation supports an optional `bounds` parameter +that restricts the cells considered to those that intersects the +bounds provided. The `bounds` parameter accepts the bounding box in +all the same <> of the +bounds specified in the Geo Bounding Box Query. This bounding box can be used with or +without an additional `geo_bounding_box` query for filtering the points prior to aggregating. +It is an independent bounding box that can intersect with, be equal to, or be disjoint +to any additional `geo_bounding_box` queries defined in the context of the aggregation. + +[source,console,id=geotilegrid-aggregation-with-bounds] +-------------------------------------------------- +POST /museums/_search?size=0 +{ + "aggregations": { + "tiles-in-bounds": { + "geohex_grid": { + "field": "location", + "precision": 12, + "bounds": { + "top_left": "52.4, 4.9", + "bottom_right": "52.3, 5.0" + } + } + } + } +} +-------------------------------------------------- +// TEST[continued] + +[source,console-result] +-------------------------------------------------- +{ + ... + "aggregations": { + "tiles-in-bounds": { + "buckets": [ + { + "key": "22/2154412/1378379", + "doc_count": 1 + }, + { + "key": "22/2154385/1378332", + "doc_count": 1 + }, + { + "key": "22/2154259/1378425", + "doc_count": 1 + } + ] + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/] + +==== Options + +[horizontal] +field:: Mandatory. The name of the field indexed with GeoPoints. + +precision:: Optional. The integer zoom of the key used to define + cells/buckets in the results. Defaults to 6. + Values outside of [0,15] will be rejected. + +bounds:: Optional. The bounding box to filter the points in the bucket. + +size:: Optional. The maximum number of geohash buckets to return + (defaults to 10,000). When results are trimmed, buckets are + prioritised based on the volumes of documents they contain. + +shard_size:: Optional. To allow for more accurate counting of the top cells + returned in the final result the aggregation defaults to + returning `max(10,(size x number-of-shards))` buckets from each + shard. If this heuristic is undesirable, the number considered + from each shard can be over-ridden using this parameter. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java index 9c8ac145fca47..126528ef533fc 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java @@ -51,7 +51,7 @@ public void writeTo(StreamOutput out) throws IOException { aggregations.writeTo(out); } - protected long hashAsLong() { + public long hashAsLong() { return hashAsLong; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java index 21a0249c485e2..c7a0f5b184a92 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java @@ -34,7 +34,7 @@ public static ObjectParser createParser( return parser; } - protected void setName(String name) { + public void setName(String name) { super.setName(name); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/action/SpatialStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/action/SpatialStatsAction.java index 4e51f35838cad..cd934c2cd5982 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/action/SpatialStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/action/SpatialStatsAction.java @@ -39,7 +39,8 @@ private SpatialStatsAction() { * Items to track. Serialized by ordinals. Append only, don't remove or change order of items in this list. */ public enum Item { - GEOLINE + GEOLINE, + GEOHEX } public static class Request extends BaseNodesRequest implements ToXContentObject { diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index 5c10a7181de2a..7930141230015 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -14,6 +14,7 @@ dependencies { compileOnly project(path: ':modules:legacy-geo') compileOnly project(':modules:lang-painless:spi') compileOnly project(path: xpackModule('core')) + api project(":libs:elasticsearch-h3") testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(path: xpackModule('vector-tile')) } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index 064e43e2b9e90..8c2c260987e65 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -31,6 +31,8 @@ import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregator; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.xcontent.ContextParser; import org.elasticsearch.xpack.core.XPackPlugin; @@ -50,9 +52,13 @@ import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoHashGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoTileGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoGridTiler; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexCellIdSource; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexGridAggregationBuilder; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexGridAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeCellIdSource; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeHashGridAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeTileGridAggregator; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.InternalGeoHexGrid; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.UnboundedGeoHashGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.UnboundedGeoTileGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeBoundsAggregator; @@ -87,6 +93,12 @@ public class SpatialPlugin extends Plugin implements ActionPlugin, MapperPlugin, License.OperationMode.GOLD ); + private final LicensedFeature.Momentary GEO_HEX_AGG_FEATURE = LicensedFeature.momentary( + "spatial", + "geo-hex-agg", + License.OperationMode.GOLD + ); + // to be overriden by tests protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); @@ -139,7 +151,12 @@ public List getAggregations() { GeoLineAggregationBuilder.NAME, GeoLineAggregationBuilder::new, usage.track(SpatialStatsAction.Item.GEOLINE, checkLicense(GeoLineAggregationBuilder.PARSER, GEO_LINE_AGG_FEATURE)) - ).addResultReader(InternalGeoLine::new).setAggregatorRegistrar(GeoLineAggregationBuilder::registerUsage) + ).addResultReader(InternalGeoLine::new).setAggregatorRegistrar(GeoLineAggregationBuilder::registerUsage), + new AggregationSpec( + GeoHexGridAggregationBuilder.NAME, + GeoHexGridAggregationBuilder::new, + usage.track(SpatialStatsAction.Item.GEOHEX, checkLicense(GeoHexGridAggregationBuilder.PARSER, GEO_HEX_AGG_FEATURE)) + ).addResultReader(InternalGeoHexGrid::new).setAggregatorRegistrar(this::registerGeoHexGridAggregator) ); } @@ -171,6 +188,47 @@ private void registerGeoShapeCentroidAggregator(ValuesSourceRegistry.Builder bui ); } + private void registerGeoHexGridAggregator(ValuesSourceRegistry.Builder builder) { + builder.register( + GeoHexGridAggregationBuilder.REGISTRY_KEY, + CoreValuesSourceType.GEOPOINT, + ( + name, + factories, + valuesSource, + precision, + geoBoundingBox, + requiredSize, + shardSize, + aggregationContext, + parent, + cardinality, + metadata) -> { + if (GEO_HEX_AGG_FEATURE.check(getLicenseState())) { + GeoHexCellIdSource cellIdSource = new GeoHexCellIdSource( + (ValuesSource.GeoPoint) valuesSource, + precision, + geoBoundingBox + ); + return new GeoHexGridAggregator( + name, + factories, + cellIdSource, + requiredSize, + shardSize, + aggregationContext, + parent, + cardinality, + metadata + ); + } + + throw LicenseUtils.newComplianceException("geohex_grid aggregation on geo_point fields"); + }, + true + ); + } + private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builder) { builder.register( GeoHashGridAggregationBuilder.REGISTRY_KEY, diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexCellIdSource.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexCellIdSource.java new file mode 100644 index 0000000000000..be2589a007707 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexCellIdSource.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.h3.CellBoundary; +import org.elasticsearch.h3.H3; +import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.search.aggregations.bucket.geogrid.CellValues; +import org.elasticsearch.search.aggregations.support.ValuesSource; + +/** + * Class to help convert {@link MultiGeoPointValues} + * to GeoHex bucketing. + */ +public class GeoHexCellIdSource extends ValuesSource.Numeric { + private final GeoPoint valuesSource; + private final int precision; + private final GeoBoundingBox geoBoundingBox; + + public GeoHexCellIdSource(GeoPoint valuesSource, int precision, GeoBoundingBox geoBoundingBox) { + this.valuesSource = valuesSource; + this.precision = precision; + this.geoBoundingBox = geoBoundingBox; + } + + public int precision() { + return precision; + } + + @Override + public boolean isFloatingPoint() { + return false; + } + + @Override + public SortedNumericDocValues longValues(LeafReaderContext ctx) { + return geoBoundingBox.isUnbounded() + ? new UnboundedCellValues(valuesSource.geoPointValues(ctx), precision) + : new BoundedCellValues(valuesSource.geoPointValues(ctx), precision, geoBoundingBox); + } + + @Override + public SortedNumericDoubleValues doubleValues(LeafReaderContext ctx) { + throw new UnsupportedOperationException(); + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext ctx) { + throw new UnsupportedOperationException(); + } + + private static class UnboundedCellValues extends CellValues { + + UnboundedCellValues(MultiGeoPointValues geoValues, int precision) { + super(geoValues, precision); + } + + @Override + protected int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int valuesIdx) { + values[valuesIdx] = H3.geoToH3(target.getLat(), target.getLon(), precision); + return valuesIdx + 1; + } + } + + private static class BoundedCellValues extends CellValues { + + private final boolean crossesDateline; + private final GeoBoundingBox bbox; + + protected BoundedCellValues(MultiGeoPointValues geoValues, int precision, GeoBoundingBox bbox) { + super(geoValues, precision); + this.crossesDateline = bbox.right() < bbox.left(); + this.bbox = bbox; + } + + @Override + public int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int valuesIdx) { + final double lat = target.getLat(); + final double lon = target.getLon(); + final long hex = H3.geoToH3(lat, lon, precision); + // validPoint is a fast check, validHex is slow + if (validPoint(lat, lon) || validHex(hex)) { + values[valuesIdx] = hex; + return valuesIdx + 1; + } + return valuesIdx; + } + + private boolean validPoint(double lat, double lon) { + if (bbox.top() >= lat && bbox.bottom() <= lat) { + if (crossesDateline) { + return bbox.left() <= lon || bbox.right() >= lon; + } else { + return bbox.left() <= lon && bbox.right() >= lon; + } + } + return false; + } + + private boolean validHex(long hex) { + CellBoundary boundary = H3.h3ToGeoBoundary(hex); + double minLat = Double.POSITIVE_INFINITY; + double minLon = Double.POSITIVE_INFINITY; + double maxLat = Double.NEGATIVE_INFINITY; + double maxLon = Double.NEGATIVE_INFINITY; + for (int i = 0; i < boundary.numPoints(); i++) { + double boundaryLat = boundary.getLatLon(i).getLatDeg(); + double boundaryLon = boundary.getLatLon(i).getLonDeg(); + minLon = Math.min(minLon, boundaryLon); + maxLon = Math.max(maxLon, boundaryLon); + minLat = Math.min(minLat, boundaryLat); + maxLat = Math.max(maxLat, boundaryLat); + } + if (bbox.top() > minLat && bbox.bottom() < maxLat) { + if (crossesDateline) { + return bbox.left() < maxLon || bbox.right() > minLon; + } else { + return bbox.left() < maxLon && bbox.right() > minLon; + } + } + return false; + } + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java new file mode 100644 index 0000000000000..6f9d1a2f509cf --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregationBuilder.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.h3.H3; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.GeoGridAggregatorSupplier; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Map; + +public class GeoHexGridAggregationBuilder extends GeoGridAggregationBuilder { + public static final String NAME = "geohex_grid"; + private static final int DEFAULT_PRECISION = 5; + private static final int DEFAULT_MAX_NUM_CELLS = 10000; + public static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey<>( + NAME, + GeoGridAggregatorSupplier.class + ); + + public static final ObjectParser PARSER = createParser( + NAME, + GeoHexGridAggregationBuilder::parsePrecision, + GeoHexGridAggregationBuilder::new + ); + + static int parsePrecision(XContentParser parser) throws IOException, ElasticsearchParseException { + final Object node = parser.currentToken().equals(XContentParser.Token.VALUE_NUMBER) + ? Integer.valueOf(parser.intValue()) + : parser.text(); + return XContentMapValues.nodeIntegerValue(node); + } + + public GeoHexGridAggregationBuilder(String name) { + super(name); + precision(DEFAULT_PRECISION); + size(DEFAULT_MAX_NUM_CELLS); + shardSize = -1; + } + + public GeoHexGridAggregationBuilder(StreamInput in) throws IOException { + super(in); + } + + @Override + public GeoGridAggregationBuilder precision(int precision) { + if (precision < 0 || precision > H3.MAX_H3_RES) { + throw new IllegalArgumentException( + "Invalid geohex aggregation precision of " + precision + "" + ". Must be between 0 and " + H3.MAX_H3_RES + ); + } + this.precision = precision; + return this; + } + + @Override + protected ValuesSourceAggregatorFactory createFactory( + String name, + ValuesSourceConfig config, + int precision, + int requiredSize, + int shardSize, + GeoBoundingBox geoBoundingBox, + AggregationContext context, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metadata + ) throws IOException { + return new GeoHexGridAggregatorFactory( + name, + config, + precision, + requiredSize, + shardSize, + geoBoundingBox, + context, + parent, + subFactoriesBuilder, + metadata + ); + } + + private GeoHexGridAggregationBuilder( + GeoHexGridAggregationBuilder clone, + AggregatorFactories.Builder factoriesBuilder, + Map metadata + ) { + super(clone, factoriesBuilder, metadata); + } + + @Override + protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metadata) { + return new GeoHexGridAggregationBuilder(this, factoriesBuilder, metadata); + } + + @Override + public String getType() { + return NAME; + } + + @Override + protected ValuesSourceRegistry.RegistryKey getRegistryKey() { + return REGISTRY_KEY; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregator.java new file mode 100644 index 0000000000000..5f5239d1624a5 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregator.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.CardinalityUpperBound; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregator; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSource; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Aggregates data expressed as h3 longs (for efficiency's sake) + * but formats results as h3 strings. + */ +public class GeoHexGridAggregator extends GeoGridAggregator { + + public GeoHexGridAggregator( + String name, + AggregatorFactories factories, + ValuesSource.Numeric valuesSource, + int requiredSize, + int shardSize, + AggregationContext context, + Aggregator parent, + CardinalityUpperBound cardinality, + Map metadata + ) throws IOException { + super(name, factories, valuesSource, requiredSize, shardSize, context, parent, cardinality, metadata); + } + + @Override + protected InternalGeoHexGrid buildAggregation( + String name, + int requiredSize, + List buckets, + Map metadata + ) { + return new InternalGeoHexGrid(name, requiredSize, buckets, metadata); + } + + @Override + public InternalGeoHexGrid buildEmptyAggregation() { + return new InternalGeoHexGrid(name, requiredSize, Collections.emptyList(), metadata()); + } + + @Override + protected InternalGeoGridBucket newEmptyBucket() { + return new InternalGeoHexGridBucket(0, 0, null); + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregatorFactory.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregatorFactory.java new file mode 100644 index 0000000000000..4870948bc7e93 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridAggregatorFactory.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.CardinalityUpperBound; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.NonCollectingAggregator; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +public class GeoHexGridAggregatorFactory extends ValuesSourceAggregatorFactory { + + private final int precision; + private final int requiredSize; + private final int shardSize; + private final GeoBoundingBox geoBoundingBox; + + GeoHexGridAggregatorFactory( + String name, + ValuesSourceConfig config, + int precision, + int requiredSize, + int shardSize, + GeoBoundingBox geoBoundingBox, + AggregationContext context, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metadata + ) throws IOException { + super(name, config, context, parent, subFactoriesBuilder, metadata); + this.precision = precision; + this.requiredSize = requiredSize; + this.shardSize = shardSize; + this.geoBoundingBox = geoBoundingBox; + } + + @Override + protected Aggregator createUnmapped(Aggregator parent, Map metadata) throws IOException { + final InternalAggregation aggregation = new InternalGeoHexGrid(name, requiredSize, Collections.emptyList(), metadata); + return new NonCollectingAggregator(name, context, parent, factories, metadata) { + @Override + public InternalAggregation buildEmptyAggregation() { + return aggregation; + } + }; + } + + @Override + protected Aggregator doCreateInternal(Aggregator parent, CardinalityUpperBound cardinality, Map metadata) + throws IOException { + return context.getValuesSourceRegistry() + .getAggregator(GeoHexGridAggregationBuilder.REGISTRY_KEY, config) + .build( + name, + factories, + config.getValuesSource(), + precision, + geoBoundingBox, + requiredSize, + shardSize, + context, + parent, + cardinality, + metadata + ); + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/InternalGeoHexGrid.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/InternalGeoHexGrid.java new file mode 100644 index 0000000000000..07c5dc35c3e72 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/InternalGeoHexGrid.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Represents a grid of cells where each cell's location is determined by a h3 cell. + * All cells in a grid are of the same precision and held internally as a single long + * for efficiency's sake. + */ +public class InternalGeoHexGrid extends InternalGeoGrid { + + InternalGeoHexGrid(String name, int requiredSize, List buckets, Map metadata) { + super(name, requiredSize, buckets, metadata); + } + + public InternalGeoHexGrid(StreamInput in) throws IOException { + super(in); + } + + @Override + public InternalGeoGrid create(List buckets) { + return new InternalGeoHexGrid(name, requiredSize, buckets, metadata); + } + + @Override + public InternalGeoGridBucket createBucket(InternalAggregations aggregations, InternalGeoGridBucket prototype) { + return new InternalGeoHexGridBucket(prototype.hashAsLong(), prototype.getDocCount(), aggregations); + } + + @Override + protected InternalGeoGrid create( + String name, + int requiredSize, + List buckets, + Map metadata + ) { + return new InternalGeoHexGrid(name, requiredSize, buckets, metadata); + } + + @Override + protected InternalGeoHexGridBucket createBucket(long hashAsLong, long docCount, InternalAggregations aggregations) { + return new InternalGeoHexGridBucket(hashAsLong, docCount, aggregations); + } + + @Override + protected Reader getBucketReader() { + return InternalGeoHexGridBucket::new; + } + + @Override + public String getWriteableName() { + return GeoHexGridAggregationBuilder.NAME; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/InternalGeoHexGridBucket.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/InternalGeoHexGridBucket.java new file mode 100644 index 0000000000000..f98b8bfe47627 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/InternalGeoHexGridBucket.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.h3.H3; +import org.elasticsearch.h3.LatLng; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; + +import java.io.IOException; + +public class InternalGeoHexGridBucket extends InternalGeoGridBucket { + + InternalGeoHexGridBucket(long hashAsLong, long docCount, InternalAggregations aggregations) { + super(hashAsLong, docCount, aggregations); + } + + /** + * Read from a stream. + */ + public InternalGeoHexGridBucket(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getKeyAsString() { + return H3.h3ToString(hashAsLong); + } + + @Override + public GeoPoint getKey() { + LatLng latLng = H3.h3ToLatLng(hashAsLong); + return new GeoPoint(latLng.getLatDeg(), latLng.getLonDeg()); + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ParsedGeoHexGrid.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ParsedGeoHexGrid.java new file mode 100644 index 0000000000000..ae8c878391405 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ParsedGeoHexGrid.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoGrid; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +public class ParsedGeoHexGrid extends ParsedGeoGrid { + + private static final ObjectParser PARSER = createParser( + ParsedGeoHexGrid::new, + ParsedGeoHexGridBucket::fromXContent, + ParsedGeoHexGridBucket::fromXContent + ); + + public static ParsedGeoGrid fromXContent(XContentParser parser, String name) throws IOException { + ParsedGeoGrid aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + @Override + public String getType() { + return GeoHexGridAggregationBuilder.NAME; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ParsedGeoHexGridBucket.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ParsedGeoHexGridBucket.java new file mode 100644 index 0000000000000..1383e46dcd9e5 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ParsedGeoHexGridBucket.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.h3.H3; +import org.elasticsearch.h3.LatLng; +import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoGridBucket; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +public class ParsedGeoHexGridBucket extends ParsedGeoGridBucket { + + @Override + public GeoPoint getKey() { + LatLng latLng = H3.h3ToLatLng(hashAsString); + return new GeoPoint(latLng.getLatDeg(), latLng.getLonDeg()); + } + + @Override + public String getKeyAsString() { + return hashAsString; + } + + static ParsedGeoHexGridBucket fromXContent(XContentParser parser) throws IOException { + return parseXContent(parser, false, ParsedGeoHexGridBucket::new, (p, bucket) -> bucket.hashAsString = p.text()); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/SpatialPluginTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/SpatialPluginTests.java index 2ea9a4205ba5a..8ca7afd4b69d3 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/SpatialPluginTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/SpatialPluginTests.java @@ -10,15 +10,19 @@ import org.elasticsearch.license.License; import org.elasticsearch.license.TestUtils; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.GeoGridAggregatorSupplier; import org.elasticsearch.search.aggregations.metrics.MetricAggregatorSupplier; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHexGridAggregationBuilder; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; import java.util.Arrays; @@ -54,6 +58,43 @@ public void testGeoCentroidLicenseCheck() { } } + public void testGeoHexLicenseCheck() { + for (License.OperationMode operationMode : License.OperationMode.values()) { + SpatialPlugin plugin = getPluginWithOperationMode(operationMode); + ValuesSourceRegistry.Builder registryBuilder = new ValuesSourceRegistry.Builder(); + List specs = plugin.getAggregations(); + specs.forEach(c -> c.getAggregatorRegistrar().accept(registryBuilder)); + ValuesSourceRegistry registry = registryBuilder.build(); + GeoGridAggregatorSupplier hexSupplier = registry.getAggregator( + GeoHexGridAggregationBuilder.REGISTRY_KEY, + new ValuesSourceConfig(CoreValuesSourceType.GEOPOINT, null, true, null, null, null, null, null, null) + ); + if (License.OperationMode.TRIAL != operationMode + && License.OperationMode.compare(operationMode, License.OperationMode.GOLD) < 0) { + ElasticsearchSecurityException exception = expectThrows( + ElasticsearchSecurityException.class, + () -> hexSupplier.build( + null, + AggregatorFactories.EMPTY, + null, + 0, + null, + 0, + 0, + null, + null, + CardinalityUpperBound.NONE, + null + ) + ); + assertThat( + exception.getMessage(), + equalTo("current license is non-compliant for [geohex_grid aggregation on geo_point fields]") + ); + } + } + } + public void testGeoGridLicenseCheck() { for (ValuesSourceRegistry.RegistryKey registryKey : Arrays.asList( GeoHashGridAggregationBuilder.REGISTRY_KEY, diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregationBuilderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregationBuilderTests.java new file mode 100644 index 0000000000000..dbe960087d91d --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregationBuilderTests.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.h3.H3; +import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.spatial.util.GeoTestUtils; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class GeoHexAggregationBuilderTests extends AbstractSerializingTestCase { + + @Override + protected GeoHexGridAggregationBuilder doParseInstance(XContentParser parser) throws IOException { + assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); + assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); + String name = parser.currentName(); + assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); + assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); + assertThat(parser.currentName(), equalTo(GeoHexGridAggregationBuilder.NAME)); + GeoHexGridAggregationBuilder parsed = GeoHexGridAggregationBuilder.PARSER.apply(parser, name); + assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT)); + assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT)); + return parsed; + } + + @Override + protected Writeable.Reader instanceReader() { + return GeoHexGridAggregationBuilder::new; + } + + @Override + protected GeoHexGridAggregationBuilder createTestInstance() { + GeoHexGridAggregationBuilder geoHexGridAggregationBuilder = new GeoHexGridAggregationBuilder("_name"); + geoHexGridAggregationBuilder.field("field"); + if (randomBoolean()) { + geoHexGridAggregationBuilder.precision(randomIntBetween(0, H3.MAX_H3_RES)); + } + if (randomBoolean()) { + geoHexGridAggregationBuilder.size(randomIntBetween(0, 256 * 256)); + } + if (randomBoolean()) { + geoHexGridAggregationBuilder.shardSize(randomIntBetween(0, 256 * 256)); + } + if (randomBoolean()) { + geoHexGridAggregationBuilder.setGeoBoundingBox(GeoTestUtils.randomBBox()); + } + return geoHexGridAggregationBuilder; + } + + public void testInvalidPrecision() { + GeoHexGridAggregationBuilder geoHexGridAggregationBuilder = new GeoHexGridAggregationBuilder("_name"); + expectThrows(IllegalArgumentException.class, () -> geoHexGridAggregationBuilder.precision(16)); + expectThrows(IllegalArgumentException.class, () -> geoHexGridAggregationBuilder.precision(-1)); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregatorTests.java new file mode 100644 index 0000000000000..18ec429a0c3a1 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexAggregatorTests.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.h3.CellBoundary; +import org.elasticsearch.h3.H3; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregatorTestCase; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; +import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; +import org.elasticsearch.xpack.spatial.util.GeoTestUtils; + +import java.util.List; + +public class GeoHexAggregatorTests extends GeoGridAggregatorTestCase { + + @Override + protected List getSearchPlugins() { + return List.of(new LocalStateSpatialPlugin()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + return List.of(GeoShapeValuesSourceType.instance(), CoreValuesSourceType.GEOPOINT); + } + + @Override + protected int randomPrecision() { + return randomIntBetween(0, H3.MAX_H3_RES); + } + + @Override + protected String hashAsString(double lng, double lat, int precision) { + return H3.geoToH3Address(lat, lng, precision); + } + + @Override + protected GeoGridAggregationBuilder createBuilder(String name) { + return new GeoHexGridAggregationBuilder(name); + } + + @Override + protected Point randomPoint() { + return GeometryTestUtils.randomPoint(); + } + + @Override + protected GeoBoundingBox randomBBox() { + return GeoTestUtils.randomBBox(); + } + + @Override + protected Rectangle getTile(double lng, double lat, int precision) { + CellBoundary boundary = H3.h3ToGeoBoundary(hashAsString(lng, lat, precision)); + double minLat = Double.POSITIVE_INFINITY; + double minLon = Double.POSITIVE_INFINITY; + double maxLat = Double.NEGATIVE_INFINITY; + double maxLon = Double.NEGATIVE_INFINITY; + for (int i = 0; i < boundary.numPoints(); i++) { + double boundaryLat = boundary.getLatLon(i).getLatDeg(); + double boundaryLon = boundary.getLatLon(i).getLonDeg(); + minLon = Math.min(minLon, boundaryLon); + maxLon = Math.max(maxLon, boundaryLon); + minLat = Math.min(minLat, boundaryLat); + maxLat = Math.max(maxLat, boundaryLat); + } + return new Rectangle(minLon, maxLon, maxLat, minLat); + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return createBuilder("foo").field(fieldName); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridTests.java new file mode 100644 index 0000000000000..6a2de785c0544 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridTests.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.h3.H3; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridTestCase; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; + +import java.util.List; +import java.util.Map; + +public class GeoHexGridTests extends GeoGridTestCase { + + @Override + protected SearchPlugin registerPlugin() { + return new LocalStateSpatialPlugin(); + } + + @Override + protected List getNamedXContents() { + return CollectionUtils.appendToCopy( + super.getNamedXContents(), + new NamedXContentRegistry.Entry( + Aggregation.class, + new ParseField(GeoHexGridAggregationBuilder.NAME), + (p, c) -> ParsedGeoHexGrid.fromXContent(p, (String) c) + ) + ); + } + + @Override + protected InternalGeoHexGrid createInternalGeoGrid( + String name, + int size, + List buckets, + Map metadata + ) { + return new InternalGeoHexGrid(name, size, buckets, metadata); + } + + @Override + protected InternalGeoHexGridBucket createInternalGeoGridBucket(Long key, long docCount, InternalAggregations aggregations) { + return new InternalGeoHexGridBucket(key, docCount, aggregations); + } + + @Override + protected long longEncode(double lng, double lat, int precision) { + return H3.geoToH3(lat, lng, precision); + } + + @Override + protected int randomPrecision() { + // precision values below 8 can lead to parsing errors + return randomIntBetween(0, H3.MAX_H3_RES); + } +} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/80_geohex_grid.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/80_geohex_grid.yml new file mode 100644 index 0000000000000..a578e80ad4ddd --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/80_geohex_grid.yml @@ -0,0 +1,143 @@ +setup: + - do: + indices.create: + index: locations + body: + settings: + number_of_shards: 3 + mappings: + properties: + location: + type: geo_point + + - do: + bulk: + refresh: true + body: + - index: + _index: locations + _id: 1 + - '{"location": "POINT(4.912350 52.374081)", "city": "Amsterdam", "name": "NEMO Science Museum"}' + - index: + _index: locations + _id: 2 + - '{"location": "POINT(4.901618 52.369219)", "city": "Amsterdam", "name": "Museum Het Rembrandthuis"}' + - index: + _index: locations + _id: 3 + - '{"location": "POINT(4.914722 52.371667)", "city": "Amsterdam", "name": "Nederlands Scheepvaartmuseum"}' + - index: + _index: locations + _id: 4 + - '{"location": "POINT(4.405200 51.222900)", "city": "Antwerp", "name": "Letterenhuis"}' + - index: + _index: locations + _id: 5 + - '{"location": "POINT(2.336389 48.861111)", "city": "Paris", "name": "Musée du Louvre"}' + - index: + _index: locations + _id: 6 + - '{"location": "POINT(2.327000 48.860000)", "city": "Paris", "name": "Musée dOrsay"}' + - do: + indices.refresh: {} + +--- +"Test geohex_grid with defaults": + + - do: + search: + index: locations + size: 0 + body: + aggs: + grid: + geohex_grid: + field: location + - match: {hits.total.value: 6 } + - length: { aggregations.grid.buckets: 3 } + - match: { aggregations.grid.buckets.0.key: "85196953fffffff" } + - match: { aggregations.grid.buckets.0.doc_count: 3 } + - match: { aggregations.grid.buckets.1.key: "851fb467fffffff" } + - match: { aggregations.grid.buckets.1.doc_count: 2 } + - match: { aggregations.grid.buckets.2.key: "851fa4c7fffffff" } + - match: { aggregations.grid.buckets.2.doc_count: 1 } + +--- +"Test geohex_grid with precision": + + - do: + search: + index: locations + size: 0 + body: + aggs: + grid: + geohex_grid: + field: location + precision: 0 + - match: { hits.total.value: 6 } + - length: { aggregations.grid.buckets: 2 } + - match: { aggregations.grid.buckets.0.key: "801ffffffffffff" } + - match: { aggregations.grid.buckets.0.doc_count: 4 } + - match: { aggregations.grid.buckets.1.key: "8019fffffffffff" } + - match: { aggregations.grid.buckets.1.doc_count: 2 } + +--- +"Test geohex_grid with size": + + - do: + search: + index: locations + size: 0 + body: + aggs: + grid: + geohex_grid: + field: location + size: 1 + - match: {hits.total.value: 6 } + - length: { aggregations.grid.buckets: 1 } + - match: { aggregations.grid.buckets.0.key: "85196953fffffff" } + - match: { aggregations.grid.buckets.0.doc_count: 3 } + +--- +"Test geohex_grid with shard size": + + - do: + search: + index: locations + size: 0 + body: + aggs: + grid: + geohex_grid: + field: location + shard_size: 10 + - match: {hits.total.value: 6 } + - length: { aggregations.grid.buckets: 3 } + - match: { aggregations.grid.buckets.0.key: "85196953fffffff" } + - match: { aggregations.grid.buckets.0.doc_count: 3 } + - match: { aggregations.grid.buckets.1.key: "851fb467fffffff" } + - match: { aggregations.grid.buckets.1.doc_count: 2 } + - match: { aggregations.grid.buckets.2.key: "851fa4c7fffffff" } + - match: { aggregations.grid.buckets.2.doc_count: 1 } + +--- +"Test geohex_grid with bounds": + + - do: + search: + index: locations + size: 0 + body: + aggs: + grid: + geohex_grid: + field: location + bounds: + top_left: "52.4, 4.9" + bottom_right: "52.3, 5.0" + - match: {hits.total.value: 6 } + - length: { aggregations.grid.buckets: 1 } + - match: { aggregations.grid.buckets.0.key: "85196953fffffff" } + - match: { aggregations.grid.buckets.0.doc_count: 3 } From 16dc137a5d69f1272d669b1bae16b2795c1cc230 Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 24 Jan 2022 10:10:29 +0100 Subject: [PATCH 02/19] fix docs --- .../bucket/geohexgrid-aggregation.asciidoc | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index d78b45a7f38d9..3d2971b09ed8b 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -29,12 +29,12 @@ to first filter the aggregation to a smaller geographic area before requesting high-levels of detail. You can only use `geohex_grid` to aggregate an explicitly mapped `geo_point`. -If the `geo_point` field contains an array, `geotile_grid` aggregates all the array values. +If the `geo_point` field contains an array, `geohex_grid` aggregates all the array values. ==== Simple low-precision request -[source,console,id=geotilegrid-aggregation-example] +[source,console,id=geohexgrid-aggregation-example] -------------------------------------------------- PUT /museums { @@ -67,7 +67,7 @@ POST /museums/_search?size=0 "large-grid": { "geohex_grid": { "field": "location", - "precision": 8 + "precision": 4 } } } @@ -84,15 +84,15 @@ Response: "large-grid": { "buckets": [ { - "key": "8/131/84", + "key": "841969dffffffff", "doc_count": 3 }, { - "key": "8/129/88", + "key": "841fb47ffffffff", "doc_count": 2 }, { - "key": "8/131/85", + "key": "841fa4dffffffff", "doc_count": 1 } ] @@ -147,15 +147,15 @@ POST /museums/_search?size=0 "zoom1": { "buckets": [ { - "key": "22/2154412/1378379", + "key": "8c1969c9b2617ff", "doc_count": 1 }, { - "key": "22/2154385/1378332", + "key": "8c1969526d753ff", "doc_count": 1 }, { - "key": "22/2154259/1378425", + "key": "8c1969526d26dff", "doc_count": 1 } ] @@ -177,7 +177,7 @@ without an additional `geo_bounding_box` query for filtering the points prior to It is an independent bounding box that can intersect with, be equal to, or be disjoint to any additional `geo_bounding_box` queries defined in the context of the aggregation. -[source,console,id=geotilegrid-aggregation-with-bounds] +[source,console,id=geohexgrid-aggregation-with-bounds] -------------------------------------------------- POST /museums/_search?size=0 { @@ -205,15 +205,15 @@ POST /museums/_search?size=0 "tiles-in-bounds": { "buckets": [ { - "key": "22/2154412/1378379", + "key": "8c1969c9b2617ff", "doc_count": 1 }, { - "key": "22/2154385/1378332", + "key": "8c1969526d753ff", "doc_count": 1 }, { - "key": "22/2154259/1378425", + "key": "8c1969526d26dff", "doc_count": 1 } ] @@ -234,7 +234,7 @@ precision:: Optional. The integer zoom of the key used to define bounds:: Optional. The bounding box to filter the points in the bucket. -size:: Optional. The maximum number of geohash buckets to return +size:: Optional. The maximum number of geohex buckets to return (defaults to 10,000). When results are trimmed, buckets are prioritised based on the volumes of documents they contain. From 96bc189118dd1a52a94f3017a399bd805bef6631 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Mon, 24 Jan 2022 10:51:14 -0500 Subject: [PATCH 03/19] Add include so page publishes --- docs/reference/aggregations/bucket.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reference/aggregations/bucket.asciidoc b/docs/reference/aggregations/bucket.asciidoc index dfdaca18e6cfb..af0e338a4588f 100644 --- a/docs/reference/aggregations/bucket.asciidoc +++ b/docs/reference/aggregations/bucket.asciidoc @@ -40,6 +40,8 @@ include::bucket/geodistance-aggregation.asciidoc[] include::bucket/geohashgrid-aggregation.asciidoc[] +include::bucket/geohexgrid-aggregation.asciidoc[] + include::bucket/geotilegrid-aggregation.asciidoc[] include::bucket/global-aggregation.asciidoc[] From 2330f62b0efb9d3207d79bd3b9c259b8f3fe6830 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 24 Jan 2022 19:46:20 +0100 Subject: [PATCH 04/19] Update docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc Co-authored-by: James Rodewig --- .../aggregations/bucket/geohexgrid-aggregation.asciidoc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index 3d2971b09ed8b..a1ee885778409 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -9,9 +9,8 @@ A multi-bucket aggregation that groups <> values into buckets that represent a grid. The resulting grid can be sparse and only contains cells that have matching data. Each cell corresponds to a -https://h3geo.org/docs/core-library/h3Indexing#h3-cell-indexp[H3 cell index] a -and labeled using the https://h3geo.org/docs/core-library/h3Indexing#h3index-representation -[H3Index representation]. +https://h3geo.org/docs/core-library/h3Indexing#h3-cell-indexp[H3 cell index] and is +labeled using the https://h3geo.org/docs/core-library/h3Indexing#h3index-representation[H3Index representation]. * High precision keys have a larger range for x and y, and represent tiles that cover only a small area. From baa26dfbab507b50718f5636f679f8d571a1fd8e Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 24 Jan 2022 19:46:28 +0100 Subject: [PATCH 05/19] Update docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc Co-authored-by: James Rodewig --- .../aggregations/bucket/geohexgrid-aggregation.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index a1ee885778409..2321219985eae 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -17,8 +17,8 @@ cover only a small area. * Low precision keys have a smaller range for x and y, and represent tiles that each cover a large area. -See https://h3geo.org/docs/core-library/restable[the table od cell areas for H3] -resolutions on how precision (zoom) correlates to size on the ground. +See https://h3geo.org/docs/core-library/restable[the table of cell areas for H3 +resolutions] on how precision (zoom) correlates to size on the ground. Precision for this aggregation can be between 0 and 15, inclusive. WARNING: The highest-precision geohex for precision 15 produces cells that cover From cf529cfdae077c6e8a1fb815649d298c0acd7bf6 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 24 Jan 2022 19:46:56 +0100 Subject: [PATCH 06/19] Update docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc Co-authored-by: James Rodewig --- .../bucket/geohexgrid-aggregation.asciidoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index 2321219985eae..a1b1fe917767d 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -21,11 +21,11 @@ See https://h3geo.org/docs/core-library/restable[the table of cell areas for H3 resolutions] on how precision (zoom) correlates to size on the ground. Precision for this aggregation can be between 0 and 15, inclusive. -WARNING: The highest-precision geohex for precision 15 produces cells that cover -less than a 10cm by 10cm of land and so high-precision requests can be very -costly in terms of RAM and result sizes. Please see the example below on how -to first filter the aggregation to a smaller geographic area before requesting -high-levels of detail. +WARNING: High-precision requests can be very expensive in terms of RAM and +result sizes. For example, the highest-precision geohex with a precision of `15` +produces cells that cover less than 10cm by 10cm. We recommend you use a +filter to limit high-precision requests to a smaller geographic area. For an example, +refer to <>. You can only use `geohex_grid` to aggregate an explicitly mapped `geo_point`. If the `geo_point` field contains an array, `geohex_grid` aggregates all the array values. From 2f83ca9fccef0cfc1746d48bb3416c8abd68eab6 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 24 Jan 2022 19:47:10 +0100 Subject: [PATCH 07/19] Update docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc Co-authored-by: James Rodewig --- .../aggregations/bucket/geohexgrid-aggregation.asciidoc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index a1b1fe917767d..d3f416f120ede 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -231,7 +231,10 @@ precision:: Optional. The integer zoom of the key used to define cells/buckets in the results. Defaults to 6. Values outside of [0,15] will be rejected. -bounds:: Optional. The bounding box to filter the points in the bucket. +bounds:: +(Optional, object) Bounding box used to filter the geo-points in each bucket. +Accepts the same bounding box formats as the +<>. size:: Optional. The maximum number of geohex buckets to return (defaults to 10,000). When results are trimmed, buckets are From af890d2e3502d2f1d9d319664970ffb8c584768b Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 24 Jan 2022 19:47:17 +0100 Subject: [PATCH 08/19] Update docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc Co-authored-by: James Rodewig --- .../aggregations/bucket/geohexgrid-aggregation.asciidoc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index d3f416f120ede..169095d3f522e 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -236,9 +236,10 @@ bounds:: Accepts the same bounding box formats as the <>. -size:: Optional. The maximum number of geohex buckets to return - (defaults to 10,000). When results are trimmed, buckets are - prioritised based on the volumes of documents they contain. +size:: +(Optional, integer) Maximum number of buckets to return. Defaults to 10,000. +When results are trimmed, buckets are prioritized based on the volume of +documents they contain. shard_size:: Optional. To allow for more accurate counting of the top cells returned in the final result the aggregation defaults to From 4aad240a6b76bf16332f77c6f917bb581cbcf0ad Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 24 Jan 2022 19:47:26 +0100 Subject: [PATCH 09/19] Update docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc Co-authored-by: James Rodewig --- .../aggregations/bucket/geohexgrid-aggregation.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index 169095d3f522e..94f8fed75c007 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -169,7 +169,7 @@ POST /museums/_search?size=0 The `geohex_grid` aggregation supports an optional `bounds` parameter that restricts the cells considered to those that intersects the -bounds provided. The `bounds` parameter accepts the bounding box in +provided bounds. The `bounds` parameter accepts the bounding box in all the same <> of the bounds specified in the Geo Bounding Box Query. This bounding box can be used with or without an additional `geo_bounding_box` query for filtering the points prior to aggregating. From 10356b2269f62f260c1d559cfdc54386dd135a60 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 24 Jan 2022 19:47:39 +0100 Subject: [PATCH 10/19] Update docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc Co-authored-by: James Rodewig --- .../aggregations/bucket/geohexgrid-aggregation.asciidoc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index 94f8fed75c007..85a4ea193a83d 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -241,8 +241,7 @@ size:: When results are trimmed, buckets are prioritized based on the volume of documents they contain. -shard_size:: Optional. To allow for more accurate counting of the top cells - returned in the final result the aggregation defaults to - returning `max(10,(size x number-of-shards))` buckets from each - shard. If this heuristic is undesirable, the number considered - from each shard can be over-ridden using this parameter. +shard_size:: +(Optional, integer) Number of buckets returned from each shard. Defaults to +`max(10,(size x number-of-shards))` to allow for more a accurate count of the +top cells in the final result. From 0e734afbd790166107277cc900fadc63c70e50f1 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 24 Jan 2022 19:47:54 +0100 Subject: [PATCH 11/19] Update docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc Co-authored-by: James Rodewig --- .../aggregations/bucket/geohexgrid-aggregation.asciidoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index 85a4ea193a83d..04616df89796f 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -27,8 +27,6 @@ produces cells that cover less than 10cm by 10cm. We recommend you use a filter to limit high-precision requests to a smaller geographic area. For an example, refer to <>. -You can only use `geohex_grid` to aggregate an explicitly mapped `geo_point`. -If the `geo_point` field contains an array, `geohex_grid` aggregates all the array values. ==== Simple low-precision request From 98e48cc585de245913ac8190527d0d3f46cd11bd Mon Sep 17 00:00:00 2001 From: iverase Date: Wed, 26 Jan 2022 13:29:03 +0100 Subject: [PATCH 12/19] iter in docs --- .../aggregations/bucket/geohexgrid-aggregation.asciidoc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index 04616df89796f..f848812cce38c 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -12,11 +12,6 @@ contains cells that have matching data. Each cell corresponds to a https://h3geo.org/docs/core-library/h3Indexing#h3-cell-indexp[H3 cell index] and is labeled using the https://h3geo.org/docs/core-library/h3Indexing#h3index-representation[H3Index representation]. -* High precision keys have a larger range for x and y, and represent tiles that -cover only a small area. -* Low precision keys have a smaller range for x and y, and represent tiles that -each cover a large area. - See https://h3geo.org/docs/core-library/restable[the table of cell areas for H3 resolutions] on how precision (zoom) correlates to size on the ground. Precision for this aggregation can be between 0 and 15, inclusive. From e04baf3e85afbb7296d38f25b17c2eb255e708e8 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 26 Jan 2022 13:29:42 +0100 Subject: [PATCH 13/19] Update docs/changelog/82924.yaml --- docs/changelog/82924.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/82924.yaml diff --git a/docs/changelog/82924.yaml b/docs/changelog/82924.yaml new file mode 100644 index 0000000000000..d53e2e6bf0ff0 --- /dev/null +++ b/docs/changelog/82924.yaml @@ -0,0 +1,5 @@ +pr: 82924 +summary: New `GeoHexGrid` aggregation +area: Geo +type: feature +issues: [] From 0af195d9c71843f93cd6b40d63f8f583e3c8f19c Mon Sep 17 00:00:00 2001 From: iverase Date: Wed, 26 Jan 2022 13:35:43 +0100 Subject: [PATCH 14/19] yaml editing --- docs/changelog/82924.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog/82924.yaml b/docs/changelog/82924.yaml index d53e2e6bf0ff0..9058173d71b34 100644 --- a/docs/changelog/82924.yaml +++ b/docs/changelog/82924.yaml @@ -2,4 +2,5 @@ pr: 82924 summary: New `GeoHexGrid` aggregation area: Geo type: feature -issues: [] +issues: + - 81579 From 5b23ee6f22229ef02a316c886ec923f7b0792293 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 26 Jan 2022 13:36:28 +0100 Subject: [PATCH 15/19] Update docs/changelog/82924.yaml --- docs/changelog/82924.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog/82924.yaml b/docs/changelog/82924.yaml index 9058173d71b34..d53e2e6bf0ff0 100644 --- a/docs/changelog/82924.yaml +++ b/docs/changelog/82924.yaml @@ -2,5 +2,4 @@ pr: 82924 summary: New `GeoHexGrid` aggregation area: Geo type: feature -issues: - - 81579 +issues: [] From 157c2a9151b092f1a2ffc7a09d5d2cf0859294be Mon Sep 17 00:00:00 2001 From: iverase Date: Wed, 26 Jan 2022 14:34:12 +0100 Subject: [PATCH 16/19] fix link --- .../aggregations/bucket/geohexgrid-aggregation.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index f848812cce38c..a226e82118a83 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -101,7 +101,7 @@ a filter like <> should be applied to narrow the subject area otherwise potentially millions of buckets will be created and returned. -[source,console] +[source,console,id=geohexgrid-high-precision-ex] -------------------------------------------------- POST /museums/_search?size=0 { From 1eebba32d0c35f7167b566e8011ce0608f800b6c Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 26 Jan 2022 09:04:27 -0500 Subject: [PATCH 17/19] Additional edits and fixes --- .../bucket/geohexgrid-aggregation.asciidoc | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index a226e82118a83..066818e2cd490 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -17,13 +17,12 @@ resolutions] on how precision (zoom) correlates to size on the ground. Precision for this aggregation can be between 0 and 15, inclusive. WARNING: High-precision requests can be very expensive in terms of RAM and -result sizes. For example, the highest-precision geohex with a precision of `15` +result sizes. For example, the highest-precision geohex with a precision of 15 produces cells that cover less than 10cm by 10cm. We recommend you use a filter to limit high-precision requests to a smaller geographic area. For an example, -refer to <>. - - +refer to <>. +[[geohexgrid-low-precision]] ==== Simple low-precision request [source,console,id=geohexgrid-aggregation-example] @@ -94,11 +93,12 @@ Response: -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/] +[[geohexgrid-high-precision]] ==== High-precision requests -When requesting detailed buckets (typically for displaying a "zoomed in" map) +When requesting detailed buckets (typically for displaying a "zoomed in" map), a filter like <> should be -applied to narrow the subject area otherwise potentially millions of buckets +applied to narrow the subject area. Otherwise, potentially millions of buckets will be created and returned. [source,console,id=geohexgrid-high-precision-ex] @@ -129,6 +129,8 @@ POST /museums/_search?size=0 -------------------------------------------------- // TEST[continued] +Response: + [source,console-result] -------------------------------------------------- { @@ -158,13 +160,14 @@ POST /museums/_search?size=0 -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/] +[[geohexgrid-addtl-bounding-box-filtering]] ==== Requests with additional bounding box filtering The `geohex_grid` aggregation supports an optional `bounds` parameter -that restricts the cells considered to those that intersects the -provided bounds. The `bounds` parameter accepts the bounding box in -all the same <> of the -bounds specified in the Geo Bounding Box Query. This bounding box can be used with or +that restricts the cells considered to those that intersect the +provided bounds. The `bounds` parameter accepts the same +<> +as the geo-bounding box query. This bounding box can be used with or without an additional `geo_bounding_box` query for filtering the points prior to aggregating. It is an independent bounding box that can intersect with, be equal to, or be disjoint to any additional `geo_bounding_box` queries defined in the context of the aggregation. @@ -189,6 +192,8 @@ POST /museums/_search?size=0 -------------------------------------------------- // TEST[continued] +Response: + [source,console-result] -------------------------------------------------- { @@ -215,14 +220,18 @@ POST /museums/_search?size=0 -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"_shards": $body._shards,"hits":$body.hits,"timed_out":false,/] +[[geohexgrid-options]] ==== Options [horizontal] -field:: Mandatory. The name of the field indexed with GeoPoints. - -precision:: Optional. The integer zoom of the key used to define - cells/buckets in the results. Defaults to 6. - Values outside of [0,15] will be rejected. +field:: +(Required, string) Field containing indexed geo-point values. This field must be +explicitly mapped as a <> field. If the field contains an +array, `geohex_grid` aggregates all array values. + +precision:: +(Optional, integer) Integer zoom of the key used to define cells/buckets in +the results. Defaults to `6`. Values outside of [`0`,`15`] will be rejected. bounds:: (Optional, object) Bounding box used to filter the geo-points in each bucket. From 73f09bbbaf81f1ff4160f7e7bcbb2337543116c4 Mon Sep 17 00:00:00 2001 From: James Rodewig <40268737+jrodewig@users.noreply.github.com> Date: Wed, 26 Jan 2022 09:10:04 -0500 Subject: [PATCH 18/19] minor edit --- .../aggregations/bucket/geohexgrid-aggregation.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc index 066818e2cd490..528b321cbd90b 100644 --- a/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohexgrid-aggregation.asciidoc @@ -225,9 +225,9 @@ Response: [horizontal] field:: -(Required, string) Field containing indexed geo-point values. This field must be -explicitly mapped as a <> field. If the field contains an -array, `geohex_grid` aggregates all array values. +(Required, string) Field containing indexed geo-point values. Must be explicitly +mapped as a <> field. If the field contains an array, +`geohex_grid` aggregates all array values. precision:: (Optional, integer) Integer zoom of the key used to define cells/buckets in From c67e4bbf372b3774b7c6edb01595ff10d9470203 Mon Sep 17 00:00:00 2001 From: iverase Date: Wed, 26 Jan 2022 16:04:27 +0100 Subject: [PATCH 19/19] fix test --- .../bucket/geogrid/GeoGridTestCase.java | 14 ++++++++++---- .../bucket/geogrid/GeoHexGridTests.java | 1 - 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java index 20bd74356f7e2..aaddf51eb1735 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java @@ -55,16 +55,22 @@ protected int maxNumberOfBuckets() { @Override protected T createTestInstance(String name, Map metadata, InternalAggregations aggregations) { final int precision = randomPrecision(); - int size = randomNumberOfBuckets(); - List buckets = new ArrayList<>(size); + final int size = randomNumberOfBuckets(); + final List buckets = new ArrayList<>(size); + final List seen = new ArrayList<>(size); + int finalSize = 0; for (int i = 0; i < size; i++) { double latitude = randomDoubleBetween(-90.0, 90.0, false); double longitude = randomDoubleBetween(-180.0, 180.0, false); long hashAsLong = longEncode(longitude, latitude, precision); - buckets.add(createInternalGeoGridBucket(hashAsLong, randomInt(IndexWriter.MAX_DOCS), aggregations)); + if (seen.contains(hashAsLong) == false) { // make sure we don't add twice the same bucket + buckets.add(createInternalGeoGridBucket(hashAsLong, randomInt(IndexWriter.MAX_DOCS), aggregations)); + seen.add(hashAsLong); + finalSize++; + } } - return createInternalGeoGrid(name, size, buckets, metadata); + return createInternalGeoGrid(name, finalSize, buckets, metadata); } @Override diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridTests.java index 6a2de785c0544..421e014452024 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHexGridTests.java @@ -61,7 +61,6 @@ protected long longEncode(double lng, double lat, int precision) { @Override protected int randomPrecision() { - // precision values below 8 can lead to parsing errors return randomIntBetween(0, H3.MAX_H3_RES); } }