Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SEDONA-349] Drop rarely used constructors and their tests in Java #954

Merged
merged 11 commits into from
Aug 10, 2023
Merged

[SEDONA-349] Drop rarely used constructors and their tests in Java #954

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
fcd9a48
Drop rarely used constructors in Java
jiayuasu Aug 8, 2023
b17c78f
Drop the constructors in Python
jiayuasu Aug 8, 2023
a192aab
Reduce loop times to 1
jiayuasu Aug 8, 2023
ae1662a
Remove calling storage level
jiayuasu Aug 8, 2023
f65bb7e
Fix the scala code
jiayuasu Aug 8, 2023
c487aef
Fix a bunch of failed test cases
jiayuasu Aug 8, 2023
a7a30a2
Fix a bunch of failed test cases
jiayuasu Aug 8, 2023
86ec83f
Fix more test cases
jiayuasu Aug 8, 2023
4634f88
Fix a bunch of test cases
jiayuasu Aug 9, 2023
dd11837
Revert "Remove calling storage level"
jiayuasu Aug 9, 2023
dbad253
Fix failed test cases
jiayuasu Aug 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 4 additions & 8 deletions R/tests/testthat/test-data-interface.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,7 @@ test_rdd_with_non_spatial_attrs <- invoke_new(
1L, # offset
sc$state$enums$delimiter$csv,
TRUE,
1L, # numPartitions
sc$state$object_cache$storage_levels$memory_only
1L # numPartitions
) %>%
apache.sedona:::new_spatial_rdd("point")

Expand Down Expand Up @@ -575,8 +574,7 @@ test_that("sedona_write_wkb() works as expected", {
0L, # offset
sc$state$enums$delimiter$wkb,
TRUE,
1L, # numPartitions
sc$state$object_cache$storage_levels$memory_only
1L # numPartitions
)

expect_result_matches_original(pt_rdd)
Expand All @@ -593,8 +591,7 @@ test_that("sedona_write_wkt() works as expected", {
0L, # offset
sc$state$enums$delimiter$wkt,
TRUE,
1L, # numPartitions
sc$state$object_cache$storage_levels$memory_only
1L # numPartitions
)

expect_result_matches_original(pt_rdd)
Expand All @@ -611,8 +608,7 @@ test_that("sedona_write_geojson() works as expected", {
0L, # offset
sc$state$enums$delimiter$geojson,
TRUE,
1L, # numPartitions
sc$state$object_cache$storage_levels$memory_only
1L # numPartitions
)

expect_result_matches_original_geojson(pt_rdd)
Expand Down
6 changes: 4 additions & 2 deletions core/src/main/java/org/apache/sedona/core/showcase/EarthdataMapperRunnableExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,8 @@ public static void testSpatialRangeQuery()
{
EarthdataHDFPointMapper earthdataHDFPoint = new EarthdataHDFPointMapper(HDFIncrement, HDFOffset, HDFRootGroupName,
HDFDataVariableList, HDFDataVariableName, urlPrefix);
PointRDD spatialRDD = new PointRDD(sc, InputLocation, numPartitions, earthdataHDFPoint, StorageLevel.MEMORY_ONLY());
PointRDD spatialRDD = new PointRDD(sc, InputLocation, numPartitions, earthdataHDFPoint);
spatialRDD.rawSpatialRDD.persist(StorageLevel.MEMORY_ONLY());
for (int i = 0; i < loopTimes; i++) {
long resultSize;
try {
Expand All @@ -164,14 +165,15 @@ public static void testSpatialRangeQueryUsingIndex()
{
EarthdataHDFPointMapper earthdataHDFPoint = new EarthdataHDFPointMapper(HDFIncrement, HDFOffset, HDFRootGroupName,
HDFDataVariableList, HDFDataVariableName, urlPrefix);
PointRDD spatialRDD = new PointRDD(sc, InputLocation, numPartitions, earthdataHDFPoint, StorageLevel.MEMORY_ONLY());
PointRDD spatialRDD = new PointRDD(sc, InputLocation, numPartitions, earthdataHDFPoint);
try {
spatialRDD.buildIndex(IndexType.RTREE, false);
}
catch (Exception e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
spatialRDD.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY());
for (int i = 0; i < loopTimes; i++) {
try {
long resultSize;
Expand Down
51 changes: 8 additions & 43 deletions core/src/main/java/org/apache/sedona/core/showcase/Example.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,6 @@ public static void main(String[] args)
testSpatialJoinQueryUsingIndex();
testDistanceJoinQuery();
testDistanceJoinQueryUsingIndex();
testCRSTransformationSpatialRangeQuery();
testCRSTransformationSpatialRangeQueryUsingIndex();
testLoadShapefileIntoPolygonRDD();
}
catch (Exception e) {
Expand All @@ -212,7 +210,7 @@ public static void main(String[] args)
public static void testSpatialRangeQuery()
throws Exception
{
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY());
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true);
objectRDD.rawSpatialRDD.persist(StorageLevel.MEMORY_ONLY());
for (int i = 0; i < eachQueryLoopTimes; i++) {
long resultSize = RangeQuery.SpatialRangeQuery(objectRDD, rangeQueryWindow, false, false).count();
Expand All @@ -228,7 +226,7 @@ public static void testSpatialRangeQuery()
public static void testSpatialRangeQueryUsingIndex()
throws Exception
{
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY());
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true);
objectRDD.buildIndex(PointRDDIndexType, false);
objectRDD.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY());
for (int i = 0; i < eachQueryLoopTimes; i++) {
Expand All @@ -245,7 +243,7 @@ public static void testSpatialRangeQueryUsingIndex()
public static void testSpatialKnnQuery()
throws Exception
{
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY());
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true);
objectRDD.rawSpatialRDD.persist(StorageLevel.MEMORY_ONLY());
for (int i = 0; i < eachQueryLoopTimes; i++) {
List<Point> result = KNNQuery.SpatialKnnQuery(objectRDD, kNNQueryPoint, 1000, false);
Expand All @@ -261,7 +259,7 @@ public static void testSpatialKnnQuery()
public static void testSpatialKnnQueryUsingIndex()
throws Exception
{
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY());
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true);
objectRDD.buildIndex(PointRDDIndexType, false);
objectRDD.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY());
for (int i = 0; i < eachQueryLoopTimes; i++) {
Expand All @@ -279,7 +277,7 @@ public static void testSpatialJoinQuery()
throws Exception
{
queryWindowRDD = new PolygonRDD(sc, PolygonRDDInputLocation, PolygonRDDStartOffset, PolygonRDDEndOffset, PolygonRDDSplitter, true);
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY());
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true);

objectRDD.spatialPartitioning(joinQueryPartitioningType);
queryWindowRDD.spatialPartitioning(objectRDD.getPartitioner());
Expand All @@ -301,7 +299,7 @@ public static void testSpatialJoinQueryUsingIndex()
throws Exception
{
queryWindowRDD = new PolygonRDD(sc, PolygonRDDInputLocation, PolygonRDDStartOffset, PolygonRDDEndOffset, PolygonRDDSplitter, true);
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY());
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true);

objectRDD.spatialPartitioning(joinQueryPartitioningType);
queryWindowRDD.spatialPartitioning(objectRDD.getPartitioner());
Expand All @@ -325,7 +323,7 @@ public static void testSpatialJoinQueryUsingIndex()
public static void testDistanceJoinQuery()
throws Exception
{
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY());
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true);
CircleRDD queryWindowRDD = new CircleRDD(objectRDD, 0.1);

objectRDD.spatialPartitioning(GridType.QUADTREE);
Expand All @@ -349,7 +347,7 @@ public static void testDistanceJoinQuery()
public static void testDistanceJoinQueryUsingIndex()
throws Exception
{
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY());
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true);
CircleRDD queryWindowRDD = new CircleRDD(objectRDD, 0.1);

objectRDD.spatialPartitioning(GridType.QUADTREE);
Expand All @@ -366,39 +364,6 @@ public static void testDistanceJoinQueryUsingIndex()
}
}

/**
* Test CRS transformation spatial range query.
*
* @throws Exception the exception
*/
public static void testCRSTransformationSpatialRangeQuery()
throws Exception
{
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY(), "epsg:4326", "epsg:3005");
objectRDD.rawSpatialRDD.persist(StorageLevel.MEMORY_ONLY());
for (int i = 0; i < eachQueryLoopTimes; i++) {
long resultSize = RangeQuery.SpatialRangeQuery(objectRDD, rangeQueryWindow, false, false).count();
assert resultSize > -1;
}
}

/**
* Test CRS transformation spatial range query using index.
*
* @throws Exception the exception
*/
public static void testCRSTransformationSpatialRangeQueryUsingIndex()
throws Exception
{
objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY(), "epsg:4326", "epsg:3005");
objectRDD.buildIndex(PointRDDIndexType, false);
objectRDD.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY());
for (int i = 0; i < eachQueryLoopTimes; i++) {
long resultSize = RangeQuery.SpatialRangeQuery(objectRDD, rangeQueryWindow, false, true).count();
assert resultSize > -1;
}
}

public static void testLoadShapefileIntoPolygonRDD()
throws Exception
{
Expand Down
Loading
Loading