Skip to content

Commit

Permalink
Revert "Remove calling storage level"
Browse files Browse the repository at this point in the history
This reverts commit ae1662a.
  • Loading branch information
jiayuasu committed Aug 9, 2023
1 parent 4634f88 commit dd11837
Showing 1 changed file with 22 additions and 0 deletions.
22 changes: 22 additions & 0 deletions R/R/data_interface.R
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ sedona_read_dsv_to_typed_rdd <- function(sc,
max(as.integer(repartition %||% 1L), 1L),
fmt
) %>%
set_storage_level(storage_level) %>%
new_spatial_rdd(type)
}

Expand Down Expand Up @@ -210,6 +211,7 @@ sedona_read_shapefile_to_typed_rdd <- function(sc,
java_context(sc),
location
) %>%
set_storage_level(storage_level) %>%
new_spatial_rdd(type)
}

Expand Down Expand Up @@ -238,6 +240,7 @@ sedona_read_geojson_to_typed_rdd <- function(sc,
has_non_spatial_attrs,
max(as.integer(repartition %||% 1L), 1L)
) %>%
set_storage_level(storage_level) %>%
new_spatial_rdd(type)
}

Expand Down Expand Up @@ -302,6 +305,7 @@ sedona_read_geojson <- function(sc,
allow_invalid_geometries,
skip_syntactically_invalid_geometries
) %>%
set_storage_level(storage_level) %>%
new_spatial_rdd(NULL)
}

Expand Down Expand Up @@ -331,6 +335,7 @@ sedona_read_wkb <- function(sc,
allow_invalid_geometries,
skip_syntactically_invalid_geometries
) %>%
set_storage_level(storage_level) %>%
new_spatial_rdd(NULL)
}

Expand Down Expand Up @@ -360,6 +365,7 @@ sedona_read_wkt <- function(sc,
allow_invalid_geometries,
skip_syntactically_invalid_geometries
) %>%
set_storage_level(storage_level) %>%
new_spatial_rdd(NULL)
}

Expand All @@ -375,6 +381,7 @@ sedona_read_shapefile <- function(sc,
java_context(sc),
location
) %>%
set_storage_level(storage_level) %>%
new_spatial_rdd(NULL)
}

Expand Down Expand Up @@ -793,3 +800,18 @@ to_delimiter_enum_value <- function(sc, delimiter) {

sc$state$enums$delimiter[[delimiter]]
}

set_storage_level <- function(rdd, storage_level) {
sc <- spark_connection(rdd)
storage_level <- sc$state$object_cache$storage_levels[[storage_level]] %||% {
storage_level_obj <- invoke_static(
sc, "org.apache.spark.storage.StorageLevel", storage_level
)
sc$state$object_cache$storage_levels[[storage_level]] <- storage_level_obj

storage_level_obj
}
invoke(rdd, "analyze", storage_level)

rdd
}

0 comments on commit dd11837

Please sign in to comment.