remove dead code from dbscan code

apache · Oct 10, 2024 · 68dba21 · 68dba21
1 parent 678da00
commit 68dba21
Show file tree

Hide file tree

Showing 3 changed files with 1 addition and 6 deletions.
diff --git a/python/sedona/stats/clustering/dbscan.py b/python/sedona/stats/clustering/dbscan.py
@@ -25,7 +25,6 @@
 from pyspark.sql import DataFrame, SparkSession
 
 ID_COLUMN_NAME = "__id"
-DEFAULT_MAX_SAMPLE_SIZE = 1000000  # 1 million
 
 
 def dbscan(

diff --git a/python/tests/stats/test_dbscan.py b/python/tests/stats/test_dbscan.py
@@ -18,7 +18,6 @@
 import pyspark.sql.functions as f
 import pytest
 
-from itertools import product
 from sedona.sql.st_constructors import ST_MakePoint
 from sedona.sql.st_functions import ST_Buffer
 from sklearn.cluster import DBSCAN as sklearnDBSCAN

diff --git a/spark/common/src/main/scala/org/apache/sedona/stats/clustering/DBSCAN.scala b/spark/common/src/main/scala/org/apache/sedona/stats/clustering/DBSCAN.scala
@@ -22,7 +22,7 @@ import org.apache.sedona.stats.Util.getGeometryColumnName
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT
 import org.apache.spark.sql.sedona_sql.expressions.st_functions.{ST_Distance, ST_DistanceSpheroid}
-import org.apache.spark.sql.{Column, DataFrame, SparkSession}
+import org.apache.spark.sql.{Column, DataFrame}
 import org.graphframes.GraphFrame
 
 object DBSCAN {
@@ -60,9 +60,6 @@ object DBSCAN {
       includeOutliers: Boolean = true,
       useSpheroid: Boolean = false): DataFrame = {
 
-    // We want to disable broadcast joins because the broadcast reference were using too much driver memory
-    val spark = SparkSession.getActiveSession.get
-
     val geometryCol = geometry match {
       case null => getGeometryColumnName(dataframe)
       case _ => geometry