[EnginePlugin][Spark]Turn off use secure random by default (#5197)

* Turn off use secure random by default close #5196 * Update Notification Mailing List * Fix ds meta service build
apache · Nov 12, 2024 · 2fdf9d3 · 2fdf9d3
1 parent 9036be6
commit 2fdf9d3
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 12 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -67,6 +67,6 @@ github:
         required_approving_review_count: 1
 notifications:
   commits: commits@linkis.apache.org
-  issues: notifications@linkis.apache.org
-  pullrequests: notifications@linkis.apache.org
+  issues: dev@linkis.apache.org
+  pullrequests: dev@linkis.apache.org
 
diff --git a/...spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala b/...spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala
@@ -156,7 +156,7 @@ object SparkConfiguration extends Logging {
     CommonVars("wds.linkis.spark.engineconn.fatal.log", "error writing class;OutOfMemoryError")
 
   val PYSPARK_PYTHON3_PATH =
-    CommonVars[String]("pyspark.python3.path", "/appcom/Install/anaconda3/bin/python")
+    CommonVars[String]("pyspark.python3.path", "python3")
 
   val ENABLE_REPLACE_PACKAGE_NAME =
     CommonVars("wds.linkis.spark.engine.scala.replace_package_header.enable", true)
@@ -182,6 +182,9 @@ object SparkConfiguration extends Logging {
 
   val LINKIS_SPARK_ETL_SUPPORT_HUDI = CommonVars[Boolean]("linkis.spark.etl.support.hudi", false)
 
+  val LINKIS_PYSPARK_USE_SECURE_RANDOM =
+    CommonVars[Boolean]("linkis.pyspark.use.secure.random", false).getValue
+
   val SCALA_PARSE_APPEND_CODE =
     CommonVars("linkis.scala.parse.append.code", "val linkisVar=1").getValue
 

diff --git a/...rk/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala b/...rk/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala
@@ -46,6 +46,7 @@ import org.apache.spark.sql.execution.datasources.csv.UDF
 
 import java.io._
 import java.net.InetAddress
+import java.security.SecureRandom
 import java.util
 
 import scala.collection.JavaConverters._
@@ -76,7 +77,12 @@ class SparkPythonExecutor(val sparkEngineSession: SparkEngineSession, val id: In
   private val lineOutputStream = new RsOutputStream
   val sqlContext = sparkEngineSession.sqlContext
   val SUCCESS = "success"
-  private lazy val py4jToken: String = SecureRandomStringUtils.randomAlphanumeric(256)
+
+  private lazy val py4jToken: String = if (SparkConfiguration.LINKIS_PYSPARK_USE_SECURE_RANDOM) {
+    SecureRandomStringUtils.randomAlphanumeric(256)
+  } else {
+    SecureRandom.getInstance("SHA1PRNG").nextInt(100000).toString
+  }
 
   private lazy val gwBuilder: GatewayServerBuilder = {
     val builder = new GatewayServerBuilder()
@@ -152,7 +158,6 @@ class SparkPythonExecutor(val sparkEngineSession: SparkEngineSession, val id: In
     )
     val userDefinePythonVersion = engineCreationContext.getOptions
       .getOrDefault("spark.python.version", "python")
-      .toString
       .toLowerCase()
     val sparkPythonVersion =
       if (

diff --git a/linkis-public-enhancements/distribution.xml b/linkis-public-enhancements/distribution.xml
@@ -290,7 +290,7 @@
     <fileSets>
         <fileSet>
             <directory>
-                ./linkis-datasource/linkis-datasource-manager/server/target/out/lib/service
+                ./linkis-datasource/linkis-datasource-manager/server/target/out/service
             </directory>
             <outputDirectory>
                     lib/metadataquery-service

diff --git a/...nts/linkis-datasource/linkis-datasource-manager/server/src/main/assembly/distribution.xml b/...nts/linkis-datasource/linkis-datasource-manager/server/src/main/assembly/distribution.xml
@@ -187,7 +187,7 @@
         <fileSet>
             <directory>${basedir}/../service/elasticsearch/target/out/lib</directory>
             <fileMode>0755</fileMode>
-            <outputDirectory>lib/service/elasticsearch</outputDirectory>
+            <outputDirectory>service/elasticsearch</outputDirectory>
             <includes>
                 <include>*.jar</include>
             </includes>
@@ -198,7 +198,7 @@
         <fileSet>
             <directory>${basedir}/../service/hive/target/out/lib</directory>
             <fileMode>0755</fileMode>
-            <outputDirectory>lib/service/hive</outputDirectory>
+            <outputDirectory>service/hive</outputDirectory>
             <includes>
                 <include>*.jar</include>
             </includes>
@@ -209,7 +209,7 @@
         <fileSet>
             <directory>${basedir}/../service/kafka/target/out/lib</directory>
             <fileMode>0755</fileMode>
-            <outputDirectory>lib/service/kafka</outputDirectory>
+            <outputDirectory>service/kafka</outputDirectory>
             <includes>
                 <include>*.jar</include>
             </includes>
@@ -220,7 +220,7 @@
         <fileSet>
             <directory>${basedir}/../service/jdbc/target/out/lib</directory>
             <fileMode>0755</fileMode>
-            <outputDirectory>lib/service/jdbc</outputDirectory>
+            <outputDirectory>service/jdbc</outputDirectory>
             <includes>
                 <include>*.jar</include>
             </includes>
@@ -231,7 +231,7 @@
         <fileSet>
             <directory>${basedir}/../service/hdfs/target/out/lib</directory>
             <fileMode>0755</fileMode>
-            <outputDirectory>lib/service/hdfs</outputDirectory>
+            <outputDirectory>service/hdfs</outputDirectory>
             <includes>
                 <include>*.jar</include>
             </includes>
@@ -242,7 +242,7 @@
         <fileSet>
             <directory>${basedir}/../service/mongodb/target/out/lib</directory>
             <fileMode>0755</fileMode>
-            <outputDirectory>lib/service/mongodb</outputDirectory>
+            <outputDirectory>service/mongodb</outputDirectory>
             <includes>
                 <include>*.jar</include>
             </includes>