update docs

Signed-off-by: remzi <13716567376yh@gmail.com>
NVIDIA · Sep 7, 2022 · 8d27657 · 8d27657
1 parent 18b571f
commit 8d27657
Show file tree

Hide file tree

Showing 4 changed files with 2 additions and 11 deletions.
diff --git a/docs/compatibility.md b/docs/compatibility.md
@@ -69,18 +69,11 @@ worked with plain Spark. Starting from 22.06 this is behavior is enabled by defa
 the config
 [`spark.rapids.sql.variableFloatAgg.enabled`](configs.md#sql.variableFloatAgg.enabled).
 
-Additionally, some aggregations on floating point columns that contain `NaN` can produce results
-different from Spark in versions prior to Spark 3.1.0.  If it is known with certainty that the
-floating point columns do not contain `NaN`, set
-[`spark.rapids.sql.hasNans`](configs.md#sql.hasNans) to `false` to run GPU enabled aggregations on
-them.
-
 In the case of a distinct count on `NaN` values, prior to Spark 3.1.0, the issue only shows up if
  you have different `NaN` values. There are several different binary values that are all considered
  to be `NaN` by floating point. The plugin treats all of these as the same value, where as Spark
  treats them all as different values. Because this is considered to be rare we do not disable
- distinct count for floating point values even if
- [`spark.rapids.sql.hasNans`](configs.md#sql.hasNans) is `true`.
+ distinct count for floating point values.
 
 ### `0.0` vs `-0.0`
 

diff --git a/docs/get-started/getting-started-gcp.md b/docs/get-started/getting-started-gcp.md
@@ -212,7 +212,7 @@ gcloud dataproc jobs submit spark \
     --region=$REGION \
     --class=com.nvidia.spark.examples.mortgage.GPUMain \
     --jars=gs://${GCS_BUCKET}/scala/sample_xgboost_apps-<version>-jar-with-dependencies.jar \
-    --properties=spark.executor.cores=${SPARK_NUM_CORES_PER_EXECUTOR},spark.task.cpus=${SPARK_NUM_CORES_PER_EXECUTOR},spark.executor.memory=${SPARK_EXECUTOR_MEMORY},spark.executor.memoryOverhead=${SPARK_EXECUTOR_MEMORYOVERHEAD},spark.executor.resource.gpu.amount=1,spark.task.resource.gpu.amount=1,spark.rapids.sql.hasNans=false,spark.rapids.sql.batchSizeBytes=512M,spark.rapids.sql.reader.batchSizeBytes=768M,spark.rapids.sql.variableFloatAgg.enabled=true,spark.rapids.memory.gpu.pooling.enabled=false \
+    --properties=spark.executor.cores=${SPARK_NUM_CORES_PER_EXECUTOR},spark.task.cpus=${SPARK_NUM_CORES_PER_EXECUTOR},spark.executor.memory=${SPARK_EXECUTOR_MEMORY},spark.executor.memoryOverhead=${SPARK_EXECUTOR_MEMORYOVERHEAD},spark.executor.resource.gpu.amount=1,spark.task.resource.gpu.amount=1,spark.rapids.sql.batchSizeBytes=512M,spark.rapids.sql.reader.batchSizeBytes=768M,spark.rapids.sql.variableFloatAgg.enabled=true,spark.rapids.memory.gpu.pooling.enabled=false \
     -- \
     -dataPath=train::${DATA_PATH}/train \
     -dataPath=trans::${DATA_PATH}/test \

diff --git a/docs/spark-profiling-tool.md b/docs/spark-profiling-tool.md
@@ -243,7 +243,6 @@ Compare Rapids Properties which are set explicitly:
 |spark.rapids.sql.concurrentGpuTasks        |null      |2         |
 |spark.rapids.sql.enabled                   |false     |true      |
 |spark.rapids.sql.explain                   |null      |NOT_ON_GPU|
-|spark.rapids.sql.hasNans                   |null      |FALSE     |
 |spark.rapids.sql.incompatibleOps.enabled   |null      |true      |
 +-------------------------------------------+----------+----------+
 ```

diff --git a/...tion_tests/src/test/scala/com/nvidia/spark/rapids/tests/mortgage/MortgageSparkSuite.scala b/...tion_tests/src/test/scala/com/nvidia/spark/rapids/tests/mortgage/MortgageSparkSuite.scala
@@ -41,7 +41,6 @@ class MortgageSparkSuite extends FunSuite {
       .config("spark.rapids.sql.explain", "ALL")
       .config("spark.rapids.sql.test.enabled", false)
       .config("spark.rapids.sql.incompatibleOps.enabled", true)
-      .config("spark.rapids.sql.hasNans", false)
     val rapidsShuffle = ShimLoader.getRapidsShuffleManagerClass
     val prop = System.getProperty("rapids.shuffle.manager.override", "false")
     if (prop.equalsIgnoreCase("true")) {